├── README.md ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── am │ │ └── jlfu │ │ ├── authorizer │ │ ├── Authorizer.java │ │ └── impl │ │ │ └── DefaultAuthorizer.java │ │ ├── fileuploader │ │ ├── exception │ │ │ ├── AuthorizationException.java │ │ │ ├── BadRequestException.java │ │ │ ├── FileCorruptedException.java │ │ │ ├── FileStillProcessingException.java │ │ │ ├── InvalidCrcException.java │ │ │ ├── JavaFileUploaderException.java │ │ │ ├── MissingParameterException.java │ │ │ └── UploadIsCurrentlyDisabled.java │ │ ├── json │ │ │ ├── CRCResult.java │ │ │ ├── FileStateJson.java │ │ │ ├── FileStateJsonBase.java │ │ │ ├── InitializationConfiguration.java │ │ │ ├── PrepareUploadJson.java │ │ │ ├── ProgressJson.java │ │ │ └── SimpleJsonObject.java │ │ ├── limiter │ │ │ ├── RateLimiter.java │ │ │ ├── RateLimiterConfigurationManager.java │ │ │ ├── RequestUploadProcessingConfiguration.java │ │ │ ├── UploadProcessingConfiguration.java │ │ │ ├── UploadProcessingOperation.java │ │ │ └── UploadProcessingOperationManager.java │ │ ├── logic │ │ │ ├── UploadProcessor.java │ │ │ └── UploadServletAsyncProcessor.java │ │ ├── utils │ │ │ ├── CRCHelper.java │ │ │ ├── ClientToFilesMap.java │ │ │ ├── ConditionProvider.java │ │ │ ├── ImportedFilesCleaner.java │ │ │ ├── LimitingList.java │ │ │ ├── ProgressCalculator.java │ │ │ ├── ProgressManager.java │ │ │ ├── RemainingTimeEstimator.java │ │ │ └── UnitConverter.java │ │ └── web │ │ │ ├── UploadServlet.java │ │ │ ├── UploadServletAction.java │ │ │ ├── UploadServletAsync.java │ │ │ ├── UploadServletAsyncListenerAdapter.java │ │ │ ├── UploadServletParameter.java │ │ │ └── utils │ │ │ ├── ExceptionCodeMappingHelper.java │ │ │ ├── FileUploadConfiguration.java │ │ │ ├── FileUploadManagerFilter.java │ │ │ ├── FileUploaderHelper.java │ │ │ └── RequestComponentContainer.java │ │ ├── identifier │ │ ├── IdentifierProvider.java │ │ └── impl │ │ │ └── DefaultIdentifierProvider.java │ │ ├── notifier │ │ ├── JLFUListener.java │ │ ├── JLFUListenerAdapter.java │ │ ├── JLFUListenerPropagator.java │ │ └── utils │ │ │ └── GenericPropagator.java │ │ └── staticstate │ │ ├── FileDeleter.java │ │ ├── JavaLargeFileUploaderService.java │ │ ├── StaticStateDirectoryManager.java │ │ ├── StaticStateIdentifierManager.java │ │ ├── StaticStateManager.java │ │ ├── StaticStateRootFolderProvider.java │ │ └── entities │ │ ├── FileProgressStatus.java │ │ ├── StaticFileState.java │ │ └── StaticStatePersistedOnFileSystemEntity.java ├── resources │ ├── META-INF │ │ └── jlfu-web-fragment-context.xml │ ├── java-large-file-uploader.properties │ └── log4j.properties └── webapp │ ├── WEB-INF │ └── web.xml │ ├── index.html │ └── js │ └── javalargefileuploader.js └── test ├── java └── com │ └── am │ └── jlfu │ ├── authorizer │ └── DefaultAuthorizerTest.java │ ├── fileuploader │ ├── limiter │ │ ├── RateLimiterConfigurationManagerTest.java │ │ ├── RateLimiterTest.java │ │ └── UploadProcessingOperationManagerTest.java │ ├── logic │ │ ├── UploadProcessorTest.java │ │ └── UploadServletAsyncProcessorTest.java │ ├── util │ │ ├── RootFolderProvider.java │ │ └── StaticStateIdentifierManagerForTestProvider.java │ ├── utils │ │ ├── ImportedFilesCleanerTest.java │ │ ├── LimitingListTest.java │ │ ├── ProgressCalculatorTest.java │ │ ├── ProgressManagerTest.java │ │ └── RemainingTimeEstimatorTest.java │ └── web │ │ └── UploadServletTest.java │ ├── notifier │ └── JLFUListenerPropagatorTest.java │ └── staticstate │ ├── FileDeleterTest.java │ ├── StaticStateIdentifierManagerTest.java │ └── StaticStateManagerTest.java └── resources ├── jlfu.test.properties ├── jlfu.test.xml ├── jmeter.xml └── log4j.properties /README.md: -------------------------------------------------------------------------------- 1 | java-large-file-uploader-demo 2 | ============================= 3 | 4 | 上传大文件java demo。 5 | 6 | 说明 7 | http://blog.csdn.net/freewebsys/article/details/41074213 8 | 9 | java-large-file-uploader-demo 10 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Java Large File Uploader Demo 4 | 4.0.0 5 | 6 | 7 | com.am 8 | java-large-file-uploader-parent 9 | 1.1.8 10 | 11 | 12 | java-large-file-uploader-jar 13 | jar 14 | 15 | 16 | 3.0.6.RELEASE 17 | 4.8.2 18 | 11.0 19 | 20 | 21 | 22 | 23 | 24 | 25 | org.springframework 26 | spring-context 27 | ${spring.version} 28 | 29 | 30 | 31 | org.springframework 32 | spring-web 33 | ${spring.version} 34 | 35 | 36 | 37 | org.springframework 38 | spring-test 39 | ${spring.version} 40 | 41 | 42 | 43 | org.springframework 44 | spring-test 45 | ${spring.version} 46 | 47 | 48 | 49 | javax.servlet 50 | javax.servlet-api 51 | 3.1.0 52 | 53 | 54 | 55 | 56 | 57 | junit 58 | junit 59 | ${junit.version} 60 | 61 | 62 | 63 | commons-fileupload 64 | commons-fileupload 65 | 1.2.2 66 | 67 | 68 | 69 | commons-io 70 | commons-io 71 | 2.2 72 | 73 | 74 | 75 | cglib 76 | cglib 77 | 2.2 78 | 79 | 80 | 81 | com.google.code.gson 82 | gson 83 | 1.7.1 84 | 85 | 86 | 87 | joda-time 88 | joda-time 89 | 1.6.2 90 | 91 | 92 | 93 | commons-lang 94 | commons-lang 95 | 2.4 96 | 97 | 98 | 99 | xstream 100 | xstream 101 | 1.2.2 102 | 103 | 104 | 105 | com.google.guava 106 | guava 107 | ${google.guava} 108 | 109 | 110 | 111 | org.slf4j 112 | slf4j-log4j12 113 | 1.6.1 114 | 115 | 116 | 117 | commons-httpclient 118 | commons-httpclient 119 | 3.1 120 | test 121 | 122 | 123 | 124 | org.hamcrest 125 | hamcrest-all 126 | 1.1 127 | test 128 | 129 | 130 | 131 | org.unitils 132 | unitils-easymock 133 | test 134 | 3.3 135 | 136 | 137 | 138 | org.unitils 139 | unitils-mock 140 | test 141 | 3.3 142 | 143 | 144 | 145 | 146 | 147 | 148 | src/main/java 149 | 150 | 151 | src/main/resources 152 | true 153 | 154 | 155 | src/main/webapp/WEB-INF/classes 156 | src/main/webapp/WEB-INF/classes 157 | 158 | 159 | org.apache.maven.plugins 160 | maven-compiler-plugin 161 | 2.3.1 162 | 163 | 1.6 164 | 1.7 165 | utf-8 166 | 167 | 168 | 169 | org.eclipse.jetty 170 | jetty-maven-plugin 171 | 9.0.2.v20130417 172 | 173 | 174 | 10 175 | foo 176 | 9999 177 | 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/authorizer/Authorizer.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.authorizer; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import javax.servlet.http.HttpServletRequest; 7 | 8 | import com.am.jlfu.fileuploader.exception.AuthorizationException; 9 | import com.am.jlfu.fileuploader.web.UploadServletAction; 10 | 11 | 12 | 13 | /** 14 | * Allows or not a user to perform an operation on the JLFU api. 15 | * 16 | * @author antoinem 17 | * 18 | */ 19 | public interface Authorizer { 20 | 21 | /** 22 | * @param request 23 | * the initial servlet request 24 | * @param action 25 | * the action that the client wishes to perform 26 | * @param clientId 27 | * the identifier of the client 28 | * @param optionalFileIds 29 | * if available, the file id(s) 30 | * @throws AuthorizationException 31 | * if the client cannot perform the action on this file 32 | */ 33 | void getAuthorization(HttpServletRequest request, UploadServletAction action, UUID clientId, UUID... optionalFileIds) 34 | throws AuthorizationException; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/authorizer/impl/DefaultAuthorizer.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.authorizer.impl; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import javax.servlet.http.HttpServletRequest; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.stereotype.Component; 11 | 12 | import com.am.jlfu.authorizer.Authorizer; 13 | import com.am.jlfu.fileuploader.exception.AuthorizationException; 14 | import com.am.jlfu.fileuploader.web.UploadServletAction; 15 | 16 | 17 | 18 | /** 19 | * Default {@link Authorizer} that never throws an {@link AuthorizationException}. 20 | * 21 | * @author antoinem 22 | * 23 | */ 24 | @Component 25 | public class DefaultAuthorizer 26 | implements Authorizer { 27 | 28 | private static final Logger log = LoggerFactory.getLogger(DefaultAuthorizer.class); 29 | 30 | @Override 31 | public void getAuthorization(HttpServletRequest request, UploadServletAction action, UUID clientId, UUID... optionalFileId) { 32 | // by default, all calls are authorized 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/AuthorizationException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | 4 | import java.util.Arrays; 5 | import java.util.UUID; 6 | 7 | import com.am.jlfu.fileuploader.web.UploadServletAction; 8 | 9 | 10 | 11 | public class AuthorizationException extends Exception { 12 | 13 | public AuthorizationException(UploadServletAction actionByParameterName, UUID clientId, UUID... optionalFileIds) { 14 | super("User " + clientId + " is not authorized to perform " + actionByParameterName + (optionalFileIds != null ? " on " + Arrays.toString(optionalFileIds) : "")); 15 | } 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/BadRequestException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | 4 | public class BadRequestException extends Exception { 5 | 6 | public BadRequestException(String message) { 7 | super(message); 8 | } 9 | 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/FileCorruptedException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | 4 | public class FileCorruptedException extends Exception{ 5 | 6 | public FileCorruptedException(String string) { 7 | super(string); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/FileStillProcessingException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | import java.util.UUID; 4 | 5 | 6 | public class FileStillProcessingException extends Exception { 7 | 8 | 9 | public FileStillProcessingException(UUID fileId) { 10 | super("The file "+fileId+" is still in a process. AsyncRequest ignored."); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/InvalidCrcException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | public class InvalidCrcException extends Exception { 4 | 5 | public InvalidCrcException(String crc32, String crc) { 6 | super("The file chunk is invalid. Expected "+crc32+" but received "+crc); 7 | } 8 | 9 | 10 | 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/JavaFileUploaderException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | 4 | import com.am.jlfu.fileuploader.web.utils.ExceptionCodeMappingHelper.ExceptionCodeMapping; 5 | 6 | 7 | 8 | /** 9 | * This exception contains a simple identifier ({@link #exceptionIdentifier}) so that the javascript 10 | * can identify it and i18n it. 11 | * 12 | * @author antoinem 13 | * 14 | */ 15 | public class JavaFileUploaderException extends Exception { 16 | 17 | private ExceptionCodeMapping exceptionCodeMapping; 18 | 19 | 20 | 21 | public JavaFileUploaderException() { 22 | } 23 | 24 | 25 | public JavaFileUploaderException(ExceptionCodeMapping exceptionCodeMapping) { 26 | super(); 27 | this.exceptionCodeMapping = exceptionCodeMapping; 28 | } 29 | 30 | 31 | public ExceptionCodeMapping getExceptionCodeMapping() { 32 | return exceptionCodeMapping; 33 | } 34 | 35 | 36 | public void setExceptionCodeMapping(ExceptionCodeMapping exceptionCodeMapping) { 37 | this.exceptionCodeMapping = exceptionCodeMapping; 38 | } 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/MissingParameterException.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | 4 | import com.am.jlfu.fileuploader.web.UploadServletParameter; 5 | 6 | 7 | 8 | public class MissingParameterException extends BadRequestException { 9 | 10 | public MissingParameterException(UploadServletParameter parameter) { 11 | super("The parameter " + parameter.name() + " is missing for this request."); 12 | } 13 | 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/exception/UploadIsCurrentlyDisabled.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.exception; 2 | 3 | import com.am.jlfu.staticstate.JavaLargeFileUploaderService; 4 | 5 | 6 | /** 7 | * Exception thrown if the uploads are not enabled at the moment. 8 | * @see JavaLargeFileUploaderService#enableFileUploader() 9 | * @see JavaLargeFileUploaderService#disableFileUploader() 10 | * @author antoinem 11 | */ 12 | public class UploadIsCurrentlyDisabled extends Exception { 13 | 14 | public UploadIsCurrentlyDisabled() { 15 | super("All uploads are currently suspended."); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/CRCResult.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | 4 | import java.io.Serializable; 5 | 6 | import com.am.jlfu.fileuploader.utils.CRCHelper; 7 | 8 | 9 | 10 | /** 11 | * This class is the result of a method of {@link CRCHelper}. 12 | * It includes the CRC32 value as a string ({@link #value}) and the number of bytes computed ( 13 | * {@link #read}) 14 | * 15 | * @author antoinem 16 | * @see CRCHelper 17 | * 18 | */ 19 | public class CRCResult 20 | implements Serializable { 21 | 22 | 23 | /** 24 | * generated id 25 | */ 26 | private static final long serialVersionUID = 5435020922997235085L; 27 | 28 | private String value; 29 | private int read; 30 | 31 | 32 | 33 | public CRCResult() { 34 | } 35 | 36 | 37 | public String getCrcAsString() { 38 | return value; 39 | } 40 | 41 | 42 | public void setCrcAsString(String crcAsString) { 43 | this.value = crcAsString; 44 | } 45 | 46 | 47 | public int getTotalRead() { 48 | return read; 49 | } 50 | 51 | 52 | public void setTotalRead(int streamLength) { 53 | this.read = streamLength; 54 | } 55 | 56 | 57 | @Override 58 | public int hashCode() { 59 | final int prime = 31; 60 | int result = 1; 61 | result = prime * result + read; 62 | result = prime * result + ((value == null) ? 0 : value.hashCode()); 63 | return result; 64 | } 65 | 66 | 67 | @Override 68 | public boolean equals(Object obj) { 69 | if (this == obj) 70 | return true; 71 | if (obj == null) 72 | return false; 73 | if (getClass() != obj.getClass()) 74 | return false; 75 | CRCResult other = (CRCResult) obj; 76 | if (read != other.read) 77 | return false; 78 | if (value == null) { 79 | if (other.value != null) 80 | return false; 81 | } 82 | else if (!value.equals(other.value)) 83 | return false; 84 | return true; 85 | } 86 | 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/FileStateJson.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | 4 | public class FileStateJson 5 | extends FileStateJsonBase { 6 | 7 | /** 8 | * generated id 9 | */ 10 | private static final long serialVersionUID = 5043865795253104456L; 11 | 12 | /** Specifies whether the file is complete or not. */ 13 | private boolean fileComplete; 14 | 15 | /** Bytes which have been completed. */ 16 | private Long fileCompletionInBytes; 17 | 18 | 19 | 20 | /** 21 | * Default constructor. 22 | */ 23 | public FileStateJson() { 24 | super(); 25 | } 26 | 27 | 28 | public Boolean getFileComplete() { 29 | return fileComplete; 30 | } 31 | 32 | 33 | public Long getFileCompletionInBytes() { 34 | return fileCompletionInBytes; 35 | } 36 | 37 | 38 | public void setFileCompletionInBytes(Long fileCompletionInBytes) { 39 | this.fileCompletionInBytes = fileCompletionInBytes; 40 | } 41 | 42 | 43 | public void setFileComplete(boolean fileComplete) { 44 | this.fileComplete = fileComplete; 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/FileStateJsonBase.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | 4 | import java.io.Serializable; 5 | import java.util.Date; 6 | 7 | 8 | /** 9 | * Shared entity (java/javascript) containing information about a file being uploaded.
10 | * @author antoinem 11 | */ 12 | public class FileStateJsonBase 13 | implements Serializable { 14 | 15 | /** 16 | * generated id 17 | */ 18 | private static final long serialVersionUID = 5043865795253104456L; 19 | 20 | /** The original file name. */ 21 | private String originalFileName; 22 | 23 | /** The original file size */ 24 | private Long originalFileSizeInBytes; 25 | 26 | /** When the file was originally created */ 27 | private Date creationDate; 28 | 29 | /** 30 | * The rate of the client in kilo bytes.
31 | * */ 32 | private Long rateInKiloBytes; 33 | 34 | /** 35 | * Amount of bytes that were correctly validated.
36 | * When resuming an upload, all bytes in the file that have not been validated are revalidated. 37 | */ 38 | private long crcedBytes; 39 | 40 | /** the first chunk crc information */ 41 | private String firstChunkCrc; 42 | 43 | 44 | 45 | /** 46 | * Default constructor. 47 | */ 48 | public FileStateJsonBase() { 49 | super(); 50 | } 51 | 52 | 53 | public String getOriginalFileName() { 54 | return originalFileName; 55 | } 56 | 57 | 58 | public void setOriginalFileName(String originalFileName) { 59 | this.originalFileName = originalFileName; 60 | } 61 | 62 | 63 | public Long getOriginalFileSizeInBytes() { 64 | return originalFileSizeInBytes; 65 | } 66 | 67 | 68 | public void setOriginalFileSizeInBytes(Long originalFileSizeInBytes) { 69 | this.originalFileSizeInBytes = originalFileSizeInBytes; 70 | } 71 | 72 | 73 | public Long getRateInKiloBytes() { 74 | return rateInKiloBytes; 75 | } 76 | 77 | 78 | public void setRateInKiloBytes(Long rateInKiloBytes) { 79 | this.rateInKiloBytes = rateInKiloBytes; 80 | } 81 | 82 | 83 | public Long getCrcedBytes() { 84 | return crcedBytes; 85 | } 86 | 87 | 88 | public void setCrcedBytes(Long crcedBytes) { 89 | this.crcedBytes = crcedBytes; 90 | } 91 | 92 | 93 | public Date getCreationDate() { 94 | return creationDate; 95 | } 96 | 97 | 98 | public void setCreationDate(Date creationDate) { 99 | this.creationDate = creationDate; 100 | } 101 | 102 | 103 | public String getFirstChunkCrc() { 104 | return firstChunkCrc; 105 | } 106 | 107 | 108 | public void setFirstChunkCrc(String firstChunkCrc) { 109 | this.firstChunkCrc = firstChunkCrc; 110 | } 111 | 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/InitializationConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | 4 | import java.io.Serializable; 5 | import java.util.Map; 6 | 7 | 8 | 9 | /** 10 | * The configuration. 11 | * 12 | * @author antoinem 13 | * 14 | */ 15 | public class InitializationConfiguration 16 | implements Serializable { 17 | 18 | /** 19 | * generated id 20 | */ 21 | private static final long serialVersionUID = -6955613223772661218L; 22 | 23 | /** 24 | * The size of the slice in bytes. 25 | */ 26 | private long inByte; 27 | 28 | /** 29 | * The list of the pending files. 30 | */ 31 | private Map pendingFiles; 32 | 33 | 34 | 35 | /** 36 | * Default constructor 37 | */ 38 | public InitializationConfiguration() { 39 | super(); 40 | } 41 | 42 | 43 | public long getInByte() { 44 | return inByte; 45 | } 46 | 47 | 48 | public void setInByte(long inByte) { 49 | this.inByte = inByte; 50 | } 51 | 52 | 53 | public Map getPendingFiles() { 54 | return pendingFiles; 55 | } 56 | 57 | 58 | public void setPendingFiles(Map pendingFiles) { 59 | this.pendingFiles = pendingFiles; 60 | } 61 | 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/PrepareUploadJson.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | 4 | import java.io.Serializable; 5 | 6 | 7 | 8 | public class PrepareUploadJson 9 | implements Serializable { 10 | 11 | /** 12 | * generated id 13 | */ 14 | private static final long serialVersionUID = -7036071811020864930L; 15 | 16 | private Integer tempId; 17 | 18 | private String fileName; 19 | 20 | private Long size; 21 | 22 | private String crc; 23 | 24 | 25 | 26 | /** 27 | * Default constructor. 28 | */ 29 | public PrepareUploadJson() { 30 | super(); 31 | } 32 | 33 | 34 | public Integer getTempId() { 35 | return tempId; 36 | } 37 | 38 | 39 | public void setTempId(Integer tempId) { 40 | this.tempId = tempId; 41 | } 42 | 43 | 44 | public String getFileName() { 45 | return fileName; 46 | } 47 | 48 | 49 | public void setFileName(String fileName) { 50 | this.fileName = fileName; 51 | } 52 | 53 | 54 | public Long getSize() { 55 | return size; 56 | } 57 | 58 | 59 | public void setSize(Long size) { 60 | this.size = size; 61 | } 62 | 63 | 64 | public String getCrc() { 65 | return crc; 66 | } 67 | 68 | 69 | public void setCrc(String crc) { 70 | this.crc = crc; 71 | } 72 | 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/ProgressJson.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | 4 | import java.io.Serializable; 5 | 6 | 7 | 8 | public class ProgressJson 9 | implements Serializable { 10 | 11 | /** 12 | * generated id 13 | */ 14 | private static final long serialVersionUID = -8710522591230352636L; 15 | 16 | protected Float progress; 17 | protected Long uploadRate; 18 | protected Long estimatedRemainingTimeInSeconds; 19 | 20 | 21 | public ProgressJson() { 22 | } 23 | 24 | 25 | /** 26 | * @return the percentage completed. 27 | */ 28 | public Float getProgress() { 29 | return progress; 30 | } 31 | 32 | 33 | public void setProgress(Float progress) { 34 | this.progress = progress; 35 | } 36 | 37 | /** 38 | * @return current file upload rate in byte per second. 39 | */ 40 | public Long getUploadRate() { 41 | return uploadRate; 42 | } 43 | 44 | 45 | public void setUploadRate(Long uploadRate) { 46 | this.uploadRate = uploadRate; 47 | } 48 | 49 | 50 | /** 51 | * @return the estimated remaining time in seconds. 52 | */ 53 | public Long getEstimatedRemainingTimeInSeconds() { 54 | return estimatedRemainingTimeInSeconds; 55 | } 56 | 57 | 58 | 59 | public void setEstimatedRemainingTimeInSeconds(Long estimatedRemainingTimeInSeconds) { 60 | this.estimatedRemainingTimeInSeconds = estimatedRemainingTimeInSeconds; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/json/SimpleJsonObject.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.json; 2 | 3 | import java.io.Serializable; 4 | 5 | 6 | public class SimpleJsonObject 7 | implements Serializable { 8 | 9 | /** 10 | * generated id 11 | */ 12 | private static final long serialVersionUID = 1815625862539981019L; 13 | 14 | /** 15 | * Value. 16 | */ 17 | private String value; 18 | 19 | 20 | 21 | public SimpleJsonObject(String value) { 22 | super(); 23 | this.value = value; 24 | } 25 | 26 | 27 | public SimpleJsonObject() { 28 | super(); 29 | } 30 | 31 | 32 | public String getValue() { 33 | return value; 34 | } 35 | 36 | 37 | public void setValue(String value) { 38 | this.value = value; 39 | } 40 | 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/limiter/RateLimiter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | import java.util.Map.Entry; 5 | import java.util.UUID; 6 | 7 | import org.apache.commons.lang.time.DateUtils; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.scheduling.annotation.Scheduled; 12 | import org.springframework.stereotype.Component; 13 | 14 | 15 | 16 | @Component 17 | public class RateLimiter 18 | { 19 | 20 | 21 | private static final Logger log = LoggerFactory.getLogger(RateLimiter.class); 22 | 23 | @Autowired 24 | RateLimiterConfigurationManager uploadProcessingConfigurationManager; 25 | 26 | @Autowired 27 | UploadProcessingOperationManager uploadProcessingOperationManager; 28 | 29 | /** Number of times the bucket is filled per second. */ 30 | public static final int NUMBER_OF_TIMES_THE_BUCKET_IS_FILLED_PER_SECOND = 10; 31 | public static final long BUCKET_FILLED_EVERY_X_MILLISECONDS = DateUtils.MILLIS_PER_SECOND / NUMBER_OF_TIMES_THE_BUCKET_IS_FILLED_PER_SECOND; 32 | 33 | 34 | 35 | @Scheduled(fixedRate = BUCKET_FILLED_EVERY_X_MILLISECONDS) 36 | public void fillBucket() { 37 | 38 | // first we need to calculate how many uploads are currently being processed 39 | int requestsBeingProcessed = 0; 40 | for (Entry entry : uploadProcessingConfigurationManager.getRequestEntries()) { 41 | requestsBeingProcessed += entry.getValue().isProcessing() ? 1 : 0; 42 | } 43 | log.trace("refilling the upload allowance of the " + requestsBeingProcessed + " uploads being processed"); 44 | 45 | // if we have entries 46 | if (requestsBeingProcessed > 0) { 47 | 48 | // calculate maximum limitation 49 | // and assign it 50 | uploadProcessingOperationManager.getMasterProcessingOperation().setDownloadAllowanceForIteration( 51 | uploadProcessingConfigurationManager.getMaximumOverAllRateInKiloBytes() * 1024 / NUMBER_OF_TIMES_THE_BUCKET_IS_FILLED_PER_SECOND); 52 | 53 | // for all pending operation 54 | for (Entry entry : uploadProcessingOperationManager.getClientsAndRequestsProcessingOperation() 55 | .entrySet()) { 56 | 57 | // process 58 | processEntry(entry); 59 | 60 | } 61 | 62 | } 63 | } 64 | 65 | 66 | private void processEntry(Entry entry) { 67 | 68 | // default per request is set to the maximum, so basically maximum by client 69 | long allowedCapacityPerSecond = uploadProcessingConfigurationManager.getMaximumRatePerClientInKiloBytes() * 1024; 70 | 71 | // extract the configuration element 72 | final RequestUploadProcessingConfiguration requestUploadProcessingConfiguration = 73 | uploadProcessingConfigurationManager.getUploadProcessingConfiguration(entry.getKey()); 74 | 75 | // calculate from the rate in the configuration 76 | Long rateInKiloBytes = requestUploadProcessingConfiguration.getRateInKiloBytes(); 77 | if (rateInKiloBytes != null) { 78 | allowedCapacityPerSecond = (int) (rateInKiloBytes * 1024); 79 | } 80 | 81 | // calculate statistics 82 | final long instantRateInBytes = entry.getValue().getAndResetBytesWritten(); 83 | requestUploadProcessingConfiguration.setInstantRateInBytes(instantRateInBytes); 84 | 85 | // calculate what we can write per iteration 86 | final long allowedCapacityPerIteration = allowedCapacityPerSecond / NUMBER_OF_TIMES_THE_BUCKET_IS_FILLED_PER_SECOND; 87 | 88 | // set it to the rate conf element 89 | entry.getValue().setDownloadAllowanceForIteration(allowedCapacityPerIteration); 90 | 91 | log.trace("giving an allowance of " + allowedCapacityPerIteration + " bytes to " + entry.getKey() + ". (consumed " + 92 | instantRateInBytes + " bytes during previous iteration)"); 93 | 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/limiter/RateLimiterConfigurationManager.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | import java.util.Map.Entry; 5 | import java.util.Set; 6 | import java.util.UUID; 7 | import java.util.concurrent.TimeUnit; 8 | 9 | import javax.annotation.PostConstruct; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.springframework.beans.factory.annotation.Autowired; 14 | import org.springframework.beans.factory.annotation.Value; 15 | import org.springframework.jmx.export.annotation.ManagedAttribute; 16 | import org.springframework.jmx.export.annotation.ManagedResource; 17 | import org.springframework.stereotype.Component; 18 | 19 | import com.am.jlfu.notifier.JLFUListenerPropagator; 20 | import com.am.jlfu.staticstate.JavaLargeFileUploaderService; 21 | import com.am.jlfu.staticstate.StaticStateManager; 22 | import com.am.jlfu.staticstate.entities.StaticFileState; 23 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 24 | import com.google.common.cache.CacheBuilder; 25 | import com.google.common.cache.CacheLoader; 26 | import com.google.common.cache.LoadingCache; 27 | import com.google.common.cache.RemovalCause; 28 | import com.google.common.cache.RemovalListener; 29 | import com.google.common.cache.RemovalNotification; 30 | 31 | 32 | 33 | @Component 34 | @ManagedResource(objectName = "JavaLargeFileUploader:name=rateLimiterConfiguration") 35 | public class RateLimiterConfigurationManager 36 | { 37 | 38 | private static final Logger log = LoggerFactory.getLogger(RateLimiterConfigurationManager.class); 39 | 40 | /** Client is evicted from the map when not accessed for that duration */ 41 | @Value("jlfu{jlfu.rateLimiterConfiguration.evictionTimeInSeconds:120}") 42 | public int clientEvictionTimeInSeconds; 43 | 44 | @Autowired 45 | private JLFUListenerPropagator jlfuListenerPropagator; 46 | 47 | @Autowired 48 | private StaticStateManager staticStateManager; 49 | 50 | @Autowired 51 | private JavaLargeFileUploaderService staticStateManagerService; 52 | 53 | /** the cache containing all configuration for requests and clients */ 54 | LoadingCache configurationMap; 55 | 56 | 57 | 58 | @PostConstruct 59 | private void initMap() { 60 | configurationMap = CacheBuilder.newBuilder() 61 | .removalListener(new RemovalListener() { 62 | 63 | @Override 64 | public void onRemoval(RemovalNotification notification) { 65 | log.debug("removal from requestconfig of " + notification.getKey() + " because of " + notification.getCause()); 66 | remove(notification.getCause(), notification.getKey()); 67 | } 68 | 69 | 70 | }) 71 | .expireAfterAccess(clientEvictionTimeInSeconds, TimeUnit.SECONDS) 72 | .build(new CacheLoader() { 73 | 74 | @Override 75 | public RequestUploadProcessingConfiguration load(UUID arg0) 76 | throws Exception { 77 | return new RequestUploadProcessingConfiguration(); 78 | } 79 | }); 80 | } 81 | 82 | 83 | 84 | private UploadProcessingConfiguration masterProcessingConfiguration = new UploadProcessingConfiguration(); 85 | 86 | // /////////////// 87 | // Configuration// 88 | // /////////////// 89 | 90 | // 10mb/s 91 | @Value("jlfu{jlfu.ratelimiter.maximumRatePerClientInKiloBytes:10240}") 92 | private volatile long maximumRatePerClientInKiloBytes; 93 | 94 | 95 | // 10mb/s 96 | @Value("jlfu{jlfu.ratelimiter.maximumOverAllRateInKiloBytes:10240}") 97 | private volatile long maximumOverAllRateInKiloBytes; 98 | 99 | 100 | 101 | // /////////////// 102 | 103 | 104 | void remove(RemovalCause cause, UUID key) { 105 | 106 | // if expired 107 | if (cause.equals(RemovalCause.EXPIRED)) { 108 | 109 | // check if client id is in state 110 | final StaticStatePersistedOnFileSystemEntity entityIfPresentWithIdentifier = 111 | staticStateManagerService.getEntityIfPresent(key); 112 | 113 | if (entityIfPresentWithIdentifier != null) { 114 | 115 | // if one of the file is not complete, it is not a natural removal 116 | // but a 117 | // timeout!! we propagate the event 118 | for (Entry lavds : entityIfPresentWithIdentifier.getFileStates().entrySet()) { 119 | if (!lavds.getValue().getStaticFileStateJson().getCrcedBytes().equals(lavds.getValue().getStaticFileStateJson() 120 | .getOriginalFileSizeInBytes())) { 121 | log.debug("inactivity detected for client " + key); 122 | jlfuListenerPropagator.getPropagator().onClientInactivity(key, clientEvictionTimeInSeconds); 123 | return; 124 | } 125 | } 126 | log.debug("natural removal for client " + key); 127 | 128 | } 129 | } 130 | } 131 | 132 | 133 | public Set> getRequestEntries() { 134 | return configurationMap.asMap().entrySet(); 135 | } 136 | 137 | 138 | public void reset(UUID fileId) { 139 | final RequestUploadProcessingConfiguration unchecked = configurationMap.getUnchecked(fileId); 140 | unchecked.setProcessing(false); 141 | } 142 | 143 | 144 | public void assignRateToRequest(UUID fileId, Long rateInKiloBytes) { 145 | configurationMap.getUnchecked(fileId).rateInKiloBytes = rateInKiloBytes; 146 | } 147 | 148 | 149 | public Long getUploadState(UUID requestIdentifier) { 150 | return configurationMap.getUnchecked(requestIdentifier).getInstantRateInBytes(); 151 | } 152 | 153 | 154 | public RequestUploadProcessingConfiguration getUploadProcessingConfiguration(UUID uuid) { 155 | return configurationMap.getUnchecked(uuid); 156 | } 157 | 158 | 159 | @ManagedAttribute 160 | public long getMaximumRatePerClientInKiloBytes() { 161 | return maximumRatePerClientInKiloBytes; 162 | } 163 | 164 | 165 | @ManagedAttribute 166 | public void setMaximumRatePerClientInKiloBytes(long maximumRatePerClientInKiloBytes) { 167 | this.maximumRatePerClientInKiloBytes = maximumRatePerClientInKiloBytes; 168 | } 169 | 170 | 171 | @ManagedAttribute 172 | public long getMaximumOverAllRateInKiloBytes() { 173 | return maximumOverAllRateInKiloBytes; 174 | } 175 | 176 | 177 | @ManagedAttribute 178 | public void setMaximumOverAllRateInKiloBytes(long maximumOverAllRateInKiloBytes) { 179 | this.maximumOverAllRateInKiloBytes = maximumOverAllRateInKiloBytes; 180 | } 181 | 182 | 183 | public UploadProcessingConfiguration getMasterProcessingConfiguration() { 184 | return masterProcessingConfiguration; 185 | } 186 | 187 | 188 | public void setClientEvictionTimeInSeconds(int clientEvictionTimeInSeconds) { 189 | this.clientEvictionTimeInSeconds = clientEvictionTimeInSeconds; 190 | } 191 | 192 | 193 | } 194 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/limiter/RequestUploadProcessingConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | public class RequestUploadProcessingConfiguration extends UploadProcessingConfiguration { 5 | 6 | /** 7 | * Boolean specifying whether the upload is processing or not. 8 | */ 9 | private volatile boolean isProcessing; 10 | 11 | /** 12 | * Boolean specifying whether the client uploading the file is telling the server that it should not process the stream read. 13 | */ 14 | private volatile boolean paused; 15 | 16 | 17 | public boolean isProcessing() { 18 | return isProcessing; 19 | } 20 | 21 | 22 | public void setProcessing(boolean isProcessing) { 23 | this.isProcessing = isProcessing; 24 | } 25 | 26 | 27 | public void pause() { 28 | this.paused = true; 29 | } 30 | 31 | public void resume() { 32 | this.paused = false; 33 | } 34 | 35 | public boolean isPaused() { 36 | return this.paused; 37 | } 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/limiter/UploadProcessingConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | public class UploadProcessingConfiguration { 5 | 6 | 7 | /** 8 | * The desired upload rate.
9 | * Can be null (the maxmimum rate is applied). 10 | */ 11 | volatile Long rateInKiloBytes; 12 | 13 | /** 14 | * The statistics. 15 | * 16 | * @return 17 | */ 18 | long instantRateInBytes; 19 | int instantRateInBytesCounter; 20 | Object instantRateInBytesLock = new Object(); 21 | 22 | 23 | 24 | public Long getRateInKiloBytes() { 25 | return rateInKiloBytes; 26 | } 27 | 28 | 29 | void setInstantRateInBytes(long instantRateInBytes) { 30 | synchronized (instantRateInBytesLock) { 31 | this.instantRateInBytesCounter++; 32 | this.instantRateInBytes += instantRateInBytes; 33 | } 34 | } 35 | 36 | 37 | long getInstantRateInBytes() { 38 | int returnValue = 0; 39 | synchronized (instantRateInBytesLock) { 40 | if (instantRateInBytesCounter > 0) { 41 | returnValue = ((int) instantRateInBytes / instantRateInBytesCounter) * RateLimiter.NUMBER_OF_TIMES_THE_BUCKET_IS_FILLED_PER_SECOND; 42 | 43 | // reset every second or so 44 | if (instantRateInBytesCounter > RateLimiter.NUMBER_OF_TIMES_THE_BUCKET_IS_FILLED_PER_SECOND) { 45 | instantRateInBytes = instantRateInBytesCounter = 0; 46 | } 47 | } 48 | } 49 | return returnValue; 50 | } 51 | 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/limiter/UploadProcessingOperation.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | public class UploadProcessingOperation { 5 | 6 | /** 7 | * Specifies the amount of bytes that have been written 8 | * */ 9 | private long bytesWritten; 10 | private Object bytesWrittenLock = new Object(); 11 | 12 | /** 13 | * Specifies the amount of bytes that can be uploaded for an iteration of the refill process 14 | * of {@link RateLimiter} 15 | * */ 16 | private long downloadAllowanceForIteration; 17 | private Object downloadAllowanceForIterationLock = new Object(); 18 | 19 | 20 | 21 | public long getDownloadAllowanceForIteration() { 22 | synchronized (downloadAllowanceForIterationLock) { 23 | return downloadAllowanceForIteration; 24 | } 25 | } 26 | 27 | 28 | void setDownloadAllowanceForIteration(long downloadAllowanceForIteration) { 29 | synchronized (downloadAllowanceForIterationLock) { 30 | this.downloadAllowanceForIteration = downloadAllowanceForIteration; 31 | } 32 | } 33 | 34 | 35 | public long getAndResetBytesWritten() { 36 | synchronized (bytesWrittenLock) { 37 | final long temp = bytesWritten; 38 | bytesWritten = 0; 39 | return temp; 40 | } 41 | } 42 | 43 | 44 | /** 45 | * Specifies the bytes that have been read from the files. 46 | * 47 | * @param bytesConsumed 48 | */ 49 | public void bytesConsumedFromAllowance(long bytesConsumed) { 50 | synchronized (bytesWrittenLock) { 51 | synchronized (downloadAllowanceForIterationLock) { 52 | bytesWritten += bytesConsumed; 53 | downloadAllowanceForIteration -= bytesConsumed; 54 | } 55 | } 56 | } 57 | 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/limiter/UploadProcessingOperationManager.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | import java.util.HashSet; 5 | import java.util.Map; 6 | import java.util.Set; 7 | import java.util.UUID; 8 | import java.util.concurrent.ConcurrentMap; 9 | 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.stereotype.Component; 14 | 15 | import com.am.jlfu.fileuploader.utils.ClientToFilesMap; 16 | import com.google.common.collect.Maps; 17 | 18 | 19 | 20 | @Component 21 | public class UploadProcessingOperationManager { 22 | 23 | private static final Logger log = LoggerFactory.getLogger(UploadProcessingOperationManager.class); 24 | 25 | @Autowired 26 | ClientToFilesMap clientToFilesMap; 27 | 28 | // //////////// 29 | // operation// 30 | // //////////// 31 | 32 | /** Operation for clients and requests. */ 33 | final ConcurrentMap clientsAndRequestsProcessingOperation = Maps.newConcurrentMap(); 34 | 35 | /** Operation for master. */ 36 | final UploadProcessingOperation masterProcessingOperation = new UploadProcessingOperation(); 37 | 38 | 39 | public Map getClientsAndRequestsProcessingOperation() { 40 | return clientsAndRequestsProcessingOperation; 41 | } 42 | 43 | 44 | public void startOperation(UUID clientId, UUID fileId) { 45 | log.debug("starting operation for client "+clientId + " and file "+fileId); 46 | 47 | // create the request one 48 | // XXX are we sure that there is only one there? 49 | clientsAndRequestsProcessingOperation.put(fileId, new UploadProcessingOperation()); 50 | 51 | // get or create the client one 52 | clientsAndRequestsProcessingOperation.putIfAbsent(clientId, new UploadProcessingOperation()); 53 | 54 | // mapping 55 | Set set = clientToFilesMap.get(clientId); 56 | if (set == null) { 57 | set = new HashSet(); 58 | clientToFilesMap.put(clientId, set); 59 | } 60 | synchronized (set) { 61 | set.add(fileId); 62 | } 63 | 64 | } 65 | 66 | 67 | public void stopOperation(UUID clientId, UUID fileId) { 68 | log.debug("stopping operation for client "+clientId + " and file "+fileId); 69 | 70 | // remove from map 71 | clientsAndRequestsProcessingOperation.remove(fileId); 72 | 73 | // remove mapping 74 | Set set = clientToFilesMap.get(clientId); 75 | if (set != null) { 76 | synchronized (set) { 77 | set.remove(fileId); 78 | 79 | // if client is empty, remove client 80 | final boolean noreMoreUploadsForThisClient = set.isEmpty(); 81 | if (noreMoreUploadsForThisClient) { 82 | clientToFilesMap.remove(clientId); 83 | clientsAndRequestsProcessingOperation.remove(clientId); 84 | } 85 | } 86 | } 87 | 88 | } 89 | 90 | 91 | public UploadProcessingOperation getClientProcessingOperation(UUID clientId) { 92 | return clientsAndRequestsProcessingOperation.get(clientId); 93 | } 94 | 95 | 96 | public UploadProcessingOperation getFileProcessingOperation(UUID fileId) { 97 | return clientsAndRequestsProcessingOperation.get(fileId); 98 | } 99 | 100 | 101 | public UploadProcessingOperation getMasterProcessingOperation() { 102 | return masterProcessingOperation; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/CRCHelper.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.zip.CRC32; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.stereotype.Component; 11 | 12 | import com.am.jlfu.fileuploader.json.CRCResult; 13 | import com.am.jlfu.fileuploader.logic.UploadServletAsyncProcessor; 14 | 15 | 16 | 17 | /** 18 | * Helper providing methods to compute crc hash. 19 | * 20 | * @see #getBufferedCrc(InputStream) 21 | * @author antoinem 22 | * 23 | */ 24 | @Component 25 | public class CRCHelper { 26 | 27 | private static final Logger log = LoggerFactory.getLogger(CRCHelper.class); 28 | 29 | 30 | 31 | /** 32 | * Returns a {@link CRCResult} computed with {@link CRC32} from the stream specified as 33 | * parameter. 34 | * 35 | * @param inputStream 36 | * @return {@link CRCResult} 37 | * @throws IOException 38 | */ 39 | public CRCResult getBufferedCrc(InputStream inputStream) 40 | throws IOException { 41 | 42 | byte[] b = new byte[UploadServletAsyncProcessor.SIZE_OF_THE_BUFFER_IN_BYTES]; 43 | int read; 44 | int totalRead = 0; 45 | CRC32 crc32 = new CRC32(); 46 | while ((read = inputStream.read(b)) != -1) { 47 | crc32.update(b, 0, read); 48 | totalRead += read; 49 | } 50 | inputStream.close(); 51 | 52 | CRCResult crcResult = new CRCResult(); 53 | crcResult.setCrcAsString(Long.toHexString(crc32.getValue())); 54 | crcResult.setTotalRead(totalRead); 55 | 56 | log.debug("obtained crc for stream with length " + totalRead + " : " + crcResult.getCrcAsString()); 57 | 58 | return crcResult; 59 | 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/ClientToFilesMap.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import java.util.Map; 4 | import java.util.Set; 5 | import java.util.UUID; 6 | import java.util.concurrent.ConcurrentMap; 7 | 8 | import org.springframework.stereotype.Component; 9 | 10 | import com.google.common.collect.ForwardingMap; 11 | import com.google.common.collect.Maps; 12 | 13 | @Component 14 | public class ClientToFilesMap extends ForwardingMap> { 15 | 16 | /** Maps a client to its current requests */ 17 | private final ConcurrentMap> clientToRequestsMapping = Maps.newConcurrentMap(); 18 | 19 | @Override 20 | protected Map> delegate() { 21 | return clientToRequestsMapping; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/ConditionProvider.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | 4 | public abstract class ConditionProvider { 5 | 6 | public abstract boolean condition(); 7 | 8 | 9 | public void onFail() { 10 | } 11 | 12 | 13 | public void onSuccess() { 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/ImportedFilesCleaner.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | 4 | import java.io.File; 5 | 6 | import org.joda.time.DateTime; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.beans.factory.annotation.Value; 11 | import org.springframework.jmx.export.annotation.ManagedAttribute; 12 | import org.springframework.jmx.export.annotation.ManagedResource; 13 | import org.springframework.stereotype.Component; 14 | 15 | import com.am.jlfu.staticstate.FileDeleter; 16 | import com.am.jlfu.staticstate.StaticStateRootFolderProvider; 17 | 18 | 19 | 20 | /** 21 | * This cleaner shall be regularly invoked to remove files that are outdated on the system.
22 | * That is checked against the last modified time which shall be no more than what is configured. 23 | * 24 | * @author antoinem 25 | */ 26 | @Component 27 | @ManagedResource(objectName = "JavaLargeFileUploader:name=importedFilesCleaner") 28 | public class ImportedFilesCleaner { 29 | 30 | private static final Logger log = LoggerFactory.getLogger(ImportedFilesCleaner.class); 31 | 32 | @Autowired 33 | StaticStateRootFolderProvider rootFolderProvider; 34 | 35 | @Autowired 36 | FileDeleter fileDeleter; 37 | 38 | @Value("jlfu{jlfu.filecleaner.maximumInactivityInHoursBeforeDelete:48}") 39 | volatile Integer maximumInactivityInHoursBeforeDelete; 40 | 41 | 42 | 43 | public void clean() { 44 | log.trace("#####################Started file cleaner job.#####################"); 45 | DateTime pastTime = new DateTime().minusHours(maximumInactivityInHoursBeforeDelete); 46 | File[] listFiles = rootFolderProvider.getRootFolder().listFiles(); 47 | for (File file : listFiles) { 48 | if (pastTime.isAfter(file.lastModified())) { 49 | log.debug("Deleting outdated file: " + file.getName()); 50 | fileDeleter.deleteFile(file); 51 | } 52 | } 53 | log.trace("Finished file cleaner job."); 54 | } 55 | 56 | 57 | @ManagedAttribute 58 | public Integer getMaximumInactivityInHoursBeforeDelete() { 59 | return maximumInactivityInHoursBeforeDelete; 60 | } 61 | 62 | 63 | @ManagedAttribute 64 | public void setMaximumInactivityInHoursBeforeDelete(Integer maximumInactivityInHoursBeforeDelete) { 65 | this.maximumInactivityInHoursBeforeDelete = maximumInactivityInHoursBeforeDelete; 66 | } 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/LimitingList.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.collect.Lists; 6 | 7 | /** 8 | * List that limits to a certain number of items. All items at the end of the list will be removed. 9 | * @author antoinem 10 | * 11 | * @param 12 | */ 13 | public class LimitingList { 14 | 15 | List list = Lists.newArrayList(); 16 | 17 | private int limit; 18 | 19 | public LimitingList(int limit) { 20 | super(); 21 | this.limit = limit; 22 | } 23 | 24 | /** 25 | * Adds an element at the beginning of the list. 26 | * @param element 27 | */ 28 | public void unshift(T element) { 29 | 30 | //unshift 31 | list.add(0,element); 32 | 33 | //process removal 34 | if(list.size() > limit) { 35 | list.remove(limit); 36 | } 37 | } 38 | 39 | 40 | public List getList() { 41 | return list; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/ProgressCalculator.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import java.io.File; 4 | import java.io.FileNotFoundException; 5 | import java.util.UUID; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Component; 11 | 12 | import com.am.jlfu.fileuploader.limiter.RateLimiterConfigurationManager; 13 | import com.am.jlfu.staticstate.JavaLargeFileUploaderService; 14 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 15 | import com.am.jlfu.staticstate.entities.StaticFileState; 16 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 17 | 18 | /** 19 | * Component to calculate the information related to a file currently uploading. 20 | * @author antoinem 21 | * 22 | */ 23 | @Component 24 | public class ProgressCalculator { 25 | 26 | private static final Logger log = LoggerFactory.getLogger(ProgressCalculator.class); 27 | 28 | @Autowired 29 | JavaLargeFileUploaderService javaLargeFileUploaderService; 30 | 31 | @Autowired 32 | RateLimiterConfigurationManager rateLimiterConfigurationManager; 33 | 34 | @Autowired 35 | RemainingTimeEstimator remainingTimeEstimator; 36 | 37 | 38 | /** 39 | * Retrieves the progress of the specified file for the specified client.
40 | * 41 | * 42 | * @param clientId 43 | * @param fileId 44 | * @return 45 | * @throws FileNotFoundException 46 | */ 47 | public FileProgressStatus getProgress(UUID clientId, UUID fileId) throws FileNotFoundException { 48 | // get the file 49 | StaticStatePersistedOnFileSystemEntity model = javaLargeFileUploaderService.getEntityIfPresent(clientId); 50 | //if cannot find the model, return null 51 | if (model == null) { 52 | return null; 53 | } 54 | return processProgress(fileId, model); 55 | 56 | } 57 | 58 | 59 | private FileProgressStatus processProgress(UUID fileId, StaticStatePersistedOnFileSystemEntity model) 60 | throws FileNotFoundException { 61 | StaticFileState fileState = model.getFileStates().get(fileId); 62 | if (fileState == null) { 63 | throw new FileNotFoundException("File with id " + fileId + " not found"); 64 | } 65 | File file = new File(fileState.getAbsoluteFullPathOfUploadedFile()); 66 | 67 | //init returned entity 68 | FileProgressStatus fileProgressStatus = new FileProgressStatus(); 69 | 70 | // compare size of the file to the expected size 71 | Long originalFileSizeInBytes = fileState.getStaticFileStateJson().getOriginalFileSizeInBytes(); 72 | long currentFileSize = file.length(); 73 | Float progress = calculateProgress(currentFileSize, originalFileSizeInBytes).floatValue(); 74 | 75 | //set it 76 | fileProgressStatus.setProgress(progress); 77 | fileProgressStatus.setTotalFileSize(originalFileSizeInBytes); 78 | fileProgressStatus.setBytesUploaded(currentFileSize); 79 | 80 | //set upload rate 81 | fileProgressStatus.setUploadRate(rateLimiterConfigurationManager.getUploadState(fileId)); 82 | 83 | //calculate estimated remaining time 84 | fileProgressStatus.setEstimatedRemainingTimeInSeconds(remainingTimeEstimator.getRemainingTime(fileId, fileProgressStatus, fileProgressStatus.getUploadRate())); 85 | 86 | //log file progress status 87 | log.debug("Calculated progress for file "+fileId+": "+fileProgressStatus); 88 | 89 | return fileProgressStatus; 90 | } 91 | 92 | 93 | 94 | Double calculateProgress(Long currentSize, Long expectedSize) { 95 | double percent = currentSize.doubleValue() / expectedSize.doubleValue() * 100d; 96 | if (percent == 100 && expectedSize - currentSize != 0) { 97 | percent = 99.99d; 98 | } 99 | return percent; 100 | } 101 | 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/ProgressManager.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | 4 | import java.io.FileNotFoundException; 5 | import java.util.Map; 6 | import java.util.Map.Entry; 7 | import java.util.Set; 8 | import java.util.UUID; 9 | 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.scheduling.annotation.Scheduled; 14 | import org.springframework.stereotype.Component; 15 | 16 | import com.am.jlfu.notifier.JLFUListener; 17 | import com.am.jlfu.notifier.JLFUListenerPropagator; 18 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 19 | import com.google.common.collect.Maps; 20 | import com.google.common.collect.Sets; 21 | 22 | 23 | 24 | /** 25 | * Component responsible for advertising the write progress of a specific file to 26 | * {@link JLFUListener}s.
27 | * Every second, it calculates the progress of all the files being uploaded. 28 | * 29 | * @author antoinem 30 | * 31 | */ 32 | @Component 33 | public class ProgressManager { 34 | private static final Logger log = LoggerFactory.getLogger(ProgressManager.class); 35 | 36 | @Autowired 37 | private JLFUListenerPropagator jlfuListenerPropagator; 38 | 39 | @Autowired 40 | private ClientToFilesMap clientToFilesMap; 41 | 42 | @Autowired 43 | private ProgressCalculator progressCalculator; 44 | 45 | /** Internal map. */ 46 | Map fileToProgressInfo = Maps.newHashMap(); 47 | 48 | /** Simple advertiser. */ 49 | ProgressManagerAdvertiser progressManagerAdvertiser = new ProgressManagerAdvertiser(); 50 | 51 | @Scheduled(fixedRate = 1000) 52 | public void calculateProgress() { 53 | 54 | synchronized (fileToProgressInfo) { 55 | 56 | //for all clients 57 | for (Entry> entry : clientToFilesMap.entrySet()) { 58 | 59 | //for all pending upload 60 | Set originSet = entry.getValue(); 61 | Set copySet; 62 | synchronized (originSet) { 63 | copySet = Sets.newHashSet(originSet); 64 | } 65 | for (UUID fileId : copySet) { 66 | 67 | try { 68 | 69 | //calculate its progress 70 | FileProgressStatus newProgress = progressCalculator.getProgress(entry.getKey(), fileId); 71 | 72 | //if progress has successfully been computed 73 | if (newProgress != null) { 74 | 75 | //get from map 76 | FileProgressStatus progressInMap = fileToProgressInfo.get(fileId); 77 | 78 | //if not present in map 79 | //or if present in map but different from previous one 80 | if (progressInMap == null || !progressInMap.getProgress().equals(newProgress.getProgress())) { 81 | 82 | //add to map 83 | fileToProgressInfo.put(fileId, newProgress); 84 | 85 | // and avertise 86 | progressManagerAdvertiser.advertise(entry.getKey(), fileId, newProgress); 87 | 88 | } 89 | } 90 | 91 | } 92 | catch (FileNotFoundException e) { 93 | log.debug("cannot retrieve progress for "+fileId); 94 | } 95 | 96 | } 97 | } 98 | 99 | } 100 | } 101 | 102 | class ProgressManagerAdvertiser { 103 | 104 | void advertise(UUID clientId, UUID fileId, FileProgressStatus newProgress) { 105 | jlfuListenerPropagator.getPropagator().onFileUploadProgress(clientId, fileId, newProgress); 106 | } 107 | } 108 | 109 | /** 110 | * Returns a calculated progress of a pending file upload.
111 | * @param fileId 112 | * @return 113 | */ 114 | public FileProgressStatus getProgress(UUID fileId) { 115 | synchronized (fileToProgressInfo) { 116 | return fileToProgressInfo.get(fileId); 117 | } 118 | } 119 | 120 | 121 | 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/RemainingTimeEstimator.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | import java.util.UUID; 6 | 7 | import org.springframework.stereotype.Component; 8 | 9 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 10 | import com.google.common.collect.Maps; 11 | 12 | /** 13 | * Component dedicated to calculate the remaining time of a pending upload. 14 | * @author antoinem 15 | */ 16 | @Component 17 | public class RemainingTimeEstimator { 18 | 19 | private static final int averageUploadRateOnTheLastX = 10; 20 | 21 | private Map> map = Maps.newConcurrentMap(); 22 | 23 | public Long getRemainingTime(UUID fileId, FileProgressStatus progress, Long uploadRate) { 24 | LimitingList newArrayList; 25 | 26 | //if we dont have an array for this file yet 27 | if ((newArrayList = map.get(fileId)) == null) { 28 | 29 | //create it and set it 30 | newArrayList = new LimitingList(averageUploadRateOnTheLastX); 31 | map.put(fileId, newArrayList); 32 | 33 | } 34 | 35 | //add the instant upload rate 36 | newArrayList.unshift(uploadRate); 37 | 38 | //calculate the average upload rate 39 | Long averageUploadRate = getAverageUploadRate(newArrayList); 40 | 41 | //return null if average upload rate is 0 42 | if (averageUploadRate == 0) { 43 | return null; 44 | } 45 | 46 | //calculate from average 47 | return calculateRemainingTime(progress, averageUploadRate); 48 | } 49 | 50 | private Long getAverageUploadRate(LimitingList newArrayList) { 51 | Long totalValue = 0l; 52 | List list = newArrayList.getList(); 53 | for (Long value : list) { 54 | totalValue += value; 55 | } 56 | return totalValue / list.size(); 57 | } 58 | 59 | long calculateRemainingTime(FileProgressStatus progress, Long uploadRate) { 60 | long calculatedTimeRemaining = (progress.getTotalFileSize() - progress.getBytesUploaded()) / uploadRate; 61 | //set the minimum to 1second remaining 62 | return Math.max(calculatedTimeRemaining, 1); 63 | } 64 | 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/utils/UnitConverter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | 4 | /** 5 | * Provdes static unit conversion methods.
6 | * A javascript method is also contained in javalargefileuploader.js 7 | * 8 | * @author antoinem 9 | */ 10 | public final class UnitConverter { 11 | 12 | public static String getFormattedTime(long secs) 13 | { 14 | if (secs < 1) { 15 | return "-"; 16 | } 17 | 18 | double hours = Math.floor(secs / (60 * 60)); 19 | 20 | double divisor_for_minutes = secs % (60 * 60); 21 | double minutes = Math.floor(divisor_for_minutes / 60); 22 | 23 | double divisor_for_seconds = divisor_for_minutes % 60; 24 | double seconds = Math.ceil(divisor_for_seconds); 25 | 26 | String returned = ""; 27 | boolean displaySeconds = true; 28 | if (hours > 0) { 29 | returned += ((int)hours) + "h"; 30 | displaySeconds = false; 31 | } 32 | if (minutes > 0) { 33 | returned += ((int)minutes) + "m"; 34 | displaySeconds &= minutes <= 10; 35 | } 36 | if (displaySeconds) { 37 | returned += ((int)seconds) + "s"; 38 | } 39 | return returned; 40 | } 41 | 42 | 43 | public static String getFormattedSize(long size) { 44 | if (size < 1024) { 45 | return format(size) + "B"; 46 | } 47 | else if (size < 1048576) { 48 | return format(size / 1024f) + "KB"; 49 | } 50 | else if (size < 1073741824) { 51 | return format(size / 1048576f) + "MB"; 52 | } 53 | else if (size < 1099511627776l) { 54 | return format(size / 1073741824f) + "GB"; 55 | } 56 | else if (size < 1125899906842624l) { 57 | return format(size / 1099511627776f) + "TB"; 58 | } 59 | return null; 60 | } 61 | 62 | 63 | public static float format(float f) { 64 | return ((float)Math.ceil(f * 100)) / 100f; 65 | } 66 | 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/UploadServlet.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web; 2 | 3 | 4 | import java.io.FileNotFoundException; 5 | import java.io.IOException; 6 | import java.io.Serializable; 7 | import java.util.Arrays; 8 | import java.util.Collection; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.UUID; 12 | 13 | import javax.servlet.annotation.WebServlet; 14 | import javax.servlet.http.HttpServletRequest; 15 | import javax.servlet.http.HttpServletResponse; 16 | 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.stereotype.Component; 21 | import org.springframework.web.HttpRequestHandler; 22 | import org.springframework.web.context.support.HttpRequestHandlerServlet; 23 | 24 | import com.am.jlfu.authorizer.Authorizer; 25 | import com.am.jlfu.fileuploader.exception.AuthorizationException; 26 | import com.am.jlfu.fileuploader.exception.FileCorruptedException; 27 | import com.am.jlfu.fileuploader.exception.FileStillProcessingException; 28 | import com.am.jlfu.fileuploader.exception.InvalidCrcException; 29 | import com.am.jlfu.fileuploader.exception.MissingParameterException; 30 | import com.am.jlfu.fileuploader.json.PrepareUploadJson; 31 | import com.am.jlfu.fileuploader.json.ProgressJson; 32 | import com.am.jlfu.fileuploader.logic.UploadProcessor; 33 | import com.am.jlfu.fileuploader.web.utils.ExceptionCodeMappingHelper; 34 | import com.am.jlfu.fileuploader.web.utils.FileUploaderHelper; 35 | import com.am.jlfu.staticstate.StaticStateIdentifierManager; 36 | import com.google.common.base.Function; 37 | import com.google.common.collect.Collections2; 38 | import com.google.common.collect.Lists; 39 | import com.google.common.collect.Maps; 40 | import com.google.gson.Gson; 41 | 42 | 43 | 44 | /** 45 | * Uploads the file from the jquery uploader. 46 | * 47 | * @author antoinem 48 | * 49 | */ 50 | @Component("javaLargeFileUploaderServlet") 51 | @WebServlet(name = "javaLargeFileUploaderServlet", urlPatterns = { "/javaLargeFileUploaderServlet" }) 52 | public class UploadServlet extends HttpRequestHandlerServlet 53 | implements HttpRequestHandler { 54 | 55 | private static final Logger log = LoggerFactory.getLogger(UploadServlet.class); 56 | 57 | @Autowired 58 | UploadProcessor uploadProcessor; 59 | 60 | @Autowired 61 | FileUploaderHelper fileUploaderHelper; 62 | 63 | @Autowired 64 | ExceptionCodeMappingHelper exceptionCodeMappingHelper; 65 | 66 | @Autowired 67 | Authorizer authorizer; 68 | 69 | @Autowired 70 | StaticStateIdentifierManager staticStateIdentifierManager; 71 | 72 | 73 | 74 | @Override 75 | public void handleRequest(HttpServletRequest request, HttpServletResponse response) 76 | throws IOException { 77 | log.trace("Handling request"); 78 | 79 | Serializable jsonObject = null; 80 | try { 81 | // extract the action from the request 82 | UploadServletAction actionByParameterName = 83 | UploadServletAction.valueOf(fileUploaderHelper.getParameterValue(request, UploadServletParameter.action)); 84 | 85 | // check authorization 86 | checkAuthorization(request, actionByParameterName); 87 | 88 | // then process the asked action 89 | jsonObject = processAction(actionByParameterName, request); 90 | 91 | 92 | // if something has to be written to the response 93 | if (jsonObject != null) { 94 | fileUploaderHelper.writeToResponse(jsonObject, response); 95 | } 96 | 97 | } 98 | // If exception, write it 99 | catch (Exception e) { 100 | exceptionCodeMappingHelper.processException(e, response); 101 | } 102 | 103 | } 104 | 105 | 106 | private void checkAuthorization(HttpServletRequest request, UploadServletAction actionByParameterName) 107 | throws MissingParameterException, AuthorizationException { 108 | 109 | // check authorization 110 | // if its not get progress (because we do not really care about authorization for get 111 | // progress and it uses an array of file ids) 112 | if (!actionByParameterName.equals(UploadServletAction.getProgress)) { 113 | 114 | // extract uuid 115 | final String fileIdFieldValue = fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId, false); 116 | 117 | // if this is init, the identifier is the one in parameter 118 | UUID clientOrJobId; 119 | String parameter = fileUploaderHelper.getParameterValue(request, UploadServletParameter.clientId, false); 120 | if (actionByParameterName.equals(UploadServletAction.getConfig) && parameter != null) { 121 | clientOrJobId = UUID.fromString(parameter); 122 | } 123 | // if not, get it from manager 124 | else { 125 | clientOrJobId = staticStateIdentifierManager.getIdentifier(); 126 | } 127 | 128 | 129 | // call authorizer 130 | authorizer.getAuthorization( 131 | request, 132 | actionByParameterName, 133 | clientOrJobId, 134 | fileIdFieldValue != null ? getFileIdsFromString(fileIdFieldValue).toArray(new UUID[] {}) : null); 135 | 136 | } 137 | } 138 | 139 | 140 | private Serializable processAction(UploadServletAction actionByParameterName, HttpServletRequest request) 141 | throws Exception { 142 | log.debug("Processing action " + actionByParameterName.name()); 143 | 144 | Serializable returnObject = null; 145 | switch (actionByParameterName) { 146 | case getConfig: 147 | String parameterValue = fileUploaderHelper.getParameterValue(request, UploadServletParameter.clientId, false); 148 | returnObject = 149 | uploadProcessor.getConfig( 150 | parameterValue != null ? UUID.fromString(parameterValue) : null); 151 | break; 152 | case verifyCrcOfUncheckedPart: 153 | returnObject = verifyCrcOfUncheckedPart(request); 154 | break; 155 | case prepareUpload: 156 | returnObject = prepareUpload(request); 157 | break; 158 | case clearFile: 159 | uploadProcessor.clearFile(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId))); 160 | break; 161 | case clearAll: 162 | uploadProcessor.clearAll(); 163 | break; 164 | case pauseFile: 165 | List uuids = getFileIdsFromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)); 166 | uploadProcessor.pauseFile(uuids); 167 | break; 168 | case resumeFile: 169 | returnObject = 170 | uploadProcessor.resumeFile(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId))); 171 | break; 172 | case setRate: 173 | uploadProcessor.setUploadRate(UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)), 174 | Long.valueOf(fileUploaderHelper.getParameterValue(request, UploadServletParameter.rate))); 175 | break; 176 | case getProgress: 177 | returnObject = getProgress(request); 178 | break; 179 | } 180 | return returnObject; 181 | } 182 | 183 | 184 | List getFileIdsFromString(String fileIds) { 185 | String[] splittedFileIds = fileIds.split(","); 186 | List uuids = Lists.newArrayList(); 187 | for (int i = 0; i < splittedFileIds.length; i++) { 188 | uuids.add(UUID.fromString(splittedFileIds[i])); 189 | } 190 | return uuids; 191 | } 192 | 193 | 194 | private Serializable getProgress(HttpServletRequest request) 195 | throws MissingParameterException { 196 | Serializable returnObject; 197 | String[] ids = 198 | new Gson() 199 | .fromJson(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId), String[].class); 200 | Collection uuids = Collections2.transform(Arrays.asList(ids), new Function() { 201 | 202 | @Override 203 | public UUID apply(String input) { 204 | return UUID.fromString(input); 205 | } 206 | 207 | }); 208 | returnObject = Maps.newHashMap(); 209 | for (UUID fileId : uuids) { 210 | try { 211 | ProgressJson progress = uploadProcessor.getProgress(fileId); 212 | ((HashMap) returnObject).put(fileId.toString(), progress); 213 | } 214 | catch (FileNotFoundException e) { 215 | log.debug("No progress will be retrieved for " + fileId + " because " + e.getMessage()); 216 | } 217 | } 218 | return returnObject; 219 | } 220 | 221 | 222 | private Serializable prepareUpload(HttpServletRequest request) 223 | throws MissingParameterException, IOException { 224 | 225 | // extract file information 226 | PrepareUploadJson[] fromJson = 227 | new Gson() 228 | .fromJson(fileUploaderHelper.getParameterValue(request, UploadServletParameter.newFiles), PrepareUploadJson[].class); 229 | 230 | // prepare them 231 | final HashMap prepareUpload = uploadProcessor.prepareUpload(fromJson); 232 | 233 | // return them 234 | return Maps.newHashMap(Maps.transformValues(prepareUpload, new Function() { 235 | 236 | public String apply(UUID input) { 237 | return input.toString(); 238 | }; 239 | })); 240 | } 241 | 242 | 243 | private Boolean verifyCrcOfUncheckedPart(HttpServletRequest request) 244 | throws IOException, MissingParameterException, FileCorruptedException, FileStillProcessingException { 245 | UUID fileId = UUID.fromString(fileUploaderHelper.getParameterValue(request, UploadServletParameter.fileId)); 246 | try { 247 | uploadProcessor.verifyCrcOfUncheckedPart(fileId, 248 | fileUploaderHelper.getParameterValue(request, UploadServletParameter.crc)); 249 | } 250 | catch (InvalidCrcException e) { 251 | // no need to log this exception, a fallback behaviour is defined in the 252 | // throwing method. 253 | // but we need to return something! 254 | return Boolean.FALSE; 255 | } 256 | return Boolean.TRUE; 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/UploadServletAction.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web; 2 | 3 | 4 | /** 5 | * One of the possible action that the servlet handles. 6 | * 7 | * @author antoinem 8 | * 9 | */ 10 | public enum UploadServletAction { 11 | 12 | getConfig, 13 | getProgress, 14 | prepareUpload, 15 | clearFile, 16 | setRate, 17 | resumeFile, 18 | pauseFile, 19 | verifyCrcOfUncheckedPart, 20 | clearAll, 21 | upload; 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/UploadServletAsync.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web; 2 | 3 | 4 | import java.io.FileNotFoundException; 5 | import java.io.IOException; 6 | import java.util.UUID; 7 | 8 | import javax.servlet.AsyncContext; 9 | import javax.servlet.ServletException; 10 | import javax.servlet.annotation.WebServlet; 11 | import javax.servlet.http.HttpServletRequest; 12 | import javax.servlet.http.HttpServletResponse; 13 | 14 | import org.apache.commons.lang.time.DateUtils; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | import org.springframework.beans.factory.annotation.Autowired; 18 | import org.springframework.stereotype.Component; 19 | import org.springframework.web.HttpRequestHandler; 20 | import org.springframework.web.context.support.HttpRequestHandlerServlet; 21 | 22 | import com.am.jlfu.authorizer.Authorizer; 23 | import com.am.jlfu.fileuploader.exception.UploadIsCurrentlyDisabled; 24 | import com.am.jlfu.fileuploader.logic.UploadServletAsyncProcessor; 25 | import com.am.jlfu.fileuploader.logic.UploadServletAsyncProcessor.WriteChunkCompletionListener; 26 | import com.am.jlfu.fileuploader.web.utils.ExceptionCodeMappingHelper; 27 | import com.am.jlfu.fileuploader.web.utils.FileUploadConfiguration; 28 | import com.am.jlfu.fileuploader.web.utils.FileUploaderHelper; 29 | import com.am.jlfu.staticstate.StaticStateIdentifierManager; 30 | import com.am.jlfu.staticstate.StaticStateManager; 31 | import com.am.jlfu.staticstate.entities.StaticFileState; 32 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 33 | 34 | 35 | 36 | @Component("javaLargeFileUploaderAsyncServlet") 37 | @WebServlet(name = "javaLargeFileUploaderAsyncServlet", urlPatterns = { "/javaLargeFileUploaderAsyncServlet" }, asyncSupported = true) 38 | public class UploadServletAsync extends HttpRequestHandlerServlet 39 | implements HttpRequestHandler { 40 | 41 | private static final Logger log = LoggerFactory.getLogger(UploadServletAsync.class); 42 | 43 | @Autowired 44 | ExceptionCodeMappingHelper exceptionCodeMappingHelper; 45 | 46 | @Autowired 47 | UploadServletAsyncProcessor uploadServletAsyncProcessor; 48 | 49 | @Autowired 50 | StaticStateIdentifierManager staticStateIdentifierManager; 51 | 52 | @Autowired 53 | StaticStateManager staticStateManager; 54 | 55 | @Autowired 56 | FileUploaderHelper fileUploaderHelper; 57 | 58 | @Autowired 59 | Authorizer authorizer; 60 | 61 | /** 62 | * Maximum time that a streaming request can take.
63 | */ 64 | private long taskTimeOut = DateUtils.MILLIS_PER_HOUR; 65 | 66 | 67 | @Override 68 | public void handleRequest(final HttpServletRequest request, final HttpServletResponse response) 69 | throws ServletException, IOException { 70 | 71 | // process the request 72 | try { 73 | 74 | //check if uploads are allowed 75 | if (!uploadServletAsyncProcessor.isEnabled()) { 76 | throw new UploadIsCurrentlyDisabled(); 77 | } 78 | 79 | // extract stuff from request 80 | final FileUploadConfiguration process = fileUploaderHelper.extractFileUploadConfiguration(request); 81 | 82 | log.debug("received upload request with config: "+process); 83 | 84 | // verify authorization 85 | final UUID clientId = staticStateIdentifierManager.getIdentifier(); 86 | authorizer.getAuthorization(request, UploadServletAction.upload, clientId, process.getFileId()); 87 | 88 | //check if that file is not paused 89 | if (uploadServletAsyncProcessor.isFilePaused(process.getFileId())) { 90 | log.debug("file "+process.getFileId()+" is paused, ignoring async request."); 91 | return; 92 | } 93 | 94 | // get the model 95 | StaticFileState fileState = staticStateManager.getEntityIfPresent().getFileStates().get(process.getFileId()); 96 | if (fileState == null) { 97 | throw new FileNotFoundException("File with id " + process.getFileId() + " not found"); 98 | } 99 | 100 | // process the request asynchronously 101 | final AsyncContext asyncContext = request.startAsync(); 102 | asyncContext.setTimeout(taskTimeOut); 103 | 104 | 105 | // add a listener to clear bucket and close inputstream when process is complete or 106 | // with 107 | // error 108 | asyncContext.addListener(new UploadServletAsyncListenerAdapter(process.getFileId()) { 109 | 110 | @Override 111 | void clean() { 112 | log.debug("request " + request + " completed."); 113 | // we do not need to clear the inputstream here. 114 | // and tell processor to clean its shit! 115 | uploadServletAsyncProcessor.clean(clientId, process.getFileId()); 116 | } 117 | }); 118 | 119 | // then process 120 | uploadServletAsyncProcessor.process(fileState, process.getFileId(), process.getCrc(), process.getInputStream(), 121 | new WriteChunkCompletionListener() { 122 | 123 | @Override 124 | public void success() { 125 | asyncContext.complete(); 126 | } 127 | 128 | 129 | @Override 130 | public void error(Exception exception) { 131 | // handles a stream ended unexpectedly , it just means the user has 132 | // stopped the 133 | // stream 134 | if (exception.getMessage() != null) { 135 | if (exception.getMessage().equals("Stream ended unexpectedly")) { 136 | log.warn("User has stopped streaming for file " + process.getFileId()); 137 | } 138 | else if (exception.getMessage().equals("User cancellation")) { 139 | log.warn("User has cancelled streaming for file id " + process.getFileId()); 140 | // do nothing 141 | } 142 | else { 143 | exceptionCodeMappingHelper.processException(exception, response); 144 | } 145 | } 146 | else { 147 | exceptionCodeMappingHelper.processException(exception, response); 148 | } 149 | 150 | asyncContext.complete(); 151 | } 152 | 153 | }); 154 | } 155 | catch (Exception e) { 156 | exceptionCodeMappingHelper.processException(e, response); 157 | } 158 | 159 | } 160 | 161 | } 162 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/UploadServletAsyncListenerAdapter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web; 2 | 3 | 4 | import java.io.IOException; 5 | import java.util.UUID; 6 | 7 | import javax.servlet.AsyncEvent; 8 | import javax.servlet.AsyncListener; 9 | 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | 14 | 15 | public abstract class UploadServletAsyncListenerAdapter 16 | implements AsyncListener { 17 | 18 | private static final Logger log = LoggerFactory.getLogger(UploadServletAsyncListenerAdapter.class); 19 | 20 | private UUID id; 21 | 22 | 23 | 24 | public UploadServletAsyncListenerAdapter(UUID identifier) { 25 | this.id = identifier; 26 | } 27 | 28 | 29 | abstract void clean(); 30 | 31 | 32 | @Override 33 | public void onComplete(AsyncEvent asyncEvent) 34 | throws IOException { 35 | log.debug("Done: ({})", id); 36 | clean(); 37 | } 38 | 39 | 40 | @Override 41 | public void onTimeout(AsyncEvent asyncEvent) 42 | throws IOException { 43 | log.warn("Asynchronous request timeout ({})", id); 44 | clean(); 45 | } 46 | 47 | 48 | @Override 49 | public void onError(AsyncEvent asyncEvent) 50 | throws IOException { 51 | log.error("Asynchronous request error (" + id + ")", asyncEvent.getThrowable()); 52 | clean(); 53 | } 54 | 55 | 56 | @Override 57 | public void onStartAsync(AsyncEvent asyncEvent) 58 | throws IOException { 59 | log.debug("Started: ({})", id); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/UploadServletParameter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web; 2 | 3 | 4 | /** 5 | * One of the possible parameter that the servlet handles. 6 | * 7 | * @author antoinem 8 | * 9 | */ 10 | public enum UploadServletParameter { 11 | 12 | action, 13 | fileId, 14 | crc, 15 | rate, 16 | newFiles, 17 | clientId; 18 | 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/utils/ExceptionCodeMappingHelper.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web.utils; 2 | 3 | 4 | import java.io.IOException; 5 | 6 | import javax.servlet.http.HttpServletResponse; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.stereotype.Component; 12 | 13 | import com.am.jlfu.fileuploader.exception.AuthorizationException; 14 | import com.am.jlfu.fileuploader.exception.FileCorruptedException; 15 | import com.am.jlfu.fileuploader.exception.FileStillProcessingException; 16 | import com.am.jlfu.fileuploader.exception.InvalidCrcException; 17 | import com.am.jlfu.fileuploader.exception.JavaFileUploaderException; 18 | import com.am.jlfu.fileuploader.exception.MissingParameterException; 19 | import com.am.jlfu.fileuploader.exception.UploadIsCurrentlyDisabled; 20 | 21 | 22 | 23 | /** 24 | * Contains the mapping of exception => ids 25 | * 26 | * @author antoinem 27 | * @see JavaFileUploaderException 28 | * 29 | */ 30 | @Component 31 | public class ExceptionCodeMappingHelper { 32 | 33 | @Autowired 34 | private FileUploaderHelper fileUploaderHelper; 35 | 36 | private static final Logger log = LoggerFactory.getLogger(FileUploaderHelper.class); 37 | 38 | 39 | 40 | public enum ExceptionCodeMapping { 41 | 42 | unkownError (0), 43 | requestIsNotMultipart (1), 44 | NoFileToUploadInTheRequest (2), 45 | InvalidCRC (3, InvalidCrcException.class), 46 | MissingParameterException (4, MissingParameterException.class), 47 | AuthorizationException (12, AuthorizationException.class), 48 | FileCorruptedException (14, FileCorruptedException.class), 49 | FileStillProcessingException (15, FileStillProcessingException.class), 50 | UploadIsCurrentlyDisabled (16, UploadIsCurrentlyDisabled.class); 51 | 52 | private int exceptionIdentifier; 53 | private Class clazz; 54 | 55 | 56 | 57 | private ExceptionCodeMapping(int exceptionIdentifier) { 58 | this.exceptionIdentifier = exceptionIdentifier; 59 | } 60 | 61 | 62 | private ExceptionCodeMapping(int exceptionIdentifier, Class clazz) { 63 | this(exceptionIdentifier); 64 | this.clazz = clazz; 65 | } 66 | 67 | 68 | public int getExceptionIdentifier() { 69 | return exceptionIdentifier; 70 | } 71 | 72 | 73 | public void setExceptionIdentifier(int exceptionIdentifier) { 74 | this.exceptionIdentifier = exceptionIdentifier; 75 | } 76 | 77 | 78 | } 79 | 80 | 81 | 82 | public void processException(Exception e, HttpServletResponse response) { 83 | ExceptionCodeMapping exceptionCodeMappingByType = ExceptionCodeMappingHelper.getExceptionCodeMappingByType(e); 84 | 85 | // log 86 | if (exceptionCodeMappingByType.equals(ExceptionCodeMapping.unkownError)) { 87 | // with stacktrace if it is unknown 88 | log.error(e.getMessage(), e); 89 | } 90 | else { 91 | // without stracktrace if it is managed 92 | log.error(e.getMessage()); 93 | } 94 | 95 | // write exception to response 96 | if (exceptionCodeMappingByType != null) { 97 | try { 98 | log.error("managed error " + exceptionCodeMappingByType.getExceptionIdentifier() + ": " + e.getMessage()); 99 | fileUploaderHelper.writeExceptionToResponse(new JavaFileUploaderException(exceptionCodeMappingByType), response); 100 | } 101 | catch (IOException ee) { 102 | log.error(ee.getMessage()); 103 | } 104 | } 105 | } 106 | 107 | 108 | public static ExceptionCodeMapping getExceptionCodeMappingByType(Exception e) { 109 | if (e instanceof JavaFileUploaderException) { 110 | return ((JavaFileUploaderException) e).getExceptionCodeMapping(); 111 | } 112 | else { 113 | for (ExceptionCodeMapping exceptionsCodeMapping : ExceptionCodeMapping.values()) { 114 | if (exceptionsCodeMapping.clazz != null && exceptionsCodeMapping.clazz.isInstance(e)) { 115 | return exceptionsCodeMapping; 116 | } 117 | } 118 | } 119 | return ExceptionCodeMapping.unkownError; 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/utils/FileUploadConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web.utils; 2 | 3 | 4 | import java.io.InputStream; 5 | import java.util.UUID; 6 | 7 | 8 | 9 | public class FileUploadConfiguration { 10 | 11 | private UUID fileId; 12 | private String crc; 13 | private InputStream inputStream; 14 | 15 | 16 | 17 | public FileUploadConfiguration() { 18 | } 19 | 20 | 21 | public UUID getFileId() { 22 | return fileId; 23 | } 24 | 25 | 26 | public void setFileId(UUID fileId) { 27 | this.fileId = fileId; 28 | } 29 | 30 | 31 | public String getCrc() { 32 | return crc; 33 | } 34 | 35 | 36 | public void setCrc(String crc) { 37 | this.crc = crc; 38 | } 39 | 40 | 41 | public InputStream getInputStream() { 42 | return inputStream; 43 | } 44 | 45 | 46 | public void setInputStream(InputStream inputStream) { 47 | this.inputStream = inputStream; 48 | } 49 | 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/utils/FileUploadManagerFilter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web.utils; 2 | 3 | 4 | import java.io.IOException; 5 | 6 | import javax.servlet.Filter; 7 | import javax.servlet.FilterChain; 8 | import javax.servlet.FilterConfig; 9 | import javax.servlet.ServletException; 10 | import javax.servlet.ServletRequest; 11 | import javax.servlet.ServletResponse; 12 | import javax.servlet.http.HttpServletRequest; 13 | import javax.servlet.http.HttpServletResponse; 14 | 15 | import org.springframework.beans.factory.annotation.Autowired; 16 | import org.springframework.stereotype.Component; 17 | 18 | 19 | 20 | @Component("jlfuFilter") 21 | public class FileUploadManagerFilter 22 | implements Filter { 23 | 24 | @Autowired 25 | RequestComponentContainer requestComponentContainer; 26 | 27 | 28 | 29 | @Override 30 | public void destroy() { 31 | 32 | } 33 | 34 | 35 | @Override 36 | public void doFilter(ServletRequest req, ServletResponse resp, FilterChain chain) 37 | throws IOException, ServletException { 38 | requestComponentContainer.populate((HttpServletRequest) req,(HttpServletResponse) resp); 39 | chain.doFilter(req, resp); 40 | requestComponentContainer.clear(); 41 | } 42 | 43 | 44 | @Override 45 | public void init(FilterConfig arg0) 46 | throws ServletException { 47 | 48 | } 49 | 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/utils/FileUploaderHelper.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web.utils; 2 | 3 | 4 | import java.io.IOException; 5 | import java.io.Serializable; 6 | import java.util.UUID; 7 | 8 | import javax.servlet.ServletResponse; 9 | import javax.servlet.http.HttpServletRequest; 10 | 11 | import org.apache.commons.fileupload.FileItemIterator; 12 | import org.apache.commons.fileupload.FileItemStream; 13 | import org.apache.commons.fileupload.FileUploadException; 14 | import org.apache.commons.fileupload.servlet.ServletFileUpload; 15 | import org.springframework.stereotype.Component; 16 | 17 | import com.am.jlfu.fileuploader.exception.JavaFileUploaderException; 18 | import com.am.jlfu.fileuploader.exception.MissingParameterException; 19 | import com.am.jlfu.fileuploader.json.SimpleJsonObject; 20 | import com.am.jlfu.fileuploader.web.UploadServletParameter; 21 | import com.am.jlfu.fileuploader.web.utils.ExceptionCodeMappingHelper.ExceptionCodeMapping; 22 | import com.google.gson.Gson; 23 | 24 | 25 | 26 | /** 27 | * Provides some common methods to deal with file upload requests. 28 | * 29 | * @author antoinem 30 | * 31 | */ 32 | @Component 33 | public class FileUploaderHelper { 34 | 35 | 36 | public FileUploadConfiguration extractFileUploadConfiguration(HttpServletRequest request) 37 | throws MissingParameterException, FileUploadException, IOException, JavaFileUploaderException { 38 | final FileUploadConfiguration fileUploadConfiguration = new FileUploadConfiguration(); 39 | 40 | // check if the request is multipart: 41 | if (!ServletFileUpload.isMultipartContent(request)) { 42 | throw new JavaFileUploaderException(ExceptionCodeMapping.requestIsNotMultipart); 43 | } 44 | 45 | // extract the fields 46 | fileUploadConfiguration.setFileId(UUID.fromString(getParameterValue(request, UploadServletParameter.fileId))); 47 | fileUploadConfiguration.setCrc(getParameterValue(request, UploadServletParameter.crc, false)); 48 | 49 | // Create a new file upload handler 50 | ServletFileUpload upload = new ServletFileUpload(); 51 | 52 | // parse the requestuest 53 | FileItemIterator iter = upload.getItemIterator(request); 54 | FileItemStream item = iter.next(); 55 | 56 | // throw exception if item is null 57 | if (item == null) { 58 | throw new JavaFileUploaderException(ExceptionCodeMapping.NoFileToUploadInTheRequest); 59 | } 60 | 61 | // extract input stream 62 | fileUploadConfiguration.setInputStream(item.openStream()); 63 | 64 | // return conf 65 | return fileUploadConfiguration; 66 | 67 | } 68 | 69 | 70 | public String getParameterValue(HttpServletRequest request, UploadServletParameter parameter) 71 | throws MissingParameterException { 72 | return getParameterValue(request, parameter, true); 73 | } 74 | 75 | 76 | public String getParameterValue(HttpServletRequest request, UploadServletParameter parameter, boolean mandatory) 77 | throws MissingParameterException { 78 | String parameterValue = request.getParameter(parameter.name()); 79 | if (parameterValue == null && mandatory) { 80 | throw new MissingParameterException(parameter); 81 | } 82 | return parameterValue; 83 | } 84 | 85 | 86 | public void writeExceptionToResponse(final JavaFileUploaderException e, ServletResponse servletResponse) 87 | throws IOException { 88 | writeToResponse(new SimpleJsonObject(Integer.valueOf(e.getExceptionCodeMapping().getExceptionIdentifier()).toString()), servletResponse); 89 | } 90 | 91 | 92 | public void writeToResponse(Serializable jsonObject, ServletResponse servletResponse) 93 | throws IOException { 94 | servletResponse.setContentType("application/json"); 95 | servletResponse.getWriter().print(new Gson().toJson(jsonObject)); 96 | servletResponse.getWriter().close(); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/fileuploader/web/utils/RequestComponentContainer.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web.utils; 2 | 3 | 4 | import javax.servlet.http.HttpServletRequest; 5 | import javax.servlet.http.HttpServletResponse; 6 | import javax.servlet.http.HttpSession; 7 | 8 | import org.springframework.stereotype.Component; 9 | 10 | 11 | 12 | /** 13 | * {@link HttpServletResponse} and {@link HttpServletRequest} are stored in a thread local and are 14 | * populated by the filter. 15 | * 16 | * @author antoinem 17 | * 18 | */ 19 | @Component 20 | public class RequestComponentContainer { 21 | 22 | 23 | private ThreadLocal responseThreadLocal = new ThreadLocal(); 24 | private ThreadLocal requestThreadLocal = new ThreadLocal(); 25 | 26 | 27 | 28 | public void populate(HttpServletRequest request, HttpServletResponse response) { 29 | responseThreadLocal.set(response); 30 | requestThreadLocal.set(request); 31 | } 32 | 33 | 34 | public void clear() { 35 | responseThreadLocal.remove(); 36 | } 37 | 38 | 39 | public HttpServletResponse getResponse() { 40 | return responseThreadLocal.get(); 41 | } 42 | 43 | 44 | public HttpServletRequest getRequest() { 45 | return requestThreadLocal.get(); 46 | 47 | } 48 | 49 | 50 | public HttpSession getSession() { 51 | return requestThreadLocal.get().getSession(); 52 | 53 | } 54 | 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/identifier/IdentifierProvider.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.identifier; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import javax.servlet.http.HttpServletRequest; 7 | import javax.servlet.http.HttpServletResponse; 8 | 9 | import com.am.jlfu.identifier.impl.DefaultIdentifierProvider; 10 | 11 | 12 | 13 | /** 14 | * Provides identification for JLFU API.
15 | * The default implementation ({@link DefaultIdentifierProvider}) is using a cookie to store a UUID 16 | * generated for every client.
17 | * You can provide your own implementation to be able to manage this identification using a more 18 | * complex system (if you want users to resume files from another browser or provide ids linked to a 19 | * job instead of a client if you are providing upload in a more sequential way). 20 | * 21 | * @author antoinem 22 | * @see DefaultIdentifierProvider 23 | * 24 | */ 25 | public interface IdentifierProvider { 26 | 27 | /** 28 | * Retrieves the client identifier.
29 | * 30 | * @param httpServletRequest 31 | * @param httpServletResponse 32 | * 33 | * @return the unique identifier identifying this client/job 34 | */ 35 | UUID getIdentifier(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse); 36 | 37 | 38 | /** 39 | * Define this method if your application supports IDs specified by the client (on 40 | * initialization). 41 | * 42 | * @param httpServletRequest 43 | * @param httpServletResponse 44 | * @param id 45 | * 46 | */ 47 | void setIdentifier(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, UUID id); 48 | 49 | 50 | /** 51 | * Removes the identifier. 52 | * 53 | * @param request 54 | * @param response 55 | */ 56 | void clearIdentifier(HttpServletRequest request, HttpServletResponse response); 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/identifier/impl/DefaultIdentifierProvider.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.identifier.impl; 2 | 3 | 4 | import java.util.UUID; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | import javax.servlet.http.Cookie; 8 | import javax.servlet.http.HttpServletRequest; 9 | import javax.servlet.http.HttpServletResponse; 10 | 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.stereotype.Component; 13 | 14 | import com.am.jlfu.identifier.IdentifierProvider; 15 | import com.am.jlfu.notifier.JLFUListenerPropagator; 16 | 17 | 18 | 19 | /** 20 | * Identifier provider that provides identification based on cookie. 21 | * 22 | * @author antoinem 23 | * 24 | */ 25 | @Component 26 | public class DefaultIdentifierProvider 27 | implements IdentifierProvider { 28 | 29 | 30 | public static final String cookieIdentifier = "jlufStaticStateCookieName"; 31 | 32 | @Autowired 33 | JLFUListenerPropagator jlfuListenerPropagator; 34 | 35 | 36 | 37 | @Override 38 | public void setIdentifier(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, UUID id) { 39 | 40 | // clear any first 41 | clearIdentifier(httpServletRequest, httpServletResponse); 42 | 43 | // then set in session 44 | httpServletRequest.getSession().setAttribute(cookieIdentifier, id); 45 | 46 | // and cookie 47 | setCookie(httpServletResponse, id); 48 | 49 | } 50 | 51 | 52 | public static Cookie getCookie(Cookie[] cookies, String id) { 53 | if (cookies != null) { 54 | for (Cookie cookie : cookies) { 55 | if (cookie.getName().equals(id)) { 56 | // found it 57 | if (cookie.getMaxAge() != 0) { 58 | return cookie; 59 | } 60 | } 61 | } 62 | } 63 | return null; 64 | } 65 | 66 | 67 | public static void setCookie(HttpServletResponse response, UUID uuid) { 68 | Cookie cookie = new Cookie(cookieIdentifier, uuid.toString()); 69 | cookie.setMaxAge((int) TimeUnit.DAYS.toSeconds(31)); 70 | response.addCookie(cookie); 71 | } 72 | 73 | 74 | UUID getUuid() { 75 | final UUID uuid = UUID.randomUUID(); 76 | jlfuListenerPropagator.getPropagator().onNewClient(uuid); 77 | return uuid; 78 | } 79 | 80 | 81 | @Override 82 | public UUID getIdentifier(HttpServletRequest req, HttpServletResponse resp) { 83 | 84 | // get from session 85 | UUID uuid = (UUID) req.getSession().getAttribute(cookieIdentifier); 86 | 87 | // if nothing in session 88 | if (uuid == null) { 89 | 90 | // check in cookie 91 | Cookie cookie = getCookie(req.getCookies(), cookieIdentifier); 92 | if (cookie != null && cookie.getValue() != null) { 93 | // set in session 94 | uuid = UUID.fromString(cookie.getValue()); 95 | req.getSession().setAttribute(cookieIdentifier, uuid); 96 | jlfuListenerPropagator.getPropagator().onClientBack(uuid); 97 | return uuid; 98 | } 99 | 100 | // if not in session nor cookie, create one 101 | // create uuid 102 | uuid = getUuid(); 103 | 104 | // and set it 105 | setIdentifier(req, resp, uuid); 106 | 107 | } 108 | return uuid; 109 | } 110 | 111 | 112 | @Override 113 | public void clearIdentifier(HttpServletRequest req, HttpServletResponse resp) { 114 | // clear session 115 | req.getSession().removeAttribute(cookieIdentifier); 116 | 117 | // remove cookie 118 | Cookie cookie = getCookie(req.getCookies(), cookieIdentifier); 119 | if (cookie != null) { 120 | cookie.setMaxAge(0); 121 | resp.addCookie(cookie); 122 | } 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/notifier/JLFUListener.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.notifier; 2 | 3 | 4 | import java.util.Collection; 5 | import java.util.UUID; 6 | 7 | import com.am.jlfu.fileuploader.limiter.RateLimiterConfigurationManager; 8 | import com.am.jlfu.identifier.impl.DefaultIdentifierProvider; 9 | import com.am.jlfu.staticstate.JavaLargeFileUploaderService; 10 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 11 | 12 | 13 | 14 | /** 15 | * Listener to be able to monitor the JLFU API on the java side. 16 | * Use {@link JLFUListenerPropagator} to register a listener. 17 | * 18 | * @author antoinem 19 | * 20 | */ 21 | public interface JLFUListener { 22 | 23 | /** 24 | * Fired when a new client has been attributed a new id.
25 | * Note that this event is sent by the {@link DefaultIdentifierProvider}. 26 | * 27 | * @param clientId 28 | */ 29 | void onNewClient(UUID clientId); 30 | 31 | 32 | /** 33 | * Fired when a client is identified with its cookie and the corresponding state is restored. 34 | * Note that this event is sent by the {@link DefaultIdentifierProvider}. 35 | * 36 | * @param clientId 37 | */ 38 | void onClientBack(UUID clientId); 39 | 40 | 41 | /** 42 | * Fired when the uploads of a client have been inactive for duration specified.
43 | * Default to {@link RateLimiterConfigurationManager#clientEvictionTimeInSeconds} 44 | * 45 | * @param clientId 46 | * @param inactivityDuration 47 | */ 48 | void onClientInactivity(UUID clientId, int inactivityDuration); 49 | 50 | 51 | /** 52 | * Fired when the upload of the file specified by the fileId is finished for the client 53 | * specified by the clientId. 54 | * 55 | * @param clientId 56 | * @param fileId 57 | */ 58 | void onFileUploadEnd(UUID clientId, UUID fileId); 59 | 60 | 61 | /** 62 | * Fired when the upload of the file specified by the fileId has been prepared for the client 63 | * specified by the clientId. 64 | * 65 | * @param clientId 66 | * @param fileId 67 | */ 68 | void onFileUploadPrepared(UUID clientId, UUID fileId); 69 | 70 | 71 | /** 72 | * Fired when all the uploads of the files specified by the fileIds have been prepared for the 73 | * client 74 | * specified by the clientId. 75 | * 76 | * @param clientId 77 | * @param fileIds 78 | */ 79 | void onAllFileUploadsPrepared(UUID identifier, Collection fileIds); 80 | 81 | 82 | /** 83 | * Fired when the upload of the file specified by the fileId has been cancelled for the client 84 | * specified by the clientId. 85 | * 86 | * @param clientId 87 | * @param fileId 88 | */ 89 | void onFileUploadCancelled(UUID clientId, UUID fileId); 90 | 91 | 92 | /** 93 | * Fired when the upload of the file specified by the fileId has been paused for the client 94 | * specified by the clientId. 95 | * 96 | * @param clientId 97 | * @param fileId 98 | */ 99 | void onFileUploadPaused(UUID clientId, UUID fileId); 100 | 101 | 102 | /** 103 | * Fired when the upload of the file specified by the fileId has been resumed for the client 104 | * specified by the clientId. 105 | * 106 | * @param clientId 107 | * @param fileId 108 | */ 109 | void onFileUploadResumed(UUID clientId, UUID fileId); 110 | 111 | /** 112 | * Fired about every second for each file currently uploading specified by the fileId for the client 113 | * specified by the clientId whose progress has changed. 114 | * 115 | * @param clientId 116 | * @param fileId 117 | * @param progress 118 | */ 119 | void onFileUploadProgress(UUID clientId, UUID fileId, FileProgressStatus progress); 120 | 121 | /** 122 | * Fired when the administration method {@link JavaLargeFileUploaderService#disableFileUploader()} is called. 123 | */ 124 | void onFileUploaderDisabled(); 125 | 126 | /** 127 | * Fired when the administration method {@link JavaLargeFileUploaderService#enableFileUploader()} is called. 128 | */ 129 | void onFileUploaderEnabled(); 130 | 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/notifier/JLFUListenerAdapter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.notifier; 2 | 3 | 4 | import java.util.Collection; 5 | import java.util.UUID; 6 | 7 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 8 | 9 | 10 | 11 | /** 12 | * Listener adapter of {@link JLFUListener}. 13 | * 14 | * @author antoinem 15 | * 16 | */ 17 | public class JLFUListenerAdapter 18 | implements JLFUListener { 19 | 20 | @Override 21 | public void onNewClient(UUID clientId) { 22 | 23 | } 24 | 25 | 26 | @Override 27 | public void onClientBack(UUID clientId) { 28 | 29 | } 30 | 31 | 32 | @Override 33 | public void onClientInactivity(UUID clientId, int inactivityDuration) { 34 | 35 | } 36 | 37 | 38 | @Override 39 | public void onFileUploadEnd(UUID clientId, UUID fileId) { 40 | 41 | } 42 | 43 | 44 | @Override 45 | public void onFileUploadPrepared(UUID clientId, UUID fileId) { 46 | 47 | } 48 | 49 | 50 | @Override 51 | public void onFileUploadCancelled(UUID clientId, UUID fileId) { 52 | 53 | } 54 | 55 | 56 | @Override 57 | public void onFileUploadPaused(UUID clientId, UUID fileId) { 58 | 59 | } 60 | 61 | 62 | @Override 63 | public void onFileUploadResumed(UUID clientId, UUID fileId) { 64 | 65 | } 66 | 67 | 68 | @Override 69 | public void onAllFileUploadsPrepared(UUID identifier, Collection fileIds) { 70 | 71 | } 72 | 73 | 74 | @Override 75 | public void onFileUploadProgress(UUID clientId, UUID fileId, FileProgressStatus progress) { 76 | 77 | } 78 | 79 | 80 | @Override 81 | public void onFileUploaderDisabled() { 82 | 83 | } 84 | 85 | 86 | @Override 87 | public void onFileUploaderEnabled() { 88 | 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/notifier/JLFUListenerPropagator.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.notifier; 2 | 3 | 4 | import org.springframework.stereotype.Component; 5 | 6 | import com.am.jlfu.notifier.utils.GenericPropagator; 7 | 8 | 9 | 10 | /** 11 | * Propagates the events to the registered listeners. 12 | * 13 | * @author antoinem 14 | * 15 | */ 16 | @Component 17 | public class JLFUListenerPropagator extends GenericPropagator { 18 | 19 | @Override 20 | protected Class getProxiedClass() { 21 | return JLFUListener.class; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/notifier/utils/GenericPropagator.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.notifier.utils; 2 | 3 | 4 | import java.lang.reflect.InvocationHandler; 5 | import java.lang.reflect.Method; 6 | import java.lang.reflect.Proxy; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | import javax.annotation.PostConstruct; 11 | 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import com.am.jlfu.notifier.JLFUListener; 16 | import com.google.common.base.Joiner; 17 | import com.google.common.collect.Lists; 18 | 19 | 20 | 21 | /** 22 | * Propagates the methods called on {@link #proxiedElement} to all objects in {@link #propagateTo}.
23 | * {@link #getProxiedClass()} has to be overridden by any subclass.
24 | * Note that in order to not block the caller, the invocation is processed in a separate thread. 25 | * 26 | * @param 27 | * @author antoinem 28 | */ 29 | public abstract class GenericPropagator { 30 | 31 | private static final Logger log = LoggerFactory.getLogger(GenericPropagator.class); 32 | 33 | 34 | /** The element proxied by {@link #initProxy()} */ 35 | private T proxiedElement; 36 | 37 | /** List of objects to propagate to */ 38 | private List propagateTo = Lists.newArrayList(); 39 | 40 | 41 | 42 | /** 43 | * @return The class of {@link #proxiedElement} 44 | */ 45 | protected abstract Class getProxiedClass(); 46 | 47 | 48 | @PostConstruct 49 | @SuppressWarnings("unchecked") 50 | private void initProxy() { 51 | 52 | // initialize the proxy 53 | proxiedElement = (T) Proxy.newProxyInstance( 54 | getProxiedClass().getClassLoader(), 55 | new Class[] { getProxiedClass() }, 56 | new InvocationHandler() { 57 | 58 | @Override 59 | public Object invoke(Object proxy, final Method method, final Object[] args) 60 | throws Throwable { 61 | synchronized (propagateTo) { 62 | process(new ArrayList(propagateTo), method, args); 63 | } 64 | return null; 65 | } 66 | 67 | 68 | private void process(final List list, final Method method, final Object[] args) { 69 | new Thread() { 70 | 71 | @Override 72 | public void run() { 73 | 74 | if (log.isTraceEnabled()) { 75 | log.trace("{propagating `" + method.getName() + "` with args `"+Joiner.on(',').join(args)+"` to `" + list.size() + "` elements}"); 76 | } 77 | 78 | for (T o : list) { 79 | try { 80 | method.invoke(o, args); 81 | } 82 | catch (Exception e) { 83 | log.error("cannot propagate " + method.getName(), e); 84 | } 85 | } 86 | } 87 | }.start(); 88 | } 89 | }); 90 | 91 | } 92 | 93 | 94 | /** 95 | * Register a propagant to {@link #propagateTo}. 96 | * 97 | * @param propagant 98 | */ 99 | public void registerListener(T propagant) { 100 | synchronized (propagateTo) { 101 | propagateTo.add(propagant); 102 | } 103 | } 104 | 105 | 106 | /** 107 | * Unregister a propagant from {@link #propagateTo}. 108 | * 109 | * @param propagant 110 | */ 111 | public void unregisterListener(JLFUListener propagant) { 112 | synchronized (propagateTo) { 113 | propagateTo.remove(propagant); 114 | } 115 | } 116 | 117 | 118 | /** 119 | * Unregister all the listeners. 120 | */ 121 | public void unregisterAllListeners() { 122 | synchronized (propagateTo) { 123 | propagateTo.clear(); 124 | } 125 | } 126 | 127 | 128 | /** 129 | * @return the propagator. 130 | */ 131 | public T getPropagator() { 132 | return proxiedElement; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/FileDeleter.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.io.File; 5 | import java.util.Arrays; 6 | import java.util.Collection; 7 | import java.util.List; 8 | import java.util.concurrent.Executors; 9 | import java.util.concurrent.ScheduledThreadPoolExecutor; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | import javax.annotation.PostConstruct; 13 | import javax.annotation.PreDestroy; 14 | 15 | import org.apache.commons.io.FileUtils; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | import org.springframework.stereotype.Component; 19 | 20 | import com.google.common.collect.ImmutableList; 21 | import com.google.common.collect.Iterables; 22 | import com.google.common.collect.Lists; 23 | 24 | 25 | 26 | /** 27 | * Takes care of deleting the files. 28 | * 29 | * @author antoinem 30 | * 31 | */ 32 | @Component 33 | public class FileDeleter 34 | implements Runnable { 35 | 36 | private static final Logger log = LoggerFactory.getLogger(FileDeleter.class); 37 | 38 | /** The executor */ 39 | private ScheduledThreadPoolExecutor executor = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1); 40 | 41 | 42 | 43 | @PostConstruct 44 | private void start() { 45 | executor.schedule(this, 10, TimeUnit.SECONDS); 46 | } 47 | 48 | 49 | 50 | /** 51 | * List of the files to delete. 52 | */ 53 | private List files = Lists.newArrayList(); 54 | 55 | 56 | 57 | @Override 58 | public void run() { 59 | 60 | // extract all the files to an immutable list 61 | ImmutableList copyOf; 62 | synchronized (this.files) { 63 | copyOf = ImmutableList.copyOf(this.files); 64 | } 65 | 66 | // log 67 | boolean weHaveFilesToDelete = !copyOf.isEmpty(); 68 | if (weHaveFilesToDelete) { 69 | log.debug(copyOf.size() + " files to delete"); 70 | } 71 | 72 | // and create a new list 73 | List successfullyDeletedFiles = Lists.newArrayList(); 74 | 75 | // delete them 76 | for (File file : copyOf) { 77 | if (delete(file)) { 78 | successfullyDeletedFiles.add(file); 79 | log.debug(file + " successfully deleted."); 80 | } 81 | else { 82 | log.debug(file + " not deleted, rescheduled for deletion."); 83 | } 84 | } 85 | 86 | // all the files have been processed 87 | // remove the deleted files from queue 88 | synchronized (this.files) { 89 | Iterables.removeAll(this.files, successfullyDeletedFiles); 90 | } 91 | 92 | // log 93 | if (weHaveFilesToDelete) { 94 | log.debug(successfullyDeletedFiles.size() + " deleted files"); 95 | } 96 | 97 | // and reschedule 98 | start(); 99 | } 100 | 101 | 102 | /** 103 | * @param file 104 | * @return true if the file has been deleted, false otherwise. 105 | */ 106 | private boolean delete(File file) { 107 | 108 | try { 109 | // if file exists 110 | if (file.exists()) { 111 | 112 | // if it is a file 113 | if (file.isFile()) { 114 | // delete it 115 | return file.delete(); 116 | } 117 | // otherwise, if it is a directoy 118 | else if (file.isDirectory()) { 119 | FileUtils.deleteDirectory(file); 120 | return true; 121 | } 122 | // if its none of them, we cannot delete them so we assume its deleted. 123 | else { 124 | return true; 125 | } 126 | 127 | } 128 | // if does not exist, we can remove it from list 129 | else { 130 | return true; 131 | } 132 | } 133 | // if we have an exception 134 | catch (Exception e) { 135 | log.error(file + " deletion exception: " + e.getMessage()); 136 | // the file has not been deleted 137 | return false; 138 | } 139 | 140 | } 141 | 142 | 143 | public void deleteFile(File... file) { 144 | deleteFiles(Arrays.asList(file)); 145 | } 146 | 147 | 148 | public void deleteFiles(Collection files) { 149 | synchronized (this.files) { 150 | this.files.addAll(files); 151 | } 152 | } 153 | 154 | 155 | /** 156 | * Returns true if the specified file is scheduled for deletion 157 | * 158 | * @param file 159 | * @return 160 | */ 161 | public boolean deletionQueueContains(File file) { 162 | synchronized (this.files) { 163 | return files.contains(file); 164 | } 165 | } 166 | 167 | 168 | @PreDestroy 169 | private void destroy() throws InterruptedException { 170 | log.debug("destroying executor"); 171 | executor.shutdown(); 172 | if (!executor.awaitTermination(1, TimeUnit.MINUTES)) { 173 | log.error("executor timed out"); 174 | List shutdownNow = executor.shutdownNow(); 175 | for (Runnable runnable : shutdownNow) { 176 | log.error(runnable + "has not been terminated"); 177 | } 178 | } 179 | } 180 | 181 | } 182 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/JavaLargeFileUploaderService.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | import java.io.File; 4 | import java.io.FileNotFoundException; 5 | import java.io.FileOutputStream; 6 | import java.io.FilenameFilter; 7 | import java.util.UUID; 8 | import java.util.concurrent.ExecutionException; 9 | import java.util.concurrent.TimeoutException; 10 | 11 | import org.apache.commons.io.IOUtils; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.jmx.export.annotation.ManagedOperation; 16 | import org.springframework.jmx.export.annotation.ManagedResource; 17 | import org.springframework.stereotype.Component; 18 | 19 | import com.am.jlfu.fileuploader.logic.UploadServletAsyncProcessor; 20 | import com.am.jlfu.fileuploader.utils.ProgressManager; 21 | import com.am.jlfu.notifier.JLFUListenerPropagator; 22 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 23 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 24 | import com.thoughtworks.xstream.XStream; 25 | 26 | /** 27 | * Provides methods related to the management of information of the files for services outside the scope of a request.
28 | * 29 | * @author antoinem 30 | * @see StaticStateManager 31 | */ 32 | @Component 33 | @ManagedResource(objectName = "JavaLargeFileUploader:name=operationManager") 34 | public class JavaLargeFileUploaderService { 35 | 36 | @Autowired 37 | FileDeleter fileDeleter; 38 | 39 | @Autowired 40 | StaticStateManager staticStateManager; 41 | 42 | @Autowired 43 | ProgressManager progressManager; 44 | 45 | @Autowired 46 | StaticStateDirectoryManager staticStateDirectoryManager; 47 | 48 | @Autowired 49 | UploadServletAsyncProcessor uploadServletAsyncProcessor; 50 | 51 | @Autowired 52 | JLFUListenerPropagator jlfuListenerPropagator; 53 | 54 | private static final Logger log = LoggerFactory.getLogger(JavaLargeFileUploaderService.class); 55 | 56 | 57 | /** 58 | * Retrieves the progress of the specified file for the specified client.
59 | * 60 | * 61 | * @param clientId 62 | * @param fileId 63 | * @return 64 | * @throws FileNotFoundException 65 | */ 66 | public FileProgressStatus getProgress(UUID clientId, UUID fileId) 67 | throws FileNotFoundException { 68 | return progressManager.getProgress(fileId); 69 | } 70 | 71 | 72 | 73 | /** 74 | * Updates an entity inside the cache and onto the filesystem. 75 | * 76 | * @param uuid the uuid of the client, identifying the file 77 | * @param entity the entity to write into that file 78 | */ 79 | public void updateEntity(UUID uuid, T entity) { 80 | log.debug("writing state for " + uuid); 81 | staticStateManager.cache.put(uuid, entity); 82 | writeEntity(new File(staticStateDirectoryManager.getUUIDFileParent(uuid), StaticStateManager.FILENAME), entity); 83 | } 84 | 85 | /** 86 | * Persists modifications onto filesystem only. 87 | * 88 | * @param uuid the uuid of the client, identifying the file 89 | * @param entity the entity to write into that file 90 | */ 91 | public void writeEntity(UUID uuid, T entity) { 92 | writeEntity(new File(staticStateDirectoryManager.getUUIDFileParent(uuid), StaticStateManager.FILENAME), entity); 93 | } 94 | 95 | /** 96 | * Persists modifications onto filesystem only. 97 | * 98 | * @param staticStateFile the file in which to write the entity 99 | * @param entity the entity to write into that file 100 | */ 101 | public void writeEntity(File staticStateFile, T entity) { 102 | write(entity, staticStateFile); 103 | } 104 | 105 | private void write(T modelFromContext, File modelFile) { 106 | XStream xStream = new XStream(); 107 | FileOutputStream fs = null; 108 | try { 109 | fs = new FileOutputStream(modelFile); 110 | xStream.toXML(modelFromContext, fs); 111 | } 112 | catch (FileNotFoundException e) { 113 | log.error("cannot write to model file for " + modelFromContext.getClass().getSimpleName() + ": " + e.getMessage(), e); 114 | } 115 | finally { 116 | IOUtils.closeQuietly(fs); 117 | } 118 | } 119 | 120 | 121 | 122 | /** 123 | * Retrieves the entity from cache using a client identifier. 124 | * 125 | * @param clientIdentifier 126 | * @return 127 | */ 128 | public T getEntityIfPresent(UUID clientIdentifier) { 129 | return staticStateManager.cache.getIfPresent(clientIdentifier); 130 | } 131 | 132 | /** 133 | * Remove the pending uploaded file identifier by this id for this client. 134 | * @param clientId 135 | * @param fileId 136 | */ 137 | public void clearFile(final UUID clientId, final UUID fileId) 138 | { 139 | log.debug("Clearing pending uploaded file and all attributes linked to it."); 140 | 141 | final File uuidFileParent = staticStateDirectoryManager.getUUIDFileParent(clientId); 142 | 143 | // remove the uploaded file for this particular id 144 | fileDeleter.deleteFile(uuidFileParent.listFiles(new FilenameFilter() { 145 | 146 | public boolean accept(File dir, String name) { 147 | return name.startsWith(fileId.toString()); 148 | } 149 | })); 150 | 151 | // remove the file information in entity 152 | T entity = getEntityIfPresent(clientId); 153 | entity.getFileStates().remove(fileId); 154 | 155 | // and save 156 | updateEntity(clientId, entity); 157 | } 158 | 159 | /** 160 | * Clear everything including cache, session, files for this client. 161 | * 162 | * @throws TimeoutException 163 | * @throws ExecutionException 164 | * @throws InterruptedException 165 | */ 166 | public void clearClient(UUID clientId) { 167 | log.debug("Clearing everything including cache, session, files."); 168 | 169 | final File uuidFileParent = staticStateDirectoryManager.getUUIDFileParent(clientId); 170 | 171 | // schedule file for deletion 172 | fileDeleter.deleteFile(uuidFileParent); 173 | 174 | // remove entity from cache 175 | staticStateManager.cache.invalidate(clientId); 176 | 177 | } 178 | 179 | 180 | /** 181 | * Enables the processing of file uploads. Clients will automatically resume their upload. 182 | * @see #disableFileUploader() 183 | */ 184 | @ManagedOperation 185 | public void enableFileUploader() { 186 | uploadServletAsyncProcessor.setEnabled(true); 187 | jlfuListenerPropagator.getPropagator().onFileUploaderEnabled(); 188 | } 189 | 190 | /** 191 | * Disables the processing of file uploads. Clients currently uploading Files will wait and automatically resume the uploads when {@link #enableFileUploader()} is called. 192 | * @see #enableFileUploader() 193 | */ 194 | @ManagedOperation 195 | public void disableFileUploader() { 196 | uploadServletAsyncProcessor.setEnabled(false); 197 | jlfuListenerPropagator.getPropagator().onFileUploaderDisabled(); 198 | } 199 | 200 | } 201 | 202 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/StaticStateDirectoryManager.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.io.File; 5 | import java.util.UUID; 6 | 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.stereotype.Component; 9 | 10 | 11 | 12 | @Component 13 | public class StaticStateDirectoryManager { 14 | 15 | @Autowired 16 | StaticStateRootFolderProvider staticStateRootFolderProvider; 17 | 18 | @Autowired 19 | StaticStateIdentifierManager staticStateIdentifierManager; 20 | 21 | 22 | 23 | /** 24 | * Retrieves the file parent of the session. 25 | * 26 | * @return 27 | */ 28 | public File getUUIDFileParent() { 29 | return getUUIDFileParent(staticStateIdentifierManager.getIdentifier()); 30 | } 31 | 32 | 33 | /** 34 | * Retrieves the file parent of the session context less. 35 | * 36 | * @param uuid 37 | * @return 38 | */ 39 | public File getUUIDFileParent(UUID uuid) { 40 | File uuidFileParent = new File(staticStateRootFolderProvider.getRootFolder(), uuid.toString()); 41 | if (!uuidFileParent.exists()) { 42 | uuidFileParent.mkdirs(); 43 | } 44 | return uuidFileParent; 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/StaticStateIdentifierManager.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Component; 8 | 9 | import com.am.jlfu.fileuploader.web.utils.RequestComponentContainer; 10 | import com.am.jlfu.identifier.IdentifierProvider; 11 | 12 | 13 | 14 | @Component 15 | public class StaticStateIdentifierManager { 16 | 17 | @Autowired 18 | IdentifierProvider identifierProvider; 19 | 20 | @Autowired 21 | RequestComponentContainer requestComponentContainer; 22 | 23 | 24 | 25 | public UUID getIdentifier() { 26 | return identifierProvider.getIdentifier(requestComponentContainer.getRequest(), requestComponentContainer.getResponse()); 27 | } 28 | 29 | 30 | public void clearIdentifier() { 31 | identifierProvider.clearIdentifier(requestComponentContainer.getRequest(), requestComponentContainer.getResponse()); 32 | } 33 | 34 | 35 | public void setIdentifier(UUID id) { 36 | identifierProvider.setIdentifier(requestComponentContainer.getRequest(), requestComponentContainer.getResponse(), id); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/StaticStateManager.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.io.File; 5 | import java.io.FileInputStream; 6 | import java.io.FileNotFoundException; 7 | import java.io.IOException; 8 | import java.util.UUID; 9 | import java.util.concurrent.ExecutionException; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.ThreadPoolExecutor; 12 | import java.util.concurrent.TimeUnit; 13 | import java.util.concurrent.TimeoutException; 14 | 15 | import org.apache.commons.io.IOUtils; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | import org.springframework.beans.factory.annotation.Autowired; 19 | import org.springframework.stereotype.Component; 20 | 21 | import com.am.jlfu.fileuploader.exception.FileCorruptedException; 22 | import com.am.jlfu.fileuploader.json.FileStateJsonBase; 23 | import com.am.jlfu.notifier.JLFUListenerPropagator; 24 | import com.am.jlfu.staticstate.entities.StaticFileState; 25 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 26 | import com.google.common.cache.CacheBuilder; 27 | import com.google.common.cache.CacheLoader; 28 | import com.google.common.cache.LoadingCache; 29 | import com.thoughtworks.xstream.XStream; 30 | 31 | 32 | 33 | /** 34 | * Manages the information related to the files.
35 | * This information is stored locally in a cache and also persisted on the file system.
36 | * All of the methods used here are to be called within the scope of a request. Most of these methods (to query and update) are also available outside of such a scope in {@link JavaLargeFileUploaderService} 37 | * This class has to be initialized with the {@link #init(Class)} method first. 38 | * 39 | * @author antoinem 40 | * 41 | * @param 42 | */ 43 | @Component 44 | public class StaticStateManager { 45 | 46 | private static final Logger log = LoggerFactory.getLogger(StaticStateManager.class); 47 | static final String FILENAME = "StaticState.xml"; 48 | 49 | @Autowired 50 | FileDeleter fileDeleter; 51 | 52 | @Autowired 53 | JLFUListenerPropagator jlfuListenerPropagator; 54 | 55 | @Autowired 56 | StaticStateIdentifierManager staticStateIdentifierManager; 57 | 58 | @Autowired 59 | StaticStateDirectoryManager staticStateDirectoryManager; 60 | 61 | @Autowired 62 | JavaLargeFileUploaderService staticStateManagerService; 63 | 64 | /** 65 | * Used to bypass generic type erasure.
66 | * Has to be manually specified with the {@link #init(Class)} method. 67 | */ 68 | Class entityType; 69 | 70 | 71 | /** The executor that could write stuff asynchronously into the static state */ 72 | private ThreadPoolExecutor fileStateUpdaterExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); 73 | 74 | 75 | 76 | private Class getEntityType() { 77 | // if not defined, try to init with default 78 | if (entityType == null) { 79 | entityType = (Class) StaticStatePersistedOnFileSystemEntity.class; 80 | } 81 | return entityType; 82 | } 83 | 84 | 85 | 86 | LoadingCache cache = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS).build(new CacheLoader() { 87 | 88 | public T load(UUID uuid) 89 | throws Exception { 90 | 91 | return createOrRestore(uuid); 92 | 93 | } 94 | }); 95 | 96 | 97 | 98 | /** 99 | * Creates or restores a new file. 100 | * 101 | * @param uuid 102 | * @return 103 | * @throws IOException 104 | */ 105 | private T createOrRestore(UUID uuid) 106 | throws IOException { 107 | 108 | // restore cache from file: 109 | File uuidFileParent = staticStateDirectoryManager.getUUIDFileParent(); 110 | 111 | // if that file is scheduled for deletion, we do not restore it 112 | if (fileDeleter.deletionQueueContains(uuidFileParent)) { 113 | log.debug("trying to restore from state a file that is scheduled for deletion"); 114 | // invalidate identifier 115 | staticStateIdentifierManager.clearIdentifier(); 116 | // get a new one 117 | // and recreate file 118 | uuidFileParent = staticStateDirectoryManager.getUUIDFileParent(); 119 | } 120 | 121 | File uuidFile = new File(uuidFileParent, FILENAME); 122 | T entity = null; 123 | 124 | if (uuidFile.exists()) { 125 | log.debug("No value in the cache for uuid " + uuid + ". Filling cache from file."); 126 | try { 127 | entity = read(uuidFile); 128 | } 129 | catch (Exception e) { 130 | log.error("Cache cannot be restored from " + uuidFile.getAbsolutePath() + "." + 131 | "The file might be empty or the model has changed since last time: " + e.getMessage(), e); 132 | } 133 | } 134 | else { 135 | log.debug("No value in the cache for uuid " + uuid + " and no value in the file. Creating a new one."); 136 | 137 | // create the file 138 | try { 139 | uuidFile.createNewFile(); 140 | } 141 | catch (IOException e) { 142 | log.error("cannot create model file: " + e.getMessage(), e); 143 | throw e; 144 | } 145 | 146 | // and persist an entity 147 | try { 148 | entity = getEntityType().newInstance(); 149 | } 150 | catch (InstantiationException e) { 151 | throw new RuntimeException(e); 152 | } 153 | catch (IllegalAccessException e) { 154 | throw new RuntimeException(e); 155 | } 156 | staticStateManagerService.writeEntity(uuidFile, entity); 157 | 158 | } 159 | 160 | // then return entity 161 | return entity; 162 | } 163 | 164 | /** 165 | * Retrieves the entity from cookie or cache if it exists or create one if it does not exists. 166 | * 167 | * @return 168 | * @throws ExecutionException 169 | */ 170 | public T getEntity() { 171 | return cache.getUnchecked(staticStateIdentifierManager.getIdentifier()); 172 | } 173 | 174 | 175 | /** 176 | * Retrieves the entity from cache or null if this entity is not present. 177 | * 178 | * @return 179 | */ 180 | public StaticStatePersistedOnFileSystemEntity getEntityIfPresent() { 181 | return staticStateManagerService.getEntityIfPresent(staticStateIdentifierManager.getIdentifier()); 182 | } 183 | 184 | 185 | /** 186 | * Persist modifications to file and cache. 187 | * 188 | * @param entity 189 | * @return 190 | * @throws ExecutionException 191 | */ 192 | public void updateEntity(T entity) { 193 | UUID uuid = staticStateIdentifierManager.getIdentifier(); 194 | staticStateManagerService.updateEntity(uuid, entity); 195 | } 196 | 197 | 198 | /** 199 | * Clear everything including cache, session, files. 200 | * 201 | * @throws TimeoutException 202 | * @throws ExecutionException 203 | * @throws InterruptedException 204 | */ 205 | public void clear() 206 | { 207 | //clear stuff on the server 208 | staticStateManagerService.clearClient(staticStateIdentifierManager.getIdentifier()); 209 | 210 | // remove cookie and session 211 | staticStateIdentifierManager.clearIdentifier(); 212 | 213 | } 214 | 215 | 216 | public void clearFile(final UUID fileId) 217 | { 218 | staticStateManagerService.clearFile(staticStateIdentifierManager.getIdentifier(), fileId); 219 | } 220 | 221 | 222 | 223 | T read(File f) { 224 | XStream xStream = new XStream(); 225 | FileInputStream fs = null; 226 | T fromXML = null; 227 | try { 228 | fs = new FileInputStream(f); 229 | fromXML = (T) xStream.fromXML(fs); 230 | } 231 | catch (FileNotFoundException e) { 232 | log.error("cannot read model file: " + e.getMessage(), e); 233 | } 234 | finally { 235 | IOUtils.closeQuietly(fs); 236 | } 237 | return fromXML; 238 | } 239 | 240 | 241 | /** 242 | * Initializes the bean with the class of the entity. Shall be called once. Calling it more than 243 | * once has no effect. 244 | * 245 | * @param clazz 246 | */ 247 | public void init(Class clazz) { 248 | entityType = clazz; 249 | } 250 | 251 | 252 | /** 253 | * Writes in the file that the last slice has been successfully uploaded. 254 | * 255 | * @param clientId 256 | * @param fileId 257 | * @return true if the file is complete 258 | * @throws FileCorruptedException 259 | */ 260 | public void setCrcBytesValidated(final UUID clientId, UUID fileId, final long validated) throws FileCorruptedException { 261 | 262 | final T entity = cache.getIfPresent(clientId); 263 | if (entity == null) { 264 | return; 265 | } 266 | final StaticFileState staticFileState = entity.getFileStates().get(fileId); 267 | if (staticFileState == null) { 268 | return; 269 | } 270 | FileStateJsonBase staticFileStateJson = staticFileState.getStaticFileStateJson(); 271 | if (staticFileStateJson == null) { 272 | return; 273 | } 274 | Long crcredBytes = staticFileStateJson.getCrcedBytes(); 275 | staticFileStateJson.setCrcedBytes( 276 | crcredBytes + validated); 277 | 278 | log.debug(validated + " more bytes have been validated appended to the already " + crcredBytes + " bytes validated for file " + fileId + 279 | " for client id " + clientId); 280 | 281 | // manage the end of file 282 | if (staticFileStateJson.getCrcedBytes().equals(staticFileStateJson.getOriginalFileSizeInBytes())) { 283 | jlfuListenerPropagator.getPropagator().onFileUploadEnd(clientId, fileId); 284 | } 285 | 286 | //checks whether we have a file corruption exception 287 | if (staticFileStateJson.getCrcedBytes() > staticFileStateJson.getOriginalFileSizeInBytes()) { 288 | throw new FileCorruptedException(staticFileStateJson.getCrcedBytes() + " crced bytes are more than it should be: " + staticFileStateJson.getOriginalFileSizeInBytes()); 289 | } 290 | 291 | fileStateUpdaterExecutor.submit(new Runnable() { 292 | 293 | @Override 294 | public void run() { 295 | // write this later on. 296 | staticStateManagerService.writeEntity(clientId, entity); 297 | } 298 | }); 299 | 300 | } 301 | 302 | } 303 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/StaticStateRootFolderProvider.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.io.File; 5 | 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.stereotype.Component; 9 | import org.springframework.web.context.WebApplicationContext; 10 | 11 | 12 | 13 | /** 14 | * Provides the root folder in which files will be uploaded.
15 | * 16 | * @author antoinem 17 | * 18 | */ 19 | @Component 20 | public class StaticStateRootFolderProvider { 21 | 22 | @Value("jlfu{jlfu.defaultUploadFolder:/JavaLargeFileUploader}") 23 | private String defaultUploadFolder; 24 | 25 | @Value("jlfu{jlfu.uploadFolderRelativePath:true}") 26 | private Boolean uploadFolderRelativePath; 27 | 28 | @Autowired(required = false) 29 | WebApplicationContext webApplicationContext; 30 | 31 | public File getRootFolder() { 32 | String realPath = defaultUploadFolder; 33 | if (uploadFolderRelativePath) { 34 | realPath = webApplicationContext.getServletContext().getRealPath(defaultUploadFolder); 35 | } 36 | File file = new File(realPath); 37 | // create if non existent 38 | if (!file.exists()) { 39 | file.mkdirs(); 40 | } 41 | // if existent but a file, runtime exception 42 | else { 43 | if (file.isFile()) { 44 | throw new RuntimeException(file.getAbsolutePath() + 45 | " is a file. The default root folder provider uses this path to store the files. Consider using a specific root folder provider or delete this file."); 46 | } 47 | } 48 | return file; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/entities/FileProgressStatus.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate.entities; 2 | 3 | import com.am.jlfu.fileuploader.json.ProgressJson; 4 | import com.am.jlfu.fileuploader.utils.UnitConverter; 5 | 6 | /** 7 | * Entity providing progress information about a file. 8 | * 9 | * @author antoinem 10 | * 11 | */ 12 | public class FileProgressStatus extends ProgressJson{ 13 | 14 | /** 15 | * Generated id. 16 | */ 17 | private static final long serialVersionUID = -6247365041854992033L; 18 | 19 | private long totalFileSize; 20 | private long bytesUploaded; 21 | 22 | /** 23 | * Default constructor. 24 | */ 25 | public FileProgressStatus() { 26 | super(); 27 | } 28 | 29 | /** 30 | * @return total file of the size in bytes. 31 | */ 32 | public long getTotalFileSize() { 33 | return totalFileSize; 34 | } 35 | 36 | 37 | public void setTotalFileSize(long totalFileSize) { 38 | this.totalFileSize = totalFileSize; 39 | } 40 | 41 | /** 42 | * @return quantity of bytes uploaded. 43 | */ 44 | public long getBytesUploaded() { 45 | return bytesUploaded; 46 | } 47 | 48 | 49 | public void setBytesUploaded(long bytesUploaded) { 50 | this.bytesUploaded = bytesUploaded; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | String s = ""; 56 | s+= "Uploaded "+bytesUploaded; 57 | s+= "/"+totalFileSize+" Bytes"; 58 | s+= "("+progress+"%)"; 59 | if (uploadRate != null) { 60 | s+= "at rate: "+UnitConverter.getFormattedSize(uploadRate) +"/s."; 61 | } 62 | if (estimatedRemainingTimeInSeconds != null) { 63 | s+= " Finishing in "+UnitConverter.getFormattedTime(estimatedRemainingTimeInSeconds)+"."; 64 | } 65 | return s; 66 | } 67 | 68 | /** 69 | * Please use {@link #getProgress()} 70 | * @return 71 | */ 72 | @Deprecated 73 | public Float getPercentageCompleted() { 74 | return getProgress(); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/entities/StaticFileState.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate.entities; 2 | 3 | 4 | import java.io.Serializable; 5 | 6 | import com.am.jlfu.fileuploader.json.FileStateJsonBase; 7 | 8 | 9 | /** 10 | * Server-side entity representing a file.
11 | * It contains the shared file information ({@link FileStateJsonBase}) and the url of the file. 12 | * @author antoinem 13 | * 14 | */ 15 | public class StaticFileState 16 | implements Serializable { 17 | 18 | 19 | /** generated id */ 20 | private static final long serialVersionUID = 2196169291933051657L; 21 | 22 | /** The full path url of the uploaded file. */ 23 | private String absoluteFullPathOfUploadedFile; 24 | 25 | /** The information related to the file upload. */ 26 | private FileStateJsonBase staticFileStateJson; 27 | 28 | 29 | 30 | /** 31 | * Default constructor. 32 | */ 33 | public StaticFileState() { 34 | super(); 35 | } 36 | 37 | 38 | public String getAbsoluteFullPathOfUploadedFile() { 39 | return absoluteFullPathOfUploadedFile; 40 | } 41 | 42 | 43 | public void setAbsoluteFullPathOfUploadedFile(String absoluteFullPathOfUploadedFile) { 44 | this.absoluteFullPathOfUploadedFile = absoluteFullPathOfUploadedFile; 45 | } 46 | 47 | 48 | public FileStateJsonBase getStaticFileStateJson() { 49 | return staticFileStateJson; 50 | } 51 | 52 | 53 | public void setStaticFileStateJson(FileStateJsonBase staticFileStateJson) { 54 | this.staticFileStateJson = staticFileStateJson; 55 | } 56 | 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/am/jlfu/staticstate/entities/StaticStatePersistedOnFileSystemEntity.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate.entities; 2 | 3 | 4 | import java.io.Serializable; 5 | import java.util.Map; 6 | import java.util.UUID; 7 | 8 | import com.am.jlfu.staticstate.StaticStateManager; 9 | import com.google.common.collect.Maps; 10 | 11 | 12 | 13 | /** 14 | * Abstract class that is persisted on the filesystem and contains information about the files being 15 | * uploaded.
16 | * You can of course extend it if you want to persist other stuff on the filesystem. If you do so, 17 | * you will have to call {@link StaticStateManager#init(Class)} with the type of the class you 18 | * defined extending this one. 19 | * 20 | * @author antoinem 21 | * 22 | */ 23 | public class StaticStatePersistedOnFileSystemEntity 24 | implements Serializable { 25 | 26 | /** generated id */ 27 | private static final long serialVersionUID = 6033009138577295466L; 28 | 29 | /** The states of the files being uploaded, the UUID being its identifier. */ 30 | private Map fileStates = Maps.newHashMap(); 31 | 32 | 33 | 34 | /** 35 | * Default constructor. 36 | */ 37 | public StaticStatePersistedOnFileSystemEntity() { 38 | super(); 39 | } 40 | 41 | 42 | public Map getFileStates() { 43 | return fileStates; 44 | } 45 | 46 | 47 | public void setFileStates(Map fileStates) { 48 | this.fileStates = fileStates; 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/jlfu-web-fragment-context.xml: -------------------------------------------------------------------------------- 1 | 2 | 14 | 15 | 16 | 18 | 19 | 20 | 21 | classpath:java-large-file-uploader.properties 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 47 | 48 | 50 | 51 | 52 | 53 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /src/main/resources/java-large-file-uploader.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freewebsys/java-large-file-uploader-demo/e301fa3461f7872b064dc9a9c64e854530a2fb91/src/main/resources/java-large-file-uploader.properties -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Loggers. 2 | 3 | log4j.rootLogger = DEBUG, console 4 | log4j.logger.org.springframework = WARN 5 | 6 | 7 | # Appenders. 8 | 9 | log4j.appender.console = org.apache.log4j.ConsoleAppender 10 | log4j.appender.console.layout = org.apache.log4j.PatternLayout 11 | log4j.appender.console.layout.ConversionPattern = %p %d{ISO8601} %C{1} %t %m %n 12 | -------------------------------------------------------------------------------- /src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | jlfuWebFragment 7 | 8 | 9 | 10 | contextConfigLocation 11 | classpath*:/META-INF/jlfu-web-fragment-context.xml 12 | 13 | 14 | 15 | org.springframework.web.context.ContextLoaderListener 16 | 17 | 18 | 19 | jlfuFilter 20 | org.springframework.web.filter.DelegatingFilterProxy 21 | true 22 | 23 | 24 | 25 | 26 | jlfuFilter 27 | /* 28 | 29 | 30 | 31 | 32 | UploadServlet 33 | com.am.jlfu.fileuploader.web.UploadServlet 34 | 35 | 36 | UploadServlet 37 | /javaLargeFileUploaderServlet 38 | 39 | 40 | 41 | UploadServletAsync 42 | com.am.jlfu.fileuploader.web.UploadServletAsync 43 | 44 | 45 | UploadServletAsync 46 | /javaLargeFileUploaderAsyncServlet 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/webapp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Java Large File Uploader Demo 4 | 5 | 6 | 7 | 8 | 218 | 219 | 220 | 221 |
222 |
223 |
224 |
225 | Pause all 226 | Resume all 227 | Clear all 228 |
229 | 230 | 231 | 232 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/authorizer/DefaultAuthorizerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.authorizer; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import javax.servlet.http.HttpServletRequest; 7 | 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 13 | 14 | import com.am.jlfu.fileuploader.exception.AuthorizationException; 15 | import com.am.jlfu.fileuploader.web.UploadServletAction; 16 | 17 | 18 | 19 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 20 | @RunWith(SpringJUnit4ClassRunner.class) 21 | public class DefaultAuthorizerTest { 22 | 23 | @Autowired 24 | Authorizer authorizer; 25 | 26 | 27 | 28 | @Test 29 | public void test() 30 | throws AuthorizationException { 31 | authorizer.getAuthorization(null, null, null, null); 32 | } 33 | 34 | 35 | @Test(expected = AuthorizationException.class) 36 | public void testException() 37 | throws AuthorizationException { 38 | new Authorizer() { 39 | 40 | @Override 41 | public void getAuthorization(HttpServletRequest request, UploadServletAction action, UUID clientId, UUID... optionalFileId) 42 | throws AuthorizationException { 43 | throw new AuthorizationException(action, clientId, optionalFileId); 44 | } 45 | }.getAuthorization(null, null, null, null); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/limiter/RateLimiterConfigurationManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | import java.util.UUID; 5 | import java.util.concurrent.ExecutionException; 6 | 7 | import org.hamcrest.CoreMatchers; 8 | import org.junit.Assert; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.mock.web.MockHttpServletRequest; 14 | import org.springframework.mock.web.MockHttpServletResponse; 15 | import org.springframework.test.context.ContextConfiguration; 16 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 17 | 18 | import com.am.jlfu.fileuploader.json.FileStateJsonBase; 19 | import com.am.jlfu.fileuploader.web.utils.RequestComponentContainer; 20 | import com.am.jlfu.notifier.JLFUListenerAdapter; 21 | import com.am.jlfu.notifier.JLFUListenerPropagator; 22 | import com.am.jlfu.staticstate.StaticStateIdentifierManager; 23 | import com.am.jlfu.staticstate.StaticStateManager; 24 | import com.am.jlfu.staticstate.entities.StaticFileState; 25 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 26 | import com.google.common.cache.RemovalCause; 27 | 28 | 29 | 30 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 31 | @RunWith(SpringJUnit4ClassRunner.class) 32 | public class RateLimiterConfigurationManagerTest { 33 | 34 | @Autowired 35 | RateLimiterConfigurationManager rateLimiterConfigurationManager; 36 | 37 | @Autowired 38 | JLFUListenerPropagator jlfuListenerPropagator; 39 | 40 | @Autowired 41 | StaticStateManager staticStateManager; 42 | 43 | @Autowired 44 | StaticStateIdentifierManager staticStateIdentifierManager; 45 | 46 | @Autowired 47 | RequestComponentContainer requestComponentContainer; 48 | 49 | private boolean assertt = false; 50 | 51 | 52 | 53 | @Before 54 | public void init() { 55 | 56 | // populate request component container 57 | requestComponentContainer.populate(new MockHttpServletRequest(), new MockHttpServletResponse()); 58 | 59 | } 60 | 61 | 62 | @Test 63 | public void testEvictionNotificationTrue() 64 | throws InterruptedException { 65 | testAssert(true); 66 | } 67 | 68 | 69 | @Test 70 | public void testEvictionNotificationFalse() 71 | throws InterruptedException { 72 | testAssert(false); 73 | } 74 | 75 | 76 | private void testAssert(boolean lala) throws InterruptedException { 77 | jlfuListenerPropagator.registerListener(new JLFUListenerAdapter() { 78 | 79 | @Override 80 | public void onClientInactivity(UUID clientId, int inactivityDuration) { 81 | assertt = true; 82 | } 83 | }); 84 | 85 | // emulate a pending upload 86 | final StaticStatePersistedOnFileSystemEntity entity = staticStateManager.getEntity(); 87 | final UUID identifier = staticStateIdentifierManager.getIdentifier(); 88 | rateLimiterConfigurationManager.configurationMap 89 | .put(identifier, new RequestUploadProcessingConfiguration()); 90 | rateLimiterConfigurationManager.configurationMap.getUnchecked(identifier); 91 | final StaticFileState value = new StaticFileState(); 92 | entity.getFileStates().put(identifier, value); 93 | final FileStateJsonBase staticFileStateJson = new FileStateJsonBase(); 94 | value.setStaticFileStateJson(staticFileStateJson); 95 | staticFileStateJson.setCrcedBytes(100l); 96 | 97 | if (lala) { 98 | staticFileStateJson.setOriginalFileSizeInBytes(10000l); 99 | } 100 | else { 101 | staticFileStateJson.setOriginalFileSizeInBytes(100l); 102 | } 103 | 104 | rateLimiterConfigurationManager.remove(RemovalCause.EXPIRED, identifier); 105 | 106 | Thread.sleep(100); 107 | if (lala) { 108 | Assert.assertThat(assertt, CoreMatchers.is(true)); 109 | } 110 | else { 111 | Assert.assertThat(assertt, CoreMatchers.is(false)); 112 | } 113 | } 114 | 115 | 116 | @Test 117 | public void testStreamExpectedToBeClosed() throws ExecutionException { 118 | UUID randomUUID = UUID.randomUUID(); 119 | rateLimiterConfigurationManager.configurationMap.put(randomUUID, new RequestUploadProcessingConfiguration()); 120 | Assert.assertThat(rateLimiterConfigurationManager.configurationMap.get(randomUUID).isPaused(), CoreMatchers.is(false)); 121 | rateLimiterConfigurationManager.configurationMap.get(randomUUID).pause(); 122 | Assert.assertThat(rateLimiterConfigurationManager.configurationMap.get(randomUUID).isPaused(), CoreMatchers.is(true)); 123 | Assert.assertThat(rateLimiterConfigurationManager.configurationMap.get(randomUUID).isPaused(), CoreMatchers.is(true)); 124 | rateLimiterConfigurationManager.configurationMap.get(randomUUID).resume(); 125 | Assert.assertThat(rateLimiterConfigurationManager.configurationMap.get(randomUUID).isPaused(), CoreMatchers.is(false)); 126 | 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/limiter/RateLimiterTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | import static org.hamcrest.Matchers.greaterThan; 5 | import static org.hamcrest.Matchers.lessThan; 6 | 7 | import java.util.Date; 8 | import java.util.List; 9 | import java.util.UUID; 10 | import java.util.concurrent.Callable; 11 | import java.util.concurrent.ExecutionException; 12 | import java.util.concurrent.ExecutorService; 13 | import java.util.concurrent.Executors; 14 | 15 | import org.junit.Assert; 16 | import org.junit.Test; 17 | import org.junit.runner.RunWith; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | import org.springframework.beans.factory.annotation.Autowired; 21 | import org.springframework.test.context.ContextConfiguration; 22 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 23 | 24 | import com.am.jlfu.fileuploader.logic.UploadServletAsyncProcessor; 25 | import com.google.common.collect.Lists; 26 | 27 | 28 | 29 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 30 | @RunWith(SpringJUnit4ClassRunner.class) 31 | // TODO make tests to check master and client limitations 32 | public class RateLimiterTest { 33 | 34 | private static final Logger log = LoggerFactory.getLogger(RateLimiterTest.class); 35 | 36 | @Autowired 37 | RateLimiterConfigurationManager uploadProcessingConfigurationManager; 38 | 39 | @Autowired 40 | UploadProcessingOperationManager uploadProcessingOperationManager; 41 | 42 | 43 | ExecutorService executorService = Executors.newFixedThreadPool(10); 44 | 45 | 46 | 47 | private void emulateUpload(Long requestRate, Long clientRate, Long masterRate, int uploadSizeInKB, int expectedDuration) 48 | throws InterruptedException { 49 | UUID fileId = UUID.randomUUID(); 50 | UUID clientId = UUID.randomUUID(); 51 | 52 | // extract config 53 | final RequestUploadProcessingConfiguration uploadProcessingConfiguration = 54 | uploadProcessingConfigurationManager.getUploadProcessingConfiguration(fileId); 55 | final UploadProcessingConfiguration clientProcessingConfiguration = 56 | uploadProcessingConfigurationManager.getUploadProcessingConfiguration(clientId); 57 | final UploadProcessingConfiguration masterProcessingConfiguration = 58 | uploadProcessingConfigurationManager.getMasterProcessingConfiguration(); 59 | 60 | // extract operation 61 | uploadProcessingOperationManager.startOperation(clientId, fileId); 62 | final UploadProcessingOperation clientProcessingOperation = uploadProcessingOperationManager.getClientProcessingOperation(clientId); 63 | final UploadProcessingOperation fileProcessingOperation = uploadProcessingOperationManager.getFileProcessingOperation(fileId); 64 | final UploadProcessingOperation masterProcessingOperation = uploadProcessingOperationManager.getMasterProcessingOperation(); 65 | 66 | // init request rate 67 | if (requestRate != null) { 68 | assignRateToRequest(requestRate, fileId, uploadProcessingConfiguration, fileProcessingOperation); 69 | } 70 | 71 | // perform upload 72 | long itTookThatLong = 73 | upload(clientId, fileId, uploadSizeInKB, uploadProcessingConfiguration, clientProcessingConfiguration, masterProcessingConfiguration, 74 | fileProcessingOperation, clientProcessingOperation, masterProcessingOperation); 75 | 76 | // specify completion 77 | uploadProcessingConfigurationManager.reset(fileId); 78 | 79 | // verify that it took around that duration 80 | itTookAround(expectedDuration, itTookThatLong); 81 | 82 | } 83 | 84 | 85 | private void itTookAround(int expectedDuration, Long itTookThatLong) { 86 | Assert.assertThat(itTookThatLong.floatValue(), lessThan((float) expectedDuration * 1.2f)); 87 | Assert.assertThat(itTookThatLong.floatValue(), greaterThan((float) expectedDuration * 0.8f)); 88 | } 89 | 90 | 91 | private void assignRateToRequest(Long requestRate, UUID id, final RequestUploadProcessingConfiguration uploadProcessingConfiguration, 92 | UploadProcessingOperation fileProcessingOperation) { 93 | 94 | uploadProcessingConfiguration.setProcessing(true); 95 | final long originalDownloadAllowanceForIteration = fileProcessingOperation.getDownloadAllowanceForIteration(); 96 | uploadProcessingConfigurationManager.assignRateToRequest(id, requestRate); 97 | 98 | // wait for the rate modification to occur 99 | while (fileProcessingOperation.getDownloadAllowanceForIteration() == originalDownloadAllowanceForIteration) { 100 | } 101 | } 102 | 103 | 104 | private long upload(UUID clientId, UUID fileId, int uploadSizeInKB, 105 | RequestUploadProcessingConfiguration requestUploadProcessingConfiguration, 106 | UploadProcessingConfiguration clientUploadProcessingConfiguration, UploadProcessingConfiguration masterUploadProcessingConfiguration, 107 | UploadProcessingOperation fileProcessingOperation, UploadProcessingOperation clientProcessingOperation, 108 | UploadProcessingOperation masterProcessingOperation) { 109 | 110 | // set the request as processing 111 | requestUploadProcessingConfiguration.setProcessing(true); 112 | 113 | // emulate an upload of a file 114 | long totalUpload = uploadSizeInKB * 1024; 115 | final Date reference = new Date(); 116 | long allowance; 117 | while (totalUpload > 0) { 118 | 119 | // calculate allowance 120 | allowance = UploadServletAsyncProcessor.minOf( 121 | (int) fileProcessingOperation.getDownloadAllowanceForIteration(), 122 | (int) clientProcessingOperation.getDownloadAllowanceForIteration(), 123 | (int) masterProcessingOperation.getDownloadAllowanceForIteration() 124 | ); 125 | 126 | // consumption 127 | fileProcessingOperation.bytesConsumedFromAllowance(allowance); 128 | clientProcessingOperation.bytesConsumedFromAllowance(allowance); 129 | masterProcessingOperation.bytesConsumedFromAllowance(allowance); 130 | 131 | totalUpload -= allowance; 132 | log.debug(clientId + " " + fileId + " uploaded " + totalUpload); 133 | } 134 | return new Date().getTime() - reference.getTime(); 135 | 136 | 137 | } 138 | 139 | 140 | @Test 141 | public void testMonoRequestLimitation() 142 | throws ExecutionException, InterruptedException { 143 | 144 | // lets say we upload a 1MB file. 145 | int upload = 1000; 146 | 147 | // set rate to 1MB, should have taken 1 second 148 | log.debug("testMonoRequestLimitation 1MB"); 149 | emulateUpload(1000l, null, null, upload, 1000); 150 | 151 | // set rate to 0.5MB, should have taken 2second 152 | log.debug("testMonoRequestLimitation 2MB"); 153 | emulateUpload(500l, null, null, upload, 2000); 154 | 155 | } 156 | 157 | 158 | @Test 159 | public void testClientRateLimitation() 160 | throws InterruptedException, ExecutionException { 161 | // client will limit 162 | testClientMaster(10000, 100000); 163 | 164 | // master will limit 165 | testClientMaster(100000, 10000); 166 | } 167 | 168 | 169 | private void testClientMaster(int client, int master) 170 | throws InterruptedException { 171 | 172 | // set client rate limitation 173 | uploadProcessingConfigurationManager.setMaximumRatePerClientInKiloBytes(client); 174 | 175 | // set master rate limitation 176 | uploadProcessingConfigurationManager.setMaximumOverAllRateInKiloBytes(master); 177 | 178 | final UUID clientId = UUID.randomUUID(); 179 | 180 | // and 10 requests are gonna upload a 10MB file 181 | int numberOfRequests = 10; 182 | List> runnables = Lists.newArrayList(); 183 | for (int i = 0; i < numberOfRequests; i++) { 184 | runnables.add(new TestRunnable(clientId, UUID.randomUUID(), 10000)); 185 | } 186 | 187 | // invoke 188 | final Date reference = new Date(); 189 | executorService.invokeAll(runnables); 190 | 191 | // shall have taken around 10seconds for all of them to complete 192 | itTookAround(10000, new Date().getTime() - reference.getTime()); 193 | } 194 | 195 | 196 | 197 | class TestRunnable 198 | implements Callable { 199 | 200 | private UUID clientId; 201 | private UUID fileId; 202 | private int fileSize; 203 | private RequestUploadProcessingConfiguration requestUploadProcessingConfiguration; 204 | private UploadProcessingConfiguration clientUploadProcessingConfiguration; 205 | private UploadProcessingConfiguration masterUploadProcessingConfiguration; 206 | private UploadProcessingOperation fileProcessingOperation; 207 | private UploadProcessingOperation masterProcessingOperation; 208 | private UploadProcessingOperation clientProcessingOperation; 209 | 210 | 211 | 212 | public TestRunnable(UUID clientId, UUID fileId, int fileSize) { 213 | this.clientId = clientId; 214 | this.fileId = fileId; 215 | this.fileSize = fileSize; 216 | requestUploadProcessingConfiguration = 217 | uploadProcessingConfigurationManager.getUploadProcessingConfiguration(fileId); 218 | clientUploadProcessingConfiguration = 219 | uploadProcessingConfigurationManager.getUploadProcessingConfiguration(clientId); 220 | masterUploadProcessingConfiguration = 221 | uploadProcessingConfigurationManager.getMasterProcessingConfiguration(); 222 | uploadProcessingOperationManager.startOperation(clientId, fileId); 223 | fileProcessingOperation = uploadProcessingOperationManager.getFileProcessingOperation(clientId); 224 | clientProcessingOperation = uploadProcessingOperationManager.getClientProcessingOperation(fileId); 225 | masterProcessingOperation = uploadProcessingOperationManager.getMasterProcessingOperation(); 226 | } 227 | 228 | 229 | @Override 230 | public Long call() { 231 | return upload(clientId, fileId, fileSize, requestUploadProcessingConfiguration, clientUploadProcessingConfiguration, 232 | masterUploadProcessingConfiguration, fileProcessingOperation, clientProcessingOperation, masterProcessingOperation); 233 | } 234 | } 235 | 236 | 237 | } 238 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/limiter/UploadProcessingOperationManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.limiter; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import org.hamcrest.CoreMatchers; 7 | import org.junit.Assert; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 14 | 15 | import com.am.jlfu.fileuploader.utils.ClientToFilesMap; 16 | 17 | 18 | 19 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 20 | @RunWith(SpringJUnit4ClassRunner.class) 21 | public class UploadProcessingOperationManagerTest { 22 | 23 | @Autowired 24 | UploadProcessingOperationManager uploadProcessingOperationManager; 25 | 26 | @Autowired 27 | ClientToFilesMap clientToFilesMap; 28 | 29 | 30 | @Before 31 | public void before() { 32 | uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.clear(); 33 | clientToFilesMap.clear(); 34 | } 35 | 36 | 37 | @Test 38 | public void test() { 39 | 40 | UUID clientId = UUID.randomUUID(); 41 | UUID fileId = UUID.randomUUID(); 42 | UUID fileId2 = UUID.randomUUID(); 43 | 44 | Assert.assertThat(uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.isEmpty(), CoreMatchers.is(true)); 45 | Assert.assertThat(clientToFilesMap.isEmpty(), CoreMatchers.is(true)); 46 | 47 | uploadProcessingOperationManager.startOperation(clientId, fileId); 48 | 49 | Assert.assertThat(uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.containsKey(clientId), CoreMatchers.is(true)); 50 | Assert.assertThat(clientToFilesMap.get(clientId).contains(fileId), CoreMatchers.is(true)); 51 | 52 | uploadProcessingOperationManager.startOperation(clientId, fileId2); 53 | 54 | Assert.assertThat(uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.containsKey(clientId), CoreMatchers.is(true)); 55 | Assert.assertThat(clientToFilesMap.get(clientId).contains(fileId), CoreMatchers.is(true)); 56 | Assert.assertThat(clientToFilesMap.get(clientId).contains(fileId2), CoreMatchers.is(true)); 57 | 58 | uploadProcessingOperationManager.stopOperation(clientId, fileId2); 59 | 60 | Assert.assertThat(uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.containsKey(clientId), CoreMatchers.is(true)); 61 | Assert.assertThat(clientToFilesMap.get(clientId).contains(fileId), CoreMatchers.is(true)); 62 | Assert.assertThat(clientToFilesMap.get(clientId).contains(fileId2), CoreMatchers.is(false)); 63 | 64 | uploadProcessingOperationManager.stopOperation(clientId, fileId); 65 | 66 | Assert.assertThat(uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.containsKey(clientId), CoreMatchers.is(false)); 67 | Assert.assertThat(clientToFilesMap.containsKey(clientId), CoreMatchers.is(false)); 68 | Assert.assertThat(uploadProcessingOperationManager.clientsAndRequestsProcessingOperation.isEmpty(), CoreMatchers.is(true)); 69 | Assert.assertThat(clientToFilesMap.isEmpty(), CoreMatchers.is(true)); 70 | 71 | 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/logic/UploadProcessorTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.logic; 2 | 3 | 4 | import static org.hamcrest.CoreMatchers.is; 5 | 6 | import java.io.ByteArrayInputStream; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.util.UUID; 10 | import java.util.concurrent.ExecutionException; 11 | import java.util.concurrent.TimeoutException; 12 | import java.util.zip.CRC32; 13 | 14 | import javax.servlet.ServletException; 15 | 16 | import org.apache.commons.io.IOUtils; 17 | import org.junit.Assert; 18 | import org.junit.Before; 19 | import org.junit.Test; 20 | import org.junit.runner.RunWith; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | import org.springframework.beans.factory.annotation.Autowired; 24 | import org.springframework.mock.web.MockHttpServletRequest; 25 | import org.springframework.mock.web.MockHttpServletResponse; 26 | import org.springframework.mock.web.MockMultipartFile; 27 | import org.springframework.test.context.ContextConfiguration; 28 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 29 | import org.springframework.web.multipart.MultipartFile; 30 | 31 | import com.am.jlfu.fileuploader.json.CRCResult; 32 | import com.am.jlfu.fileuploader.json.InitializationConfiguration; 33 | import com.am.jlfu.fileuploader.utils.CRCHelper; 34 | import com.am.jlfu.fileuploader.web.UploadServletAsync; 35 | import com.am.jlfu.fileuploader.web.utils.RequestComponentContainer; 36 | import com.am.jlfu.staticstate.StaticStateIdentifierManager; 37 | import com.am.jlfu.staticstate.StaticStateManager; 38 | import com.am.jlfu.staticstate.entities.StaticFileState; 39 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 40 | 41 | 42 | 43 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 44 | @RunWith(SpringJUnit4ClassRunner.class) 45 | public class UploadProcessorTest { 46 | 47 | private static final Logger log = LoggerFactory.getLogger(UploadProcessorTest.class); 48 | 49 | @Autowired 50 | CRCHelper crcHelper; 51 | 52 | @Autowired 53 | UploadProcessor uploadProcessor; 54 | 55 | @Autowired 56 | UploadServletAsync uploadServletAsync; 57 | 58 | @Autowired 59 | StaticStateManager staticStateManager; 60 | 61 | @Autowired 62 | StaticStateIdentifierManager staticStateIdentifierManager; 63 | 64 | @Autowired 65 | RequestComponentContainer requestComponentContainer; 66 | 67 | MockMultipartFile file; 68 | 69 | String fileName = "zenameofzefile.owf"; 70 | 71 | private Long fileSize; 72 | 73 | private byte[] content; 74 | 75 | 76 | 77 | @Before 78 | public void init() 79 | throws IOException, InterruptedException, ExecutionException, TimeoutException { 80 | 81 | // populate request component container 82 | requestComponentContainer.populate(new MockHttpServletRequest(), new MockHttpServletResponse()); 83 | 84 | 85 | staticStateManager.clear(); 86 | content = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 }; 87 | file = new MockMultipartFile("blob", content); 88 | fileSize = Integer.valueOf(content.length).longValue(); 89 | } 90 | 91 | 92 | private void assertState(StaticFileState state, boolean absolutePathOfUploadedFileFilled, Boolean fileComplete, String originalFileName, 93 | Long fileSize, 94 | Long completion) { 95 | Assert.assertNotNull(state); 96 | Assert.assertNotNull(state.getStaticFileStateJson()); 97 | if (absolutePathOfUploadedFileFilled) { 98 | Assert.assertNotNull(state.getAbsoluteFullPathOfUploadedFile()); 99 | } 100 | else { 101 | Assert.assertNull(state.getAbsoluteFullPathOfUploadedFile()); 102 | } 103 | Assert.assertEquals(fileName, state.getStaticFileStateJson().getOriginalFileName()); 104 | Assert.assertEquals(fileSize, state.getStaticFileStateJson().getOriginalFileSizeInBytes()); 105 | } 106 | 107 | 108 | 109 | public static class TestFileSplitResult { 110 | 111 | ByteArrayInputStream stream; 112 | String crc; 113 | } 114 | 115 | 116 | 117 | public static TestFileSplitResult getByteArrayFromInputStream(InputStream inputStream, long start, long length) 118 | throws IOException { 119 | TestFileSplitResult testFileSplitResult = new TestFileSplitResult(); 120 | 121 | inputStream.skip(start); 122 | 123 | // read file 124 | byte[] b = new byte[Math.min((int) (length - start), inputStream.available())]; 125 | inputStream.read(b, 0, b.length); 126 | inputStream.close(); 127 | testFileSplitResult.stream = new ByteArrayInputStream(b); 128 | 129 | // get crc 130 | CRC32 crc32 = new CRC32(); 131 | crc32.update(b); 132 | testFileSplitResult.crc = Long.toHexString(crc32.getValue()); 133 | 134 | return testFileSplitResult; 135 | } 136 | 137 | 138 | public static TestFileSplitResult getByteArrayFromFile(MultipartFile file2, long start, long length) 139 | throws IOException { 140 | InputStream inputStream = file2.getInputStream(); 141 | return getByteArrayFromInputStream(inputStream, start, length); 142 | } 143 | 144 | 145 | @Test 146 | public void testCancelFileUpload() 147 | throws ServletException, IOException, InterruptedException, ExecutionException, TimeoutException { 148 | 149 | // begin a file upload process 150 | UUID fileId = uploadProcessor.prepareUpload(fileSize, fileName, "lala"); 151 | 152 | // assert that the state has what we want 153 | StaticFileState value = staticStateManager.getEntity().getFileStates().get(fileId); 154 | assertState(value, true, false, fileName, fileSize, 0l); 155 | 156 | // assert that we have it in the pending files 157 | Assert.assertThat(uploadProcessor.getConfig(null).getPendingFiles().keySet().toArray()[0].toString(), is(fileId.toString())); 158 | 159 | // cancel 160 | uploadProcessor.clearFile(fileId); 161 | 162 | // assert that file is reset 163 | Assert.assertThat(staticStateManager.getEntity().getFileStates().containsKey(fileId), is(false)); 164 | 165 | // assert that we dont have it in the pending files anymore 166 | Assert.assertThat(uploadProcessor.getConfig(null).getPendingFiles().containsKey(fileId), is(false)); 167 | } 168 | 169 | 170 | @Test 171 | public void testConfig() 172 | throws IOException { 173 | InitializationConfiguration config = uploadProcessor.getConfig(null); 174 | Assert.assertNotNull(config.getInByte()); 175 | } 176 | 177 | 178 | @Test 179 | public void testIdSpecification() { 180 | UUID randomUUID = UUID.randomUUID(); 181 | uploadProcessor.getConfig(randomUUID); 182 | Assert.assertThat(staticStateIdentifierManager.getIdentifier(), is(randomUUID)); 183 | } 184 | 185 | 186 | @Test 187 | public void testIdReSpecification() { 188 | testIdSpecification(); 189 | UUID randomUUID = UUID.randomUUID(); 190 | uploadProcessor.getConfig(randomUUID); 191 | Assert.assertThat(staticStateIdentifierManager.getIdentifier(), is(randomUUID)); 192 | } 193 | 194 | 195 | @Test 196 | public void testCrcBuffered() 197 | throws IOException { 198 | 199 | // with method 200 | CRCResult withMethod = crcHelper.getBufferedCrc(file.getInputStream()); 201 | 202 | // without buffer 203 | CRC32 crc32 = new CRC32(); 204 | crc32.update(IOUtils.toByteArray(file.getInputStream())); 205 | String hexString = Long.toHexString(crc32.getValue()); 206 | 207 | Assert.assertThat(withMethod.getCrcAsString(), is(hexString)); 208 | Assert.assertThat(withMethod.getTotalRead(), is(content.length)); 209 | 210 | } 211 | 212 | } 213 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/util/RootFolderProvider.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.util; 2 | 3 | 4 | import java.io.File; 5 | import java.io.IOException; 6 | 7 | import org.springframework.context.annotation.Primary; 8 | import org.springframework.stereotype.Component; 9 | 10 | import com.am.jlfu.staticstate.StaticStateRootFolderProvider; 11 | 12 | 13 | 14 | @Component 15 | @Primary 16 | public class RootFolderProvider 17 | extends StaticStateRootFolderProvider { 18 | 19 | private File file; 20 | 21 | 22 | 23 | @Override 24 | public File getRootFolder() { 25 | if (file == null) { 26 | try { 27 | file = File.createTempFile("lala", "test"); 28 | file.delete(); 29 | file.mkdir(); 30 | } 31 | catch (IOException e) { 32 | e.printStackTrace(); 33 | } 34 | } 35 | return file; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/util/StaticStateIdentifierManagerForTestProvider.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.util; 2 | 3 | 4 | import org.springframework.context.annotation.Primary; 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.am.jlfu.staticstate.StaticStateIdentifierManager; 8 | 9 | 10 | 11 | @Component 12 | @Primary 13 | public class StaticStateIdentifierManagerForTestProvider extends StaticStateIdentifierManager { 14 | 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/utils/ImportedFilesCleanerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | 4 | import java.io.File; 5 | import java.io.IOException; 6 | 7 | import org.joda.time.DateTime; 8 | import org.junit.Assert; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 14 | 15 | import com.am.jlfu.staticstate.FileDeleter; 16 | import com.am.jlfu.staticstate.StaticStateRootFolderProvider; 17 | 18 | 19 | 20 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 21 | @RunWith(SpringJUnit4ClassRunner.class) 22 | public class ImportedFilesCleanerTest { 23 | 24 | @Autowired 25 | StaticStateRootFolderProvider staticStateRootFolderProvider; 26 | 27 | @Autowired 28 | ImportedFilesCleaner importedFilesCleaner; 29 | 30 | @Autowired 31 | FileDeleter fileDeleter; 32 | 33 | 34 | 35 | @Test 36 | public void test() 37 | throws IOException { 38 | // put some files 39 | File rootFolder = staticStateRootFolderProvider.getRootFolder(); 40 | 41 | // old one 42 | File oldDir = new File(rootFolder, "oldDir"); 43 | oldDir.mkdir(); 44 | oldDir.setLastModified(new DateTime().minusMonths(3).getMillis()); 45 | 46 | // recent one 47 | File recentDir = new File(rootFolder, "recentDir"); 48 | recentDir.mkdir(); 49 | 50 | // process 51 | importedFilesCleaner.clean(); 52 | 53 | // call file deleter 54 | fileDeleter.run(); 55 | 56 | // assume old is deleted 57 | Assert.assertFalse(oldDir.exists()); 58 | 59 | // assume new os still there 60 | Assert.assertTrue(recentDir.exists()); 61 | 62 | 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/utils/LimitingListTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import org.hamcrest.CoreMatchers; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.springframework.test.context.ContextConfiguration; 8 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 9 | 10 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 11 | @RunWith(SpringJUnit4ClassRunner.class) 12 | public class LimitingListTest { 13 | 14 | 15 | @Test 16 | public void test() { 17 | Integer a = 1; 18 | Integer b = 2; 19 | Integer c = 3; 20 | Integer d = 4; 21 | LimitingList limitingList = new LimitingList(2); 22 | limitingList.unshift(a); 23 | limitingList.unshift(b); 24 | Assert.assertThat(limitingList.list.get(0), CoreMatchers.is(b)); 25 | Assert.assertThat(limitingList.list.get(1), CoreMatchers.is(a)); 26 | limitingList.unshift(c); 27 | Assert.assertThat(limitingList.list.get(0), CoreMatchers.is(c)); 28 | Assert.assertThat(limitingList.list.get(1), CoreMatchers.is(b)); 29 | Assert.assertThat(limitingList.list.size(), CoreMatchers.is(2)); 30 | limitingList.unshift(d); 31 | Assert.assertThat(limitingList.list.get(0), CoreMatchers.is(d)); 32 | Assert.assertThat(limitingList.list.get(1), CoreMatchers.is(c)); 33 | Assert.assertThat(limitingList.list.size(), CoreMatchers.is(2)); 34 | 35 | 36 | } 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/utils/ProgressCalculatorTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import static org.hamcrest.CoreMatchers.is; 4 | import static org.hamcrest.CoreMatchers.not; 5 | 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.test.context.ContextConfiguration; 11 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 12 | 13 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 14 | @RunWith(SpringJUnit4ClassRunner.class) 15 | public class ProgressCalculatorTest { 16 | 17 | @Autowired 18 | ProgressCalculator progressCalculator; 19 | 20 | @Test 21 | public void progressCalculationTest() { 22 | // check basic 30% (30/100) 23 | Assert.assertThat(Double.valueOf(30), is(progressCalculator.calculateProgress(30l, 100l))); 24 | Long bigValue = 1000000000000000000l; 25 | // check that we dont return 100% if values are not exactly equals 26 | Assert.assertThat(Double.valueOf(100), is(not(progressCalculator.calculateProgress(bigValue - 1, bigValue)))); 27 | // check that we return 100% if values are equals 28 | Assert.assertThat(Double.valueOf(100), is(progressCalculator.calculateProgress(bigValue, bigValue))); 29 | // check that we return 0% when 0/x 30 | Assert.assertThat(Double.valueOf(0), is(progressCalculator.calculateProgress(0l, 240l))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/utils/ProgressManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import java.io.FileNotFoundException; 4 | import java.util.Set; 5 | import java.util.UUID; 6 | 7 | import org.hamcrest.CoreMatchers; 8 | import org.junit.Assert; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.test.context.ContextConfiguration; 14 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 15 | import org.springframework.test.util.ReflectionTestUtils; 16 | import org.unitils.mock.Mock; 17 | import org.unitils.mock.core.MockObject; 18 | 19 | import com.am.jlfu.fileuploader.utils.ProgressManager.ProgressManagerAdvertiser; 20 | import com.am.jlfu.notifier.JLFUListenerPropagator; 21 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 22 | import com.google.common.collect.Sets; 23 | 24 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 25 | @RunWith(SpringJUnit4ClassRunner.class) 26 | public class ProgressManagerTest { 27 | 28 | @Autowired 29 | ClientToFilesMap clientToFilesMap; 30 | 31 | @Autowired 32 | ProgressManager progressManager; 33 | 34 | @Autowired 35 | JLFUListenerPropagator jlfuListenerPropagator; 36 | 37 | Mock progressCalculator = new MockObject(ProgressCalculator.class, new Object()); 38 | Mock progressManagerAdvertiser = new MockObject(ProgressManagerAdvertiser.class, new Object()); 39 | 40 | private UUID clientId = UUID.randomUUID(); 41 | private UUID fileId = UUID.randomUUID(); 42 | 43 | @Before 44 | public void init() { 45 | 46 | //init client to files map 47 | clientToFilesMap.clear(); 48 | Set newHashSet = Sets.newHashSet(); 49 | clientToFilesMap.put(clientId, newHashSet); 50 | newHashSet.add(fileId); 51 | 52 | //reset progress manager map 53 | progressManager.fileToProgressInfo.clear(); 54 | 55 | //set mock 56 | ReflectionTestUtils.setField(progressManager, "progressCalculator", progressCalculator.getMock()); 57 | ReflectionTestUtils.setField(progressManager, "progressManagerAdvertiser", progressManagerAdvertiser.getMock()); 58 | 59 | } 60 | 61 | @Test 62 | public void testWithProgress() throws FileNotFoundException { 63 | assertReturnedIsCorrect(15f, true); 64 | assertReturnedIsCorrect(30f, true); 65 | assertReturnedIsCorrect(30f, false); 66 | } 67 | 68 | private void assertReturnedIsCorrect(float returnedValue, boolean shallBePropagated) 69 | throws FileNotFoundException { 70 | 71 | //mock service 72 | FileProgressStatus fileProgressStatus = new FileProgressStatus(); 73 | fileProgressStatus.setProgress(returnedValue); 74 | progressCalculator.onceReturns(fileProgressStatus).getProgress(clientId, fileId); 75 | 76 | //calculate progress 77 | progressManager.calculateProgress(); 78 | 79 | //assert map is filled 80 | Assert.assertThat(progressManager.fileToProgressInfo.get(fileId).getProgress(), CoreMatchers.is(returnedValue)); 81 | 82 | //assert that event is propagated 83 | if (shallBePropagated) { 84 | progressManagerAdvertiser.assertInvoked().advertise(clientId, fileId, fileProgressStatus); 85 | } else { 86 | progressManagerAdvertiser.assertNotInvoked().advertise(clientId, fileId, fileProgressStatus); 87 | } 88 | } 89 | 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/utils/RemainingTimeEstimatorTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.utils; 2 | 3 | import java.util.UUID; 4 | 5 | import org.hamcrest.CoreMatchers; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.test.context.ContextConfiguration; 11 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 12 | 13 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 14 | 15 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 16 | @RunWith(SpringJUnit4ClassRunner.class) 17 | public class RemainingTimeEstimatorTest { 18 | 19 | @Autowired 20 | RemainingTimeEstimator remainingTimeEstimator; 21 | 22 | @Test 23 | public void testGetRemainingTime() { 24 | UUID clientId = UUID.randomUUID(); 25 | 26 | FileProgressStatus progress = new FileProgressStatus(); 27 | progress.setTotalFileSize(1000); 28 | progress.setBytesUploaded(0); 29 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 100l), CoreMatchers.is(10l)); 30 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 300l), CoreMatchers.is(5l)); 31 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 32 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 33 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 34 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 35 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 36 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 37 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 38 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.is(5l)); 39 | Assert.assertThat(remainingTimeEstimator.getRemainingTime(clientId, progress, 200l), CoreMatchers.not(5l)); 40 | } 41 | 42 | 43 | 44 | @Test 45 | public void testCalculateRemainingTime() { 46 | processRemainingTimeTest(1000l, 0l, 100l, 10l); 47 | processRemainingTimeTest(1000l, 500l, 100l, 5l); 48 | processRemainingTimeTest(1000l, 1000l, 100l, 1l); 49 | } 50 | 51 | 52 | private void processRemainingTimeTest(long fileSize, long start, long rate, long expectedSeconds) { 53 | FileProgressStatus progress = new FileProgressStatus(); 54 | progress.setTotalFileSize(fileSize); 55 | progress.setBytesUploaded(start); 56 | long calculateRemainingTime = remainingTimeEstimator.calculateRemainingTime(progress, rate); 57 | Assert.assertThat(calculateRemainingTime, CoreMatchers.is(expectedSeconds)); 58 | } 59 | 60 | 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/fileuploader/web/UploadServletTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.fileuploader.web; 2 | 3 | 4 | import static org.hamcrest.CoreMatchers.is; 5 | 6 | import java.io.IOException; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.UUID; 11 | 12 | import javax.servlet.ServletException; 13 | 14 | import org.hamcrest.CoreMatchers; 15 | import org.junit.Assert; 16 | import org.junit.Before; 17 | import org.junit.Test; 18 | import org.junit.runner.RunWith; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.mock.web.MockHttpServletRequest; 21 | import org.springframework.mock.web.MockHttpServletResponse; 22 | import org.springframework.mock.web.MockMultipartFile; 23 | import org.springframework.test.context.ContextConfiguration; 24 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 25 | 26 | import com.am.jlfu.fileuploader.json.FileStateJson; 27 | import com.am.jlfu.fileuploader.json.InitializationConfiguration; 28 | import com.am.jlfu.fileuploader.json.PrepareUploadJson; 29 | import com.am.jlfu.fileuploader.json.ProgressJson; 30 | import com.am.jlfu.fileuploader.json.SimpleJsonObject; 31 | import com.am.jlfu.fileuploader.web.utils.ExceptionCodeMappingHelper.ExceptionCodeMapping; 32 | import com.am.jlfu.fileuploader.web.utils.RequestComponentContainer; 33 | import com.google.gson.Gson; 34 | import com.google.gson.reflect.TypeToken; 35 | 36 | 37 | 38 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 39 | @RunWith(SpringJUnit4ClassRunner.class) 40 | public class UploadServletTest { 41 | 42 | @Autowired 43 | UploadServlet uploadServlet; 44 | 45 | @Autowired 46 | UploadServletAsync uploadServletAsync; 47 | 48 | @Autowired 49 | RequestComponentContainer requestComponentContainer; 50 | 51 | MockHttpServletRequest request; 52 | MockHttpServletResponse response; 53 | 54 | private byte[] content = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 }; 55 | private MockMultipartFile file = new MockMultipartFile("blob", content); 56 | 57 | 58 | 59 | @Before 60 | public void init() { 61 | 62 | request = new MockHttpServletRequest(); 63 | response = new MockHttpServletResponse(); 64 | 65 | // populate request component container 66 | requestComponentContainer.populate(request, response); 67 | 68 | } 69 | 70 | 71 | @Test 72 | public void getConfig() 73 | throws IOException { 74 | 75 | // init an upload to emulate a pending file 76 | String fileId = prepareUpload(); 77 | 78 | // set action parameter 79 | request.clearAttributes(); 80 | response = new MockHttpServletResponse(); 81 | request.setParameter(UploadServletParameter.action.name(), UploadServletAction.getConfig.name()); 82 | 83 | // handle request 84 | uploadServlet.handleRequest(request, response); 85 | 86 | // extract config from response 87 | InitializationConfiguration fromJson = new Gson().fromJson(response.getContentAsString(), InitializationConfiguration.class); 88 | Assert.assertNotNull(fromJson.getInByte()); 89 | Assert.assertThat(response.getStatus(), is(200)); 90 | Map pendingFiles = fromJson.getPendingFiles(); 91 | Assert.assertThat(pendingFiles.size(), is(1)); 92 | Assert.assertThat(pendingFiles.keySet().iterator().next(), is(fileId)); 93 | } 94 | 95 | 96 | @Test 97 | public void getProgressWithBadId() 98 | throws IOException { 99 | 100 | // set action parameter 101 | request.setParameter(UploadServletParameter.action.name(), UploadServletAction.getProgress.name()); 102 | String id = "a bad id"; 103 | request.setParameter(UploadServletParameter.fileId.name(), new Gson().toJson(new String[] { id })); 104 | 105 | // handle request 106 | uploadServlet.handleRequest(request, response); 107 | 108 | SimpleJsonObject fromJson = new Gson().fromJson(response.getContentAsString(), SimpleJsonObject.class); 109 | Assert.assertThat(fromJson.getValue(), is("0")); 110 | 111 | } 112 | 113 | 114 | @Test 115 | public void getProgress() 116 | throws IOException { 117 | 118 | // init an upload to emulate a pending file 119 | String fileId = prepareUpload(); 120 | 121 | // set action parameter 122 | request.clearAttributes(); 123 | response = new MockHttpServletResponse(); 124 | request.setParameter(UploadServletParameter.action.name(), UploadServletAction.getProgress.name()); 125 | request.setParameter(UploadServletParameter.fileId.name(), new Gson().toJson(new String[] { fileId })); 126 | 127 | // handle request 128 | uploadServlet.handleRequest(request, response); 129 | Assert.assertThat(response.getStatus(), is(200)); 130 | 131 | HashMap fromJson = new Gson().fromJson(response.getContentAsString(), new TypeToken>() { 132 | }.getType()); 133 | ProgressJson[] array = new ProgressJson[] {}; 134 | array = fromJson.values().toArray(array); 135 | Assert.assertThat(array[0].getProgress(), is(Float.valueOf(0))); 136 | 137 | } 138 | 139 | 140 | @Test 141 | public void uploadNotMultipartParams() 142 | throws IOException, ServletException { 143 | 144 | // handle request 145 | uploadServletAsync.handleRequest(request, response); 146 | SimpleJsonObject fromJson = new Gson().fromJson(response.getContentAsString(), SimpleJsonObject.class); 147 | Assert.assertThat(ExceptionCodeMapping.requestIsNotMultipart.getExceptionIdentifier(), is(Integer.valueOf(fromJson.getValue()))); 148 | 149 | } 150 | 151 | 152 | @Test 153 | public void prepareUploadTest() 154 | throws IOException { 155 | prepareUpload(); 156 | } 157 | 158 | 159 | public String prepareUpload() 160 | throws IOException { 161 | 162 | return (String) prepareUpload(1).values().toArray()[0]; 163 | 164 | } 165 | 166 | 167 | public Map prepareUpload(int size) 168 | throws IOException { 169 | 170 | // set action parameter 171 | request.setParameter(UploadServletParameter.action.name(), UploadServletAction.prepareUpload.name()); 172 | PrepareUploadJson[] prepareUploadJsons = new PrepareUploadJson[size]; 173 | for (int i = 0; i < size; i++) { 174 | PrepareUploadJson j = new PrepareUploadJson(); 175 | j.setTempId(i); 176 | j.setFileName("file " + i); 177 | j.setSize(123456l); 178 | prepareUploadJsons[i] = j; 179 | } 180 | request.setParameter(UploadServletParameter.newFiles.name(), new Gson().toJson(prepareUploadJsons)); 181 | 182 | // handle request 183 | uploadServlet.handleRequest(request, response); 184 | Assert.assertThat(response.getStatus(), is(200)); 185 | HashMap fromJson = new Gson().fromJson(response.getContentAsString(), new TypeToken>() { 186 | }.getType()); 187 | 188 | return fromJson; 189 | } 190 | 191 | 192 | @Test 193 | public void prepareUploadMulti() 194 | throws IOException { 195 | Map prepareUpload = prepareUpload(10); 196 | Assert.assertThat(prepareUpload.size(), is(10)); 197 | } 198 | 199 | 200 | // upload, 201 | // prepareUpload, 202 | // clearFile, 203 | // clearAll; 204 | 205 | 206 | @Test 207 | public void clearFileWithMissingParameter() 208 | throws IOException { 209 | 210 | // set action parameter 211 | request.setParameter(UploadServletParameter.action.name(), UploadServletAction.clearFile.name()); 212 | 213 | // handle request 214 | uploadServlet.handleRequest(request, response); 215 | 216 | // assert that we have an error 217 | SimpleJsonObject fromJson = new Gson().fromJson(response.getContentAsString(), SimpleJsonObject.class); 218 | 219 | Assert.assertThat(ExceptionCodeMapping.MissingParameterException.getExceptionIdentifier(), is(Integer.valueOf(fromJson.getValue()))); 220 | 221 | } 222 | 223 | 224 | @Test 225 | public void testGetMultiFileIdsFromString() { 226 | UUID uuid1 = UUID.randomUUID(); 227 | UUID uuid2 = UUID.randomUUID(); 228 | List fileIdsFromString = uploadServlet.getFileIdsFromString(uuid1+","+uuid2); 229 | Assert.assertThat(fileIdsFromString.size(), CoreMatchers.is(2)); 230 | Assert.assertTrue(fileIdsFromString.contains(uuid1)); 231 | Assert.assertTrue(fileIdsFromString.contains(uuid2)); 232 | } 233 | 234 | @Test 235 | public void testGetOneFileIdFromString() { 236 | UUID uuid1 = UUID.randomUUID(); 237 | List fileIdsFromString = uploadServlet.getFileIdsFromString(uuid1.toString()); 238 | Assert.assertThat(fileIdsFromString.size(), CoreMatchers.is(1)); 239 | Assert.assertTrue(fileIdsFromString.contains(uuid1)); 240 | } 241 | 242 | 243 | } 244 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/notifier/JLFUListenerPropagatorTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.notifier; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import org.hamcrest.CoreMatchers; 7 | import org.junit.Assert; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 14 | 15 | import com.am.jlfu.staticstate.entities.FileProgressStatus; 16 | 17 | 18 | 19 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 20 | @RunWith(SpringJUnit4ClassRunner.class) 21 | public class JLFUListenerPropagatorTest { 22 | 23 | @Autowired 24 | JLFUListenerPropagator jlfuListenerPropagator; 25 | 26 | private volatile int testCounter; 27 | 28 | private JLFUListenerAdapter listener; 29 | 30 | 31 | 32 | @Before 33 | public void before() { 34 | jlfuListenerPropagator.unregisterAllListeners(); 35 | 36 | testCounter = 0; 37 | listener = new JLFUListenerAdapter() { 38 | 39 | @Override 40 | public void onNewClient(UUID clientId) { 41 | testCounter++; 42 | } 43 | }; 44 | 45 | } 46 | 47 | 48 | @Test 49 | public void test() throws InterruptedException { 50 | 51 | // add two listener 52 | jlfuListenerPropagator.registerListener(listener); 53 | jlfuListenerPropagator.registerListener(listener); 54 | 55 | // trigger event 56 | jlfuListenerPropagator.getPropagator().onNewClient(UUID.randomUUID()); 57 | Thread.sleep(100); 58 | 59 | // assert 60 | Assert.assertThat(testCounter, CoreMatchers.is(2)); 61 | 62 | // unregister one listener 63 | jlfuListenerPropagator.unregisterListener(listener); 64 | 65 | // trigger event 66 | jlfuListenerPropagator.getPropagator().onNewClient(UUID.randomUUID()); 67 | Thread.sleep(100); 68 | 69 | // assert 70 | Assert.assertThat(testCounter, CoreMatchers.is(3)); 71 | 72 | } 73 | 74 | @Test 75 | public void testNotBlocked() { 76 | jlfuListenerPropagator.registerListener(new JLFUListenerAdapter() { 77 | @Override 78 | public void onClientBack(UUID clientId) { 79 | try { 80 | Thread.sleep(10000); 81 | } 82 | catch (InterruptedException e) { 83 | e.printStackTrace(); 84 | } 85 | Assert.fail(); 86 | } 87 | }); 88 | jlfuListenerPropagator.getPropagator().onClientBack(UUID.randomUUID()); 89 | } 90 | 91 | @Test 92 | public void log() { 93 | jlfuListenerPropagator.registerListener(new JLFUListenerAdapter()); 94 | FileProgressStatus progress = new FileProgressStatus(); 95 | progress.setBytesUploaded(123); 96 | progress.setProgress(1234f); 97 | progress.setTotalFileSize(12340124); 98 | jlfuListenerPropagator.getPropagator().onFileUploadProgress(UUID.randomUUID(),UUID.randomUUID(), progress); 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/staticstate/FileDeleterTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import static org.hamcrest.CoreMatchers.is; 5 | 6 | import java.io.File; 7 | import java.io.FileInputStream; 8 | import java.io.IOException; 9 | import java.util.List; 10 | import java.util.concurrent.ExecutorService; 11 | import java.util.concurrent.Executors; 12 | import java.util.concurrent.TimeUnit; 13 | 14 | import org.junit.Assert; 15 | import org.junit.Before; 16 | import org.junit.Test; 17 | import org.junit.runner.RunWith; 18 | import org.springframework.beans.factory.annotation.Autowired; 19 | import org.springframework.test.context.ContextConfiguration; 20 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 21 | 22 | import com.google.common.collect.Lists; 23 | import com.google.common.util.concurrent.Futures; 24 | import com.google.common.util.concurrent.ListenableFuture; 25 | import com.google.common.util.concurrent.MoreExecutors; 26 | 27 | 28 | 29 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 30 | @RunWith(SpringJUnit4ClassRunner.class) 31 | public class FileDeleterTest { 32 | 33 | @Autowired 34 | FileDeleter fileDeleter; 35 | 36 | @Autowired 37 | StaticStateRootFolderProvider staticStateRootFolderProvider; 38 | 39 | private int number = 100; 40 | private ExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(20)); 41 | private File[] files = new File[number]; 42 | 43 | 44 | 45 | @Before 46 | public void writeFiles() 47 | throws IOException { 48 | for (int i = 0; i < number; i++) { 49 | if (i % 2 == 0) { 50 | File file = new File(staticStateRootFolderProvider.getRootFolder(), i + "testdir"); 51 | file.mkdirs(); 52 | files[i] = file; 53 | } 54 | else { 55 | files[i] = File.createTempFile("temp", "temp"); 56 | } 57 | } 58 | } 59 | 60 | 61 | @Test 62 | public void deleteMultipleFilesSubmittedConcurrently() 63 | throws Exception { 64 | List> futures = Lists.newArrayList(); 65 | fileDeleter.run(); 66 | for (int i = 0; i < number; i++) { 67 | final File toDelete = files[i]; 68 | futures.add((ListenableFuture) exec.submit(new Runnable() { 69 | 70 | @Override 71 | public void run() { 72 | fileDeleter.deleteFile(toDelete); 73 | } 74 | })); 75 | } 76 | fileDeleter.run(); 77 | ListenableFuture> allAsList = Futures.allAsList(futures); 78 | fileDeleter.run(); 79 | Futures.get(allAsList, 2, TimeUnit.SECONDS, Exception.class); 80 | fileDeleter.run(); 81 | for (File file : files) { 82 | Assert.assertThat(file.exists(), is(Boolean.FALSE)); 83 | } 84 | } 85 | 86 | 87 | @Test 88 | public void deleteFileThatIsOpen() 89 | throws IOException, InterruptedException { 90 | File file = File.createTempFile("temp", "temp"); 91 | FileInputStream fileInputStream = new FileInputStream(file); 92 | fileInputStream.read(); 93 | fileDeleter.deleteFile(file); 94 | fileDeleter.run(); 95 | Assert.assertThat(file.exists(), is(Boolean.TRUE)); 96 | fileInputStream.close(); 97 | fileDeleter.run(); 98 | Assert.assertThat(file.exists(), is(Boolean.FALSE)); 99 | } 100 | 101 | 102 | @Test 103 | public void deleteFileThatIsOpenInADirectory() 104 | throws IOException, InterruptedException { 105 | File dir = new File(staticStateRootFolderProvider.getRootFolder(), "zetestdir"); 106 | dir.mkdirs(); 107 | File file = new File(dir, "file"); 108 | file.createNewFile(); 109 | Assert.assertThat(file.exists(), is(Boolean.TRUE)); 110 | Assert.assertThat(dir.exists(), is(Boolean.TRUE)); 111 | FileInputStream fileInputStream = new FileInputStream(file); 112 | fileInputStream.read(); 113 | fileDeleter.deleteFile(dir); 114 | fileDeleter.run(); 115 | Assert.assertThat(file.exists(), is(Boolean.TRUE)); 116 | Assert.assertThat(dir.exists(), is(Boolean.TRUE)); 117 | fileInputStream.close(); 118 | fileDeleter.run(); 119 | Assert.assertThat(file.exists(), is(Boolean.FALSE)); 120 | Assert.assertThat(dir.exists(), is(Boolean.FALSE)); 121 | } 122 | 123 | 124 | @Test 125 | public void deleteFileThatIsNotAFile() 126 | throws IOException { 127 | File fake = new File("lalala"); 128 | File file = File.createTempFile("temp", "temp"); 129 | Assert.assertThat(fake.exists(), is(Boolean.FALSE)); 130 | Assert.assertThat(file.exists(), is(Boolean.TRUE)); 131 | fileDeleter.deleteFile(fake); 132 | fileDeleter.deleteFile(file); 133 | fileDeleter.run(); 134 | Assert.assertThat(fake.exists(), is(Boolean.FALSE)); 135 | Assert.assertThat(file.exists(), is(Boolean.FALSE)); 136 | } 137 | 138 | } 139 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/staticstate/StaticStateIdentifierManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.util.UUID; 5 | 6 | import javax.servlet.http.Cookie; 7 | 8 | import org.junit.Assert; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.mock.web.MockHttpServletRequest; 14 | import org.springframework.mock.web.MockHttpServletResponse; 15 | import org.springframework.test.context.ContextConfiguration; 16 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 17 | 18 | import com.am.jlfu.fileuploader.web.utils.RequestComponentContainer; 19 | import com.am.jlfu.identifier.impl.DefaultIdentifierProvider; 20 | 21 | 22 | 23 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 24 | @RunWith(SpringJUnit4ClassRunner.class) 25 | public class StaticStateIdentifierManagerTest { 26 | 27 | @Autowired 28 | StaticStateIdentifierManager staticStateIdentifierManager; 29 | 30 | @Autowired 31 | RequestComponentContainer requestComponentContainer; 32 | 33 | MockHttpServletRequest mockHttpServletRequest = new MockHttpServletRequest(); 34 | MockHttpServletResponse mockHttpServletResponse = new MockHttpServletResponse(); 35 | 36 | 37 | 38 | @Before 39 | public void init() { 40 | 41 | // populate request component container 42 | requestComponentContainer.populate(mockHttpServletRequest, mockHttpServletResponse); 43 | 44 | // clean cookie 45 | mockHttpServletRequest.clearAttributes(); 46 | mockHttpServletRequest.setCookies(new Cookie[] {}); 47 | mockHttpServletResponse.reset(); 48 | } 49 | 50 | 51 | @Test 52 | public void testNoIdInCookieOrSession() { 53 | 54 | // assert session is empty 55 | Assert.assertNull(mockHttpServletRequest.getSession().getAttribute(DefaultIdentifierProvider.cookieIdentifier)); 56 | 57 | // assert cookie is empty 58 | Assert.assertNull(DefaultIdentifierProvider.getCookie(mockHttpServletRequest.getCookies(), DefaultIdentifierProvider.cookieIdentifier)); 59 | 60 | // get id 61 | UUID identifier = staticStateIdentifierManager.getIdentifier(); 62 | 63 | // copy cookies from response to request 64 | mockHttpServletRequest.setCookies(mockHttpServletResponse.getCookies()); 65 | 66 | Assert.assertNotNull(identifier); 67 | 68 | // assert cookie filled 69 | Assert.assertEquals(identifier, 70 | UUID.fromString(DefaultIdentifierProvider.getCookie(mockHttpServletRequest.getCookies(), DefaultIdentifierProvider.cookieIdentifier) 71 | .getValue())); 72 | 73 | // assert session filled 74 | Assert.assertEquals(identifier, mockHttpServletRequest.getSession().getAttribute(DefaultIdentifierProvider.cookieIdentifier)); 75 | 76 | // then clear identifier 77 | staticStateIdentifierManager.clearIdentifier(); 78 | 79 | // copy cookies from response to request 80 | mockHttpServletRequest.setCookies(mockHttpServletResponse.getCookies()); 81 | 82 | // assert session is empty 83 | Assert.assertNull(mockHttpServletRequest.getSession().getAttribute(DefaultIdentifierProvider.cookieIdentifier)); 84 | 85 | // assert cookie is either empty or maxage below 0 86 | Assert.assertNull(DefaultIdentifierProvider.getCookie(mockHttpServletRequest.getCookies(), DefaultIdentifierProvider.cookieIdentifier)); 87 | } 88 | 89 | 90 | @Test 91 | public void testNoIdInSession() { 92 | 93 | // assert session is empty 94 | Assert.assertNull(mockHttpServletRequest.getSession().getAttribute(DefaultIdentifierProvider.cookieIdentifier)); 95 | 96 | // assert cookie is empty 97 | Assert.assertNull(DefaultIdentifierProvider.getCookie(mockHttpServletRequest.getCookies(), DefaultIdentifierProvider.cookieIdentifier)); 98 | 99 | // set cookie 100 | UUID identifierOriginal = staticStateIdentifierManager.getIdentifier(); 101 | DefaultIdentifierProvider.setCookie(mockHttpServletResponse, identifierOriginal); 102 | 103 | // copy cookies from response to request 104 | mockHttpServletRequest.setCookies(mockHttpServletResponse.getCookies()); 105 | 106 | // assert cookie filled 107 | Assert.assertEquals(identifierOriginal, 108 | UUID.fromString(DefaultIdentifierProvider.getCookie(mockHttpServletRequest.getCookies(), DefaultIdentifierProvider.cookieIdentifier) 109 | .getValue())); 110 | 111 | // get id 112 | UUID identifier = staticStateIdentifierManager.getIdentifier(); 113 | Assert.assertEquals(identifierOriginal, identifier); 114 | 115 | // assert session is filled with id 116 | Assert.assertEquals(identifier, mockHttpServletRequest.getSession().getAttribute(DefaultIdentifierProvider.cookieIdentifier)); 117 | 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /src/test/java/com/am/jlfu/staticstate/StaticStateManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.am.jlfu.staticstate; 2 | 3 | 4 | import java.io.File; 5 | import java.io.IOException; 6 | import java.util.UUID; 7 | import java.util.concurrent.ExecutionException; 8 | import java.util.concurrent.TimeoutException; 9 | 10 | import junit.framework.Assert; 11 | 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | import org.junit.runner.RunWith; 15 | import org.springframework.beans.factory.annotation.Autowired; 16 | import org.springframework.mock.web.MockHttpServletRequest; 17 | import org.springframework.mock.web.MockHttpServletResponse; 18 | import org.springframework.test.context.ContextConfiguration; 19 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 20 | 21 | import com.am.jlfu.fileuploader.json.FileStateJsonBase; 22 | import com.am.jlfu.fileuploader.web.utils.RequestComponentContainer; 23 | import com.am.jlfu.staticstate.entities.StaticFileState; 24 | import com.am.jlfu.staticstate.entities.StaticStatePersistedOnFileSystemEntity; 25 | 26 | 27 | 28 | @ContextConfiguration(locations = { "classpath:jlfu.test.xml" }) 29 | @RunWith(SpringJUnit4ClassRunner.class) 30 | public class StaticStateManagerTest { 31 | 32 | @Autowired 33 | FileDeleter fileDeleter; 34 | 35 | @Autowired 36 | StaticStateManager staticStateManager; 37 | 38 | @Autowired 39 | StaticStateDirectoryManager staticStatedDirectoryManager; 40 | 41 | @Autowired 42 | StaticStateIdentifierManager staticStateIdentifierManager; 43 | 44 | @Autowired 45 | RequestComponentContainer requestComponentContainer; 46 | 47 | 48 | 49 | @Before 50 | public void init() { 51 | 52 | // populate request component container 53 | requestComponentContainer.populate(new MockHttpServletRequest(), new MockHttpServletResponse()); 54 | 55 | staticStateManager.init(StaticStatePersistedOnFileSystemEntity.class); 56 | } 57 | 58 | 59 | @Test 60 | public void testClear() 61 | throws InterruptedException, ExecutionException, TimeoutException { 62 | 63 | // get entity 64 | staticStateManager.getEntity(); 65 | 66 | // assert directory is there 67 | File uuidFileParent = staticStatedDirectoryManager.getUUIDFileParent(); 68 | Assert.assertTrue(uuidFileParent.exists()); 69 | 70 | // clear 71 | staticStateManager.clear(); 72 | 73 | // force file deleter to delete stuff 74 | fileDeleter.run(); 75 | 76 | // assert directory is deleted 77 | Assert.assertFalse(uuidFileParent.exists()); 78 | } 79 | 80 | 81 | @Test 82 | public void testClearFile() 83 | throws IOException, InterruptedException, ExecutionException, TimeoutException { 84 | String randomValue = "a"; 85 | UUID fileId = UUID.randomUUID(); 86 | 87 | // get entity 88 | StaticStatePersistedOnFileSystemEntity entity = staticStateManager.getEntity(); 89 | StaticFileState value = new StaticFileState(); 90 | FileStateJsonBase staticFileStateJson = new FileStateJsonBase(); 91 | value.setStaticFileStateJson(staticFileStateJson); 92 | entity.getFileStates().put(fileId, value); 93 | 94 | // populate it 95 | value.setAbsoluteFullPathOfUploadedFile(randomValue); 96 | staticFileStateJson.setOriginalFileName(randomValue); 97 | staticFileStateJson.setOriginalFileSizeInBytes(123000l); 98 | 99 | // create a file 100 | File file = new File(staticStatedDirectoryManager.getUUIDFileParent(), fileId.toString()); 101 | file.createNewFile(); 102 | Assert.assertTrue(file.exists()); 103 | 104 | // clear it 105 | staticStateManager.clearFile(fileId); 106 | 107 | // reget it 108 | StaticFileState staticFileState = staticStateManager.getEntity().getFileStates().get(fileId); 109 | Assert.assertNull(staticFileState); 110 | 111 | // force file deleter to run 112 | fileDeleter.run(); 113 | 114 | // assert file deleted 115 | Assert.assertFalse(file.exists()); 116 | } 117 | 118 | 119 | @Test 120 | public void testGetEntityFromFile() { 121 | String absoluteFullPathOfUploadedFile = "value"; 122 | UUID fileId = UUID.randomUUID(); 123 | 124 | // get entity 125 | StaticStatePersistedOnFileSystemEntity entity = staticStateManager.getEntity(); 126 | StaticFileState value = new StaticFileState(); 127 | entity.getFileStates().put(fileId, value); 128 | 129 | // put some stuff in the file 130 | value.setAbsoluteFullPathOfUploadedFile(absoluteFullPathOfUploadedFile); 131 | staticStateManager.updateEntity(entity); 132 | 133 | // remove from cache 134 | staticStateManager.cache.invalidate(staticStateIdentifierManager.getIdentifier()); 135 | 136 | // get again (it will load from file into cache) 137 | staticStateManager.getEntity(); 138 | 139 | // check everything is good 140 | Assert.assertEquals(absoluteFullPathOfUploadedFile, value.getAbsoluteFullPathOfUploadedFile()); 141 | 142 | 143 | } 144 | 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/test/resources/jlfu.test.properties: -------------------------------------------------------------------------------- 1 | jlfu.filecleaner.cron=0/30 * * * * ? 2 | jlfu.filecleaner.maximumInactivityInHoursBeforeDelete=48 3 | -------------------------------------------------------------------------------- /src/test/resources/jlfu.test.xml: -------------------------------------------------------------------------------- 1 | 2 | 32 | 33 | 34 | 35 | 36 | 37 | 39 | 40 | 41 | 42 | classpath:jlfu.test.properties 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Loggers. 4 | 5 | log4j.rootLogger = debug, console 6 | log4j.logger.com.am = debug 7 | log4j.logger.org.springframework = error 8 | 9 | 10 | # Appenders. 11 | 12 | log4j.appender.console = org.apache.log4j.ConsoleAppender 13 | log4j.appender.console.layout = org.apache.log4j.PatternLayout 14 | log4j.appender.console.layout.ConversionPattern = %p %d{ISO8601} %C{1} %t %m %n 15 | --------------------------------------------------------------------------------