├── .classpath ├── .gitignore ├── .project ├── .settings └── org.eclipse.jdt.core.prefs ├── README.md ├── bin ├── libwrapper-linux-x86-32.so ├── libwrapper-linux-x86-64.so ├── libwrapper-macosx-arm-64.dylib ├── libwrapper-macosx-universal-32.jnilib ├── libwrapper-macosx-universal-64.jnilib ├── psqueue.bat ├── psqueue.sh ├── wrapper-linux-x86-32 ├── wrapper-linux-x86-64 ├── wrapper-macosx-arm-64 ├── wrapper-macosx-universal-32 ├── wrapper-macosx-universal-64 ├── wrapper-windows-x86-32.dll ├── wrapper-windows-x86-32.exe ├── wrapper-windows-x86-64.dll ├── wrapper-windows-x86-64.exe ├── wrapper.conf └── wrapper.jar ├── conf ├── conf.xml └── log4j.xml ├── help-doc └── Test-URL.txt ├── lib ├── EasyFastJson-2.7.2.jar ├── istack-commons-runtime.jar ├── javax.activation-api.jar ├── jaxb-api-2.3.1.jar ├── jaxb-runtime.jar ├── log4j-1.2.17.jar ├── netty-all-4.1.7.Final.jar ├── slf4j-api.jar └── slf4j-log4j.jar └── src ├── com └── leansoft │ └── bigqueue │ ├── BigArrayImpl.java │ ├── BigQueueImpl.java │ ├── FanOutQueueImpl.java │ ├── FanOutQueueImplEx.java │ ├── IBigArray.java │ ├── IBigQueue.java │ ├── IFanOutQueue.java │ ├── cache │ ├── ILRUCache.java │ └── LRUCacheImpl.java │ ├── page │ ├── IMappedPage.java │ ├── IMappedPageFactory.java │ ├── MappedPageFactoryImpl.java │ └── MappedPageImpl.java │ └── utils │ ├── Calculator.java │ ├── FileUtil.java │ └── FolderNameValidator.java └── wjw └── psqueue ├── msg ├── ResAdd.java ├── ResData.java ├── ResList.java ├── ResQueueStatus.java ├── ResSubStatus.java └── ResultCode.java └── server ├── App.java ├── Conf.java ├── HttpRequestHandler.java ├── HttpServerChannelInitializer.java ├── Wrapper.java └── jmx ├── AppMXBean.java ├── JMXTools.java └── annotation ├── Description.java ├── MBean.java └── ManagedOperation.java /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /classes/ 2 | /db 3 | /log 4 | 5 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | PSQueueServer 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.jdt.core.javanature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate 4 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 5 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 6 | org.eclipse.jdt.core.compiler.compliance=1.8 7 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 8 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 9 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 10 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 11 | org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled 12 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 13 | org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning 14 | org.eclipse.jdt.core.compiler.release=disabled 15 | org.eclipse.jdt.core.compiler.source=1.8 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 支持任意个队列,每个队列任意多个消费者的基于 HTTP GET/POST 协议的轻量级开源简单消息队列服务!协议用纯java实现.可以达到每秒并发40000-50000个请求. 2 | 3 | 队列（Queue）又称先进先出表（First In First Out），即先进入队列的元素，先从队列中取出。加入元素的一头叫“队头”，取出元素的一头叫“队尾”。利用消息队列可以很好地异步处理数据传送和存储，当你频繁地向数据库中插入数据、频繁地向搜索引擎提交数据，就可采取消息队列来异步插入。另外，还可以将较慢的处理逻辑、有并发数量限制的处理逻辑，通过消息队列放在后台处理，例如FLV视频转换、发送手机短信、发送电子邮件等。 4 | 5 | #### PSQueueServer 具有以下特征： 6 | 7 | + [x] 非常简单，基于 HTTP GET/POST 协议。PHP、Java、Perl、Shell、Python、Ruby等支持HTTP协议的编程语言均可调用。 8 | + [x] 完善的JMX管理接口,所有方法全部可以由JMX来管理.为了安全管理方法需要口令! 9 | + [x] 每个队列支持任意多消费者。 10 | + [x] 非常快速，入队列、出队列速度超过40000次/秒。 11 | + [x] 高并发，支持5K以上的并发连接。 12 | + [x] 支持多队列。 13 | + [x] 队列个数无限制,只要系统的磁盘空间够用(缺省单个队列占用磁盘空间是2G)。 14 | + [x] 低内存消耗，海量数据存储，存储几十GB的数据只需不到200MB的物理内存缓冲区。 15 | + [x] 可以实时查看指定队列状态（未读队列数量）。 16 | + [x] 可以查看指定队列,指定消费者的内容，包括未出、已出的队列内容。 17 | + [x] 查看队列内容时，支持多字符集编码。 18 | + [x] 创新设计:消费者可以故意倒回到老的偏移量再次消费数据。这违反了队列的常见约定，但被证明是许多消费者的基本特征。 19 | 20 | ### 注意: 21 | > 22 | 当向队列`add`添加数据而且当队列的`size`超出队列的`capacity`时,是不会报队列满的错误的! 23 | 系统会定时清理(缺省30分钟),把队列的size调整到跟`capacity`一致,这时超出`capacity`的队列尾部的数据会被清理掉!!! -------------------------------------------------------------------------------- /bin/libwrapper-linux-x86-32.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/libwrapper-linux-x86-32.so -------------------------------------------------------------------------------- /bin/libwrapper-linux-x86-64.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/libwrapper-linux-x86-64.so -------------------------------------------------------------------------------- /bin/libwrapper-macosx-arm-64.dylib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/libwrapper-macosx-arm-64.dylib -------------------------------------------------------------------------------- /bin/libwrapper-macosx-universal-32.jnilib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/libwrapper-macosx-universal-32.jnilib -------------------------------------------------------------------------------- /bin/libwrapper-macosx-universal-64.jnilib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/libwrapper-macosx-universal-64.jnilib -------------------------------------------------------------------------------- /bin/psqueue.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal 3 | 4 | rem Copyright (c) 1999, 2009 Tanuki Software, Ltd. 5 | rem http://www.tanukisoftware.com 6 | rem All rights reserved. 7 | rem 8 | rem This software is the proprietary information of Tanuki Software. 9 | rem You shall use it only in accordance with the terms of the 10 | rem license agreement you entered into with Tanuki Software. 11 | rem http://wrapper.tanukisoftware.org/doc/english/licenseOverview.html 12 | rem 13 | rem Java Service Wrapper command based script. 14 | rem Optimized for use with version 3.3.5 of the Wrapper. 15 | rem 16 | 17 | if "%OS%"=="Windows_NT" goto nt 18 | echo This script only works with NT-based versions of Windows. 19 | goto :eof 20 | 21 | :nt 22 | rem 23 | rem Find the application home. 24 | rem 25 | rem %~dp0 is location of current script under NT 26 | set _REALPATH=%~dp0 27 | 28 | rem Decide on the wrapper binary. 29 | set _WRAPPER_BASE=wrapper 30 | set _WRAPPER_EXE=%_REALPATH%%_WRAPPER_BASE%-windows-x86-64.exe 31 | if exist "%_WRAPPER_EXE%" goto validate 32 | set _WRAPPER_EXE=%_REALPATH%%_WRAPPER_BASE%-windows-x86-32.exe 33 | if exist "%_WRAPPER_EXE%" goto validate 34 | set _WRAPPER_EXE=%_REALPATH%%_WRAPPER_BASE%.exe 35 | if exist "%_WRAPPER_EXE%" goto validate 36 | echo Unable to locate a Wrapper executable using any of the following names: 37 | echo %_REALPATH%%_WRAPPER_BASE%-windows-x86-32.exe 38 | echo %_REALPATH%%_WRAPPER_BASE%-windows-x86-64.exe 39 | echo %_REALPATH%%_WRAPPER_BASE%.exe 40 | pause 41 | goto :eof 42 | 43 | :validate 44 | rem Find the requested command. 45 | for /F %%v in ('echo %1^|findstr "^console$ ^start$ ^pause$ ^resume$ ^stop$ ^restart$ ^install$ ^remove"') do call :exec set COMMAND=%%v 46 | 47 | if "%COMMAND%" == "" ( 48 | echo Usage: %0 { console : start : pause : resume : stop : restart : install : remove } 49 | pause 50 | goto :eof 51 | ) else ( 52 | shift 53 | ) 54 | 55 | rem 56 | rem Find the wrapper.conf 57 | rem 58 | :conf 59 | set _WRAPPER_CONF="%_REALPATH%\wrapper.conf" 60 | 61 | rem 62 | rem Run the application. 63 | rem At runtime, the current directory will be that of wrapper.exe 64 | rem 65 | call :%COMMAND% 66 | if errorlevel 1 pause 67 | goto :eof 68 | 69 | :console 70 | "%_WRAPPER_EXE%" -c %_WRAPPER_CONF% 71 | goto :eof 72 | 73 | :start 74 | "%_WRAPPER_EXE%" -t %_WRAPPER_CONF% 75 | goto :eof 76 | 77 | :pause 78 | "%_WRAPPER_EXE%" -a %_WRAPPER_CONF% 79 | goto :eof 80 | 81 | :resume 82 | "%_WRAPPER_EXE%" -e %_WRAPPER_CONF% 83 | goto :eof 84 | 85 | :stop 86 | "%_WRAPPER_EXE%" -p %_WRAPPER_CONF% 87 | goto :eof 88 | 89 | :install 90 | "%_WRAPPER_EXE%" -i %_WRAPPER_CONF% 91 | goto :eof 92 | 93 | :remove 94 | "%_WRAPPER_EXE%" -r %_WRAPPER_CONF% 95 | goto :eof 96 | 97 | :restart 98 | call :stop 99 | call :start 100 | goto :eof 101 | 102 | :exec 103 | %* 104 | goto :eof 105 | 106 | -------------------------------------------------------------------------------- /bin/psqueue.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # 4 | # Copyright (c) 1999, 2011 Tanuki Software, Ltd. 5 | # http://www.tanukisoftware.com 6 | # All rights reserved. 7 | # 8 | # This software is the proprietary information of Tanuki Software. 9 | # You shall use it only in accordance with the terms of the 10 | # license agreement you entered into with Tanuki Software. 11 | # http://wrapper.tanukisoftware.com/doc/english/licenseOverview.html 12 | # 13 | # Java Service Wrapper sh script. Suitable for starting and stopping 14 | # wrapped Java applications on UNIX platforms. 15 | # 16 | 17 | #----------------------------------------------------------------------------- 18 | # These settings can be modified to fit the needs of your application 19 | # Optimized for use with version 3.5.9 of the Wrapper. 20 | 21 | if [ "x$JAVA_HOME" = "x" ] 22 | then 23 | export JAVA_HOME=/usr/java/default 24 | fi 25 | export PATH=$JAVA_HOME/bin:$PATH 26 | 27 | # Application 28 | APP_NAME="psqueue" 29 | APP_LONG_NAME="psqueue" 30 | 31 | # Wrapper 32 | WRAPPER_CMD="./wrapper" 33 | WRAPPER_CONF="./wrapper.conf" 34 | 35 | # Priority at which to run the wrapper. See "man nice" for valid priorities. 36 | # nice is only used if a priority is specified. 37 | PRIORITY= 38 | 39 | # Location of the pid file. 40 | PIDDIR="." 41 | 42 | # FIXED_COMMAND tells the script to use a hard coded action rather than 43 | # expecting the first parameter of the command line to be the command. 44 | # By default the command will will be expected to be the first parameter. 45 | #FIXED_COMMAND=console 46 | 47 | # PASS_THROUGH tells the script to pass all arguments through to the JVM 48 | # as is. If FIXED_COMMAND is specified then all arguments will be passed. 49 | # If not set then all arguments starting with the second will be passed. 50 | #PASS_THROUGH=true 51 | 52 | # If uncommented, causes the Wrapper to be shutdown using an anchor file. 53 | # When launched with the 'start' command, it will also ignore all INT and 54 | # TERM signals. 55 | #IGNORE_SIGNALS=true 56 | 57 | # Wrapper will start the JVM asynchronously. Your application may have some 58 | # initialization tasks and it may be desirable to wait a few seconds 59 | # before returning. For example, to delay the invocation of following 60 | # startup scripts. Setting WAIT_AFTER_STARTUP to a positive number will 61 | # cause the start command to delay for the indicated period of time 62 | # (in seconds). 63 | # 64 | WAIT_AFTER_STARTUP=0 65 | 66 | # If set, wait for the wrapper to report that the daemon has started 67 | WAIT_FOR_STARTED_STATUS=true 68 | WAIT_FOR_STARTED_TIMEOUT=120 69 | 70 | # If set, the status, start_msg and stop_msg commands will print out detailed 71 | # state information on the Wrapper and Java processes. 72 | #DETAIL_STATUS=true 73 | 74 | # If set, the 'pause' and 'resume' commands will be enabled. These make it 75 | # possible to pause the JVM or Java application without completely stopping 76 | # the Wrapper. See the wrapper.pausable and wrapper.pausable.stop_jvm 77 | # properties for more information. 78 | #PAUSABLE=true 79 | 80 | # If specified, the Wrapper will be run as the specified user. 81 | # IMPORTANT - Make sure that the user has the required privileges to write 82 | # the PID file and wrapper.log files. Failure to be able to write the log 83 | # file will cause the Wrapper to exit without any way to write out an error 84 | # message. 85 | # NOTE - This will set the user which is used to run the Wrapper as well as 86 | # the JVM and is not useful in situations where a privileged resource or 87 | # port needs to be allocated prior to the user being changed. 88 | #RUN_AS_USER= 89 | 90 | # By default we show a detailed usage block. Uncomment to show brief usage. 91 | #BRIEF_USAGE=true 92 | 93 | # flag for using upstart when installing (rather than init.d rc.d) 94 | USE_UPSTART= 95 | 96 | # When installing on On Mac OSX platforms, the following domain will be used to 97 | # prefix the plist file name. 98 | PLIST_DOMAIN=org.tanukisoftware.wrapper 99 | 100 | # The following two lines are used by the chkconfig command. Change as is 101 | # appropriate for your application. They should remain commented. 102 | # chkconfig: 2345 20 80 103 | # description: Test Wrapper Sample Application 104 | 105 | # Initialization block for the install_initd and remove_initd scripts used by 106 | # SUSE linux distributions. 107 | ### BEGIN INIT INFO 108 | # Provides: testwrapper 109 | # Required-Start: $local_fs $network $syslog 110 | # Should-Start: 111 | # Required-Stop: 112 | # Default-Start: 2 3 4 5 113 | # Default-Stop: 0 1 6 114 | # Short-Description: Test Wrapper Sample Application 115 | # Description: Test Wrapper Sample Application Description 116 | ### END INIT INFO 117 | 118 | # Do not modify anything beyond this point 119 | #----------------------------------------------------------------------------- 120 | 121 | if [ -n "$FIXED_COMMAND" ] 122 | then 123 | COMMAND="$FIXED_COMMAND" 124 | else 125 | COMMAND="$1" 126 | fi 127 | 128 | 129 | # Required for HP-UX Startup 130 | if [ `uname -s` = "HP-UX" -o `uname -s` = "HP-UX64" ] ; then 131 | PATH=$PATH:/usr/bin 132 | fi 133 | 134 | # Get the fully qualified path to the script 135 | case $0 in 136 | /*) 137 | SCRIPT="$0" 138 | ;; 139 | *) 140 | PWD=`pwd` 141 | SCRIPT="$PWD/$0" 142 | ;; 143 | esac 144 | 145 | # Resolve the true real path without any sym links. 146 | CHANGED=true 147 | while [ "X$CHANGED" != "X" ] 148 | do 149 | # Change spaces to ":" so the tokens can be parsed. 150 | SAFESCRIPT=`echo $SCRIPT | sed -e 's; ;:;g'` 151 | # Get the real path to this script, resolving any symbolic links 152 | TOKENS=`echo $SAFESCRIPT | sed -e 's;/; ;g'` 153 | REALPATH= 154 | for C in $TOKENS; do 155 | # Change any ":" in the token back to a space. 156 | C=`echo $C | sed -e 's;:; ;g'` 157 | REALPATH="$REALPATH/$C" 158 | # If REALPATH is a sym link, resolve it. Loop for nested links. 159 | while [ -h "$REALPATH" ] ; do 160 | LS="`ls -ld "$REALPATH"`" 161 | LINK="`expr "$LS" : '.*-> \(.*\)$'`" 162 | if expr "$LINK" : '/.*' > /dev/null; then 163 | # LINK is absolute. 164 | REALPATH="$LINK" 165 | else 166 | # LINK is relative. 167 | REALPATH="`dirname "$REALPATH"`""/$LINK" 168 | fi 169 | done 170 | done 171 | 172 | if [ "$REALPATH" = "$SCRIPT" ] 173 | then 174 | CHANGED="" 175 | else 176 | SCRIPT="$REALPATH" 177 | fi 178 | done 179 | 180 | # Get the location of the script. 181 | REALDIR=`dirname "$REALPATH"` 182 | # Normalize the path 183 | REALDIR=`cd ${REALDIR}; pwd` 184 | 185 | # If the PIDDIR is relative, set its value relative to the full REALPATH to avoid problems if 186 | # the working directory is later changed. 187 | FIRST_CHAR=`echo $PIDDIR | cut -c1,1` 188 | if [ "$FIRST_CHAR" != "/" ] 189 | then 190 | PIDDIR=$REALDIR/$PIDDIR 191 | fi 192 | # Same test for WRAPPER_CMD 193 | FIRST_CHAR=`echo $WRAPPER_CMD | cut -c1,1` 194 | if [ "$FIRST_CHAR" != "/" ] 195 | then 196 | WRAPPER_CMD=$REALDIR/$WRAPPER_CMD 197 | fi 198 | # Same test for WRAPPER_CONF 199 | FIRST_CHAR=`echo $WRAPPER_CONF | cut -c1,1` 200 | if [ "$FIRST_CHAR" != "/" ] 201 | then 202 | WRAPPER_CONF=$REALDIR/$WRAPPER_CONF 203 | fi 204 | 205 | # Process ID 206 | ANCHORFILE="$PIDDIR/$APP_NAME.anchor" 207 | COMMANDFILE="$PIDDIR/$APP_NAME.command" 208 | STATUSFILE="$PIDDIR/$APP_NAME.status" 209 | JAVASTATUSFILE="$PIDDIR/$APP_NAME.java.status" 210 | PIDFILE="$PIDDIR/$APP_NAME.pid" 211 | LOCKDIR="/var/lock/subsys" 212 | LOCKFILE="$LOCKDIR/$APP_NAME" 213 | pid="" 214 | 215 | # Resolve the location of the 'ps' command 216 | PSEXE="/usr/ucb/ps" 217 | if [ ! -x "$PSEXE" ] 218 | then 219 | PSEXE="/usr/bin/ps" 220 | if [ ! -x "$PSEXE" ] 221 | then 222 | PSEXE="/bin/ps" 223 | if [ ! -x "$PSEXE" ] 224 | then 225 | eval echo `gettext 'Unable to locate "ps".'` 226 | eval echo `gettext 'Please report this message along with the location of the command on your system.'` 227 | exit 1 228 | fi 229 | fi 230 | fi 231 | 232 | # Resolve the os 233 | DIST_OS=`uname -s | tr "[A-Z]" "[a-z]" | tr -d ' '` 234 | case "$DIST_OS" in 235 | 'sunos') 236 | DIST_OS="solaris" 237 | ;; 238 | 'hp-ux' | 'hp-ux64') 239 | # HP-UX needs the XPG4 version of ps (for -o args) 240 | DIST_OS="hpux" 241 | UNIX95="" 242 | export UNIX95 243 | ;; 244 | 'darwin') 245 | DIST_OS="macosx" 246 | ;; 247 | 'unix_sv') 248 | DIST_OS="unixware" 249 | ;; 250 | 'os/390') 251 | DIST_OS="zos" 252 | ;; 253 | esac 254 | 255 | # Resolve the architecture 256 | if [ "$DIST_OS" = "macosx" ] 257 | then 258 | OS_VER=`sw_vers | grep 'ProductVersion:' | grep -o '[0-9]*\.[0-9]*\.[0-9]*'` 259 | DIST_ARCH="universal" 260 | if [[ "$OS_VER" < "10.5.0" ]] 261 | then 262 | DIST_BITS="32" 263 | else 264 | DIST_BITS="64" 265 | fi 266 | APP_PLIST_BASE=${PLIST_DOMAIN}.${APP_NAME} 267 | APP_PLIST=${APP_PLIST_BASE}.plist 268 | else 269 | DIST_ARCH= 270 | DIST_ARCH=`uname -p 2>/dev/null | tr "[A-Z]" "[a-z]" | tr -d ' '` 271 | if [ "X$DIST_ARCH" = "X" ] 272 | then 273 | DIST_ARCH="unknown" 274 | fi 275 | if [ "$DIST_ARCH" = "unknown" ] 276 | then 277 | DIST_ARCH=`uname -m 2>/dev/null | tr "[A-Z]" "[a-z]" | tr -d ' '` 278 | fi 279 | case "$DIST_ARCH" in 280 | 'athlon' | 'i386' | 'i486' | 'i586' | 'i686') 281 | DIST_ARCH="x86" 282 | if [ "${DIST_OS}" = "solaris" ] ; then 283 | DIST_BITS=`isainfo -b` 284 | else 285 | DIST_BITS="32" 286 | fi 287 | ;; 288 | 'amd64' | 'x86_64') 289 | DIST_ARCH="x86" 290 | DIST_BITS="64" 291 | ;; 292 | 'ia32') 293 | DIST_ARCH="ia" 294 | DIST_BITS="32" 295 | ;; 296 | 'ia64' | 'ia64n' | 'ia64w') 297 | DIST_ARCH="ia" 298 | DIST_BITS="64" 299 | ;; 300 | 'ip27') 301 | DIST_ARCH="mips" 302 | DIST_BITS="32" 303 | ;; 304 | 'power' | 'powerpc' | 'power_pc' | 'ppc64') 305 | if [ "${DIST_ARCH}" = "ppc64" ] ; then 306 | DIST_BITS="64" 307 | else 308 | DIST_BITS="32" 309 | fi 310 | DIST_ARCH="ppc" 311 | if [ "${DIST_OS}" = "aix" ] ; then 312 | if [ `getconf KERNEL_BITMODE` -eq 64 ]; then 313 | DIST_BITS="64" 314 | else 315 | DIST_BITS="32" 316 | fi 317 | fi 318 | ;; 319 | 'pa_risc' | 'pa-risc') 320 | DIST_ARCH="parisc" 321 | if [ `getconf KERNEL_BITS` -eq 64 ]; then 322 | DIST_BITS="64" 323 | else 324 | DIST_BITS="32" 325 | fi 326 | ;; 327 | 'sun4u' | 'sparcv9' | 'sparc') 328 | DIST_ARCH="sparc" 329 | DIST_BITS=`isainfo -b` 330 | ;; 331 | '9000/800' | '9000/785') 332 | DIST_ARCH="parisc" 333 | if [ `getconf KERNEL_BITS` -eq 64 ]; then 334 | DIST_BITS="64" 335 | else 336 | DIST_BITS="32" 337 | fi 338 | ;; 339 | '2064' | '2066' | '2084' | '2086' | '2094' | '2096' | '2097' | '2098' | '2817') 340 | DIST_ARCH="390" 341 | DIST_BITS="64" 342 | ;; 343 | esac 344 | fi 345 | 346 | # OSX always places Java in the same location so we can reliably set JAVA_HOME 347 | if [ "$DIST_OS" = "macosx" ] 348 | then 349 | if [ -z "$JAVA_HOME" ]; then 350 | JAVA_HOME="/Library/Java/Home"; export JAVA_HOME 351 | fi 352 | fi 353 | 354 | # Test Echo 355 | ECHOTEST=`echo -n "x"` 356 | if [ "$ECHOTEST" = "x" ] 357 | then 358 | ECHOOPT="-n " 359 | else 360 | ECHOOPT="" 361 | fi 362 | 363 | 364 | gettext() { 365 | "$WRAPPER_CMD" --translate "$1" "$WRAPPER_CONF" 2>/dev/null 366 | if [ $? != 0 ] ; then 367 | echo "$1" 368 | fi 369 | } 370 | 371 | outputFile() { 372 | if [ -f "$1" ] 373 | then 374 | eval echo `gettext ' $1 Found but not executable.'`; 375 | else 376 | echo " $1" 377 | fi 378 | } 379 | 380 | # Decide on the wrapper binary to use. 381 | # If the bits of the OS could be detected, we will try to look for the 382 | # binary with the correct bits value. If it doesn't exist, fall back 383 | # and look for the 32-bit binary. If that doesn't exist either then 384 | # look for the default. 385 | WRAPPER_TEST_CMD="" 386 | if [ -f "$WRAPPER_CMD-$DIST_OS-$DIST_ARCH-$DIST_BITS" ] 387 | then 388 | WRAPPER_TEST_CMD="$WRAPPER_CMD-$DIST_OS-$DIST_ARCH-$DIST_BITS" 389 | if [ ! -x "$WRAPPER_TEST_CMD" ] 390 | then 391 | chmod +x "$WRAPPER_TEST_CMD" 2>/dev/null 392 | fi 393 | if [ -x "$WRAPPER_TEST_CMD" ] 394 | then 395 | WRAPPER_CMD="$WRAPPER_TEST_CMD" 396 | else 397 | outputFile "$WRAPPER_TEST_CMD" 398 | WRAPPER_TEST_CMD="" 399 | fi 400 | fi 401 | if [ -f "$WRAPPER_CMD-$DIST_OS-$DIST_ARCH-32" -a -z "$WRAPPER_TEST_CMD" ] 402 | then 403 | WRAPPER_TEST_CMD="$WRAPPER_CMD-$DIST_OS-$DIST_ARCH-32" 404 | if [ ! -x "$WRAPPER_TEST_CMD" ] 405 | then 406 | chmod +x "$WRAPPER_TEST_CMD" 2>/dev/null 407 | fi 408 | if [ -x "$WRAPPER_TEST_CMD" ] 409 | then 410 | WRAPPER_CMD="$WRAPPER_TEST_CMD" 411 | else 412 | outputFile "$WRAPPER_TEST_CMD" 413 | WRAPPER_TEST_CMD="" 414 | fi 415 | fi 416 | if [ -f "$WRAPPER_CMD" -a -z "$WRAPPER_TEST_CMD" ] 417 | then 418 | WRAPPER_TEST_CMD="$WRAPPER_CMD" 419 | if [ ! -x "$WRAPPER_TEST_CMD" ] 420 | then 421 | chmod +x "$WRAPPER_TEST_CMD" 2>/dev/null 422 | fi 423 | if [ -x "$WRAPPER_TEST_CMD" ] 424 | then 425 | WRAPPER_CMD="$WRAPPER_TEST_CMD" 426 | else 427 | outputFile "$WRAPPER_TEST_CMD" 428 | WRAPPER_TEST_CMD="" 429 | fi 430 | fi 431 | if [ -z "$WRAPPER_TEST_CMD" ] 432 | then 433 | eval echo `gettext 'Unable to locate any of the following binaries:'` 434 | outputFile "$WRAPPER_CMD-$DIST_OS-$DIST_ARCH-$DIST_BITS" 435 | if [ ! "$DIST_BITS" = "32" ] 436 | then 437 | outputFile "$WRAPPER_CMD-$DIST_OS-$DIST_ARCH-32" 438 | fi 439 | outputFile "$WRAPPER_CMD" 440 | 441 | exit 1 442 | fi 443 | 444 | 445 | # Build the nice clause 446 | if [ "X$PRIORITY" = "X" ] 447 | then 448 | CMDNICE="" 449 | else 450 | CMDNICE="nice -$PRIORITY" 451 | fi 452 | 453 | # Build the anchor file clause. 454 | if [ "X$IGNORE_SIGNALS" = "X" ] 455 | then 456 | ANCHORPROP= 457 | IGNOREPROP= 458 | else 459 | ANCHORPROP=wrapper.anchorfile=\"$ANCHORFILE\" 460 | IGNOREPROP=wrapper.ignore_signals=TRUE 461 | fi 462 | 463 | # Build the status file clause. 464 | if [ "X$DETAIL_STATUS$WAIT_FOR_STARTED_STATUS" = "X" ] 465 | then 466 | STATUSPROP= 467 | else 468 | STATUSPROP="wrapper.statusfile=\"$STATUSFILE\" wrapper.java.statusfile=\"$JAVASTATUSFILE\"" 469 | fi 470 | 471 | # Build the command file clause. 472 | if [ -n "$PAUSABLE" ] 473 | then 474 | COMMANDPROP="wrapper.commandfile=\"$COMMANDFILE\" wrapper.pausable=TRUE" 475 | else 476 | COMMANDPROP= 477 | fi 478 | 479 | if [ ! -n "$WAIT_FOR_STARTED_STATUS" ] 480 | then 481 | WAIT_FOR_STARTED_STATUS=true 482 | fi 483 | 484 | if [ $WAIT_FOR_STARTED_STATUS = true ] ; then 485 | DETAIL_STATUS=true 486 | fi 487 | 488 | 489 | # Build the lock file clause. Only create a lock file if the lock directory exists on this platform. 490 | LOCKPROP= 491 | if [ -d $LOCKDIR ] 492 | then 493 | if [ -w $LOCKDIR ] 494 | then 495 | LOCKPROP=wrapper.lockfile=\"$LOCKFILE\" 496 | fi 497 | fi 498 | 499 | prepAdditionalParams() { 500 | ADDITIONAL_PARA="" 501 | if [ -n "$PASS_THROUGH" ] ; then 502 | ADDITIONAL_PARA="--" 503 | fi 504 | while [ -n "$1" ] ; do 505 | ADDITIONAL_PARA="$ADDITIONAL_PARA \"$1\"" 506 | shift 507 | done 508 | } 509 | 510 | checkUser() { 511 | # $1 touchLock flag 512 | # $2.. [command] args 513 | 514 | # Check the configured user. If necessary rerun this script as the desired user. 515 | if [ "X$RUN_AS_USER" != "X" ] 516 | then 517 | # Resolve the location of the 'id' command 518 | IDEXE="/usr/xpg4/bin/id" 519 | if [ ! -x "$IDEXE" ] 520 | then 521 | IDEXE="/usr/bin/id" 522 | if [ ! -x "$IDEXE" ] 523 | then 524 | eval echo `gettext 'Unable to locate "id".'` 525 | eval echo `gettext 'Please report this message along with the location of the command on your system.'` 526 | exit 1 527 | fi 528 | fi 529 | 530 | if [ "`$IDEXE -u -n`" = "$RUN_AS_USER" ] 531 | then 532 | # Already running as the configured user. Avoid password prompts by not calling su. 533 | RUN_AS_USER="" 534 | fi 535 | fi 536 | if [ "X$RUN_AS_USER" != "X" ] 537 | then 538 | # If LOCKPROP and $RUN_AS_USER are defined then the new user will most likely not be 539 | # able to create the lock file. The Wrapper will be able to update this file once it 540 | # is created but will not be able to delete it on shutdown. If $1 is set then 541 | # the lock file should be created for the current command 542 | if [ "X$LOCKPROP" != "X" ] 543 | then 544 | if [ "X$1" != "X" ] 545 | then 546 | # Resolve the primary group 547 | RUN_AS_GROUP=`groups $RUN_AS_USER | awk '{print $3}' | tail -1` 548 | if [ "X$RUN_AS_GROUP" = "X" ] 549 | then 550 | RUN_AS_GROUP=$RUN_AS_USER 551 | fi 552 | touch $LOCKFILE 553 | chown $RUN_AS_USER:$RUN_AS_GROUP $LOCKFILE 554 | fi 555 | fi 556 | 557 | # Still want to change users, recurse. This means that the user will only be 558 | # prompted for a password once. Variables shifted by 1 559 | shift 560 | 561 | # Wrap the parameters so they can be passed. 562 | ADDITIONAL_PARA="" 563 | while [ -n "$1" ] ; do 564 | ADDITIONAL_PARA="$ADDITIONAL_PARA \"$1\"" 565 | shift 566 | done 567 | 568 | # Use "runuser" if this exists. runuser should be used on RedHat in preference to su. 569 | # 570 | if test -f "/sbin/runuser" 571 | then 572 | /sbin/runuser - $RUN_AS_USER -c "\"$REALPATH\" $ADDITIONAL_PARA" 573 | else 574 | su - $RUN_AS_USER -c "\"$REALPATH\" $ADDITIONAL_PARA" 575 | fi 576 | RUN_AS_USER_EXITCODE=$? 577 | # Now that we are the original user again, we may need to clean up the lock file. 578 | if [ "X$LOCKPROP" != "X" ] 579 | then 580 | getpid 581 | if [ "X$pid" = "X" ] 582 | then 583 | # Wrapper is not running so make sure the lock file is deleted. 584 | if [ -f "$LOCKFILE" ] 585 | then 586 | rm "$LOCKFILE" 587 | fi 588 | fi 589 | fi 590 | 591 | exit $RUN_AS_USER_EXITCODE 592 | fi 593 | } 594 | 595 | getpid() { 596 | pid="" 597 | if [ -f "$PIDFILE" ] 598 | then 599 | if [ -r "$PIDFILE" ] 600 | then 601 | pid=`cat "$PIDFILE"` 602 | if [ "X$pid" != "X" ] 603 | then 604 | # It is possible that 'a' process with the pid exists but that it is not the 605 | # correct process. This can happen in a number of cases, but the most 606 | # common is during system startup after an unclean shutdown. 607 | # The ps statement below looks for the specific wrapper command running as 608 | # the pid. If it is not found then the pid file is considered to be stale. 609 | case "$DIST_OS" in 610 | 'freebsd') 611 | pidtest=`$PSEXE -p $pid -o args | tail -1` 612 | if [ "X$pidtest" = "XCOMMAND" ] 613 | then 614 | pidtest="" 615 | fi 616 | ;; 617 | 'macosx') 618 | pidtest=`$PSEXE -ww -p $pid -o command | grep -F "$WRAPPER_CMD" | tail -1` 619 | ;; 620 | 'solaris') 621 | if [ -f "/usr/bin/pargs" ] 622 | then 623 | pidtest=`pargs $pid | fgrep "$WRAPPER_CMD" | tail -1` 624 | else 625 | case "$PSEXE" in 626 | '/usr/ucb/ps') 627 | pidtest=`$PSEXE -auxww $pid | fgrep "$WRAPPER_CMD" | tail -1` 628 | ;; 629 | '/usr/bin/ps') 630 | TRUNCATED_CMD=`$PSEXE -o comm -p $pid | tail -1` 631 | COUNT=`echo $TRUNCATED_CMD | wc -m` 632 | COUNT=`echo ${COUNT}` 633 | COUNT=`expr $COUNT - 1` 634 | TRUNCATED_CMD=`echo $WRAPPER_CMD | cut -c1-$COUNT` 635 | pidtest=`$PSEXE -o comm -p $pid | fgrep "$TRUNCATED_CMD" | tail -1` 636 | ;; 637 | '/bin/ps') 638 | TRUNCATED_CMD=`$PSEXE -o comm -p $pid | tail -1` 639 | COUNT=`echo $TRUNCATED_CMD | wc -m` 640 | COUNT=`echo ${COUNT}` 641 | COUNT=`expr $COUNT - 1` 642 | TRUNCATED_CMD=`echo $WRAPPER_CMD | cut -c1-$COUNT` 643 | pidtest=`$PSEXE -o comm -p $pid | fgrep "$TRUNCATED_CMD" | tail -1` 644 | ;; 645 | *) 646 | echo "Unsupported ps command $PSEXE" 647 | exit 1 648 | ;; 649 | esac 650 | fi 651 | ;; 652 | 'hpux') 653 | pidtest=`$PSEXE -p $pid -x -o args | grep -F "$WRAPPER_CMD" | tail -1` 654 | ;; 655 | *) 656 | pidtest=`$PSEXE -p $pid -o args | grep -F "$WRAPPER_CMD" | tail -1` 657 | ;; 658 | esac 659 | 660 | if [ "X$pidtest" = "X" ] 661 | then 662 | # This is a stale pid file. 663 | rm -f "$PIDFILE" 664 | eval echo `gettext 'Removed stale pid file: $PIDFILE'` 665 | pid="" 666 | fi 667 | fi 668 | else 669 | eval echo `gettext 'Cannot read $PIDFILE.'` 670 | exit 1 671 | fi 672 | fi 673 | } 674 | 675 | getstatus() { 676 | STATUS= 677 | if [ -f "$STATUSFILE" ] 678 | then 679 | if [ -r "$STATUSFILE" ] 680 | then 681 | STATUS=`cat "$STATUSFILE"` 682 | fi 683 | fi 684 | if [ "X$STATUS" = "X" ] 685 | then 686 | STATUS="Unknown" 687 | fi 688 | 689 | JAVASTATUS= 690 | if [ -f "$JAVASTATUSFILE" ] 691 | then 692 | if [ -r "$JAVASTATUSFILE" ] 693 | then 694 | JAVASTATUS=`cat "$JAVASTATUSFILE"` 695 | fi 696 | fi 697 | if [ "X$JAVASTATUS" = "X" ] 698 | then 699 | JAVASTATUS="Unknown" 700 | fi 701 | } 702 | 703 | testpid() { 704 | case "$DIST_OS" in 705 | 'solaris') 706 | case "$PSEXE" in 707 | '/usr/ucb/ps') 708 | pid=`$PSEXE $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` 709 | ;; 710 | '/usr/bin/ps') 711 | pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` 712 | ;; 713 | '/bin/ps') 714 | pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` 715 | ;; 716 | *) 717 | echo "Unsupported ps command $PSEXE" 718 | exit 1 719 | ;; 720 | esac 721 | ;; 722 | *) 723 | pid=`$PSEXE -p $pid | grep $pid | grep -v grep | awk '{print $1}' | tail -1` 2>/dev/null 724 | ;; 725 | esac 726 | if [ "X$pid" = "X" ] 727 | then 728 | # Process is gone so remove the pid file. 729 | rm -f "$PIDFILE" 730 | pid="" 731 | fi 732 | } 733 | 734 | launchdtrap() { 735 | stopit 736 | exit 737 | } 738 | 739 | waitforwrapperstop() { 740 | getpid 741 | while [ "X$pid" != "X" ] ; do 742 | sleep 1 743 | getpid 744 | done 745 | } 746 | 747 | launchdinternal() { 748 | getpid 749 | trap launchdtrap TERM 750 | if [ "X$pid" = "X" ] 751 | then 752 | prepAdditionalParams "$@" 753 | 754 | # The string passed to eval must handles spaces in paths correctly. 755 | COMMAND_LINE="$CMDNICE \"$WRAPPER_CMD\" \"$WRAPPER_CONF\" wrapper.syslog.ident=\"$APP_NAME\" wrapper.pidfile=\"$PIDFILE\" wrapper.name=\"$APP_NAME\" wrapper.displayname=\"$APP_LONG_NAME\" wrapper.daemonize=TRUE $ANCHORPROP $IGNOREPROP $STATUSPROP $COMMANDPROP $LOCKPROP $ADDITIONAL_PARA" 756 | eval $COMMAND_LINE 757 | else 758 | eval echo `gettext '$APP_LONG_NAME is already running.'` 759 | exit 1 760 | fi 761 | # launchd expects that this script stay up and running so we need to do our own monitoring of the Wrapper process. 762 | if [ $WAIT_FOR_STARTED_STATUS = true ] 763 | then 764 | waitforwrapperstop 765 | fi 766 | } 767 | 768 | console() { 769 | eval echo `gettext 'Running $APP_LONG_NAME...'` 770 | getpid 771 | if [ "X$pid" = "X" ] 772 | then 773 | trap '' 3 774 | 775 | prepAdditionalParams "$@" 776 | 777 | # The string passed to eval must handles spaces in paths correctly. 778 | COMMAND_LINE="$CMDNICE \"$WRAPPER_CMD\" \"$WRAPPER_CONF\" wrapper.syslog.ident=\"$APP_NAME\" wrapper.pidfile=\"$PIDFILE\" wrapper.name=\"$APP_NAME\" wrapper.displayname=\"$APP_LONG_NAME\" $ANCHORPROP $STATUSPROP $COMMANDPROP $LOCKPROP $ADDITIONAL_PARA" 779 | eval $COMMAND_LINE 780 | else 781 | eval echo `gettext '$APP_LONG_NAME is already running.'` 782 | exit 1 783 | fi 784 | } 785 | 786 | waitforjavastartup() { 787 | getstatus 788 | eval echo $ECHOOPT `gettext 'Waiting for $APP_LONG_NAME...'` 789 | 790 | # Wait until the timeout or we have something besides Unknown. 791 | counter=15 792 | while [ "$JAVASTATUS" = "Unknown" -a $counter -gt 0 -a -n "$JAVASTATUS" ] ; do 793 | echo $ECHOOPT"." 794 | sleep 1 795 | getstatus 796 | counter=`expr $counter - 1` 797 | done 798 | 799 | if [ -n "$WAIT_FOR_STARTED_TIMEOUT" ] ; then 800 | counter=$WAIT_FOR_STARTED_TIMEOUT 801 | else 802 | counter=120 803 | fi 804 | while [ "$JAVASTATUS" != "STARTED" -a "$JAVASTATUS" != "Unknown" -a $counter -gt 0 -a -n "$JAVASTATUS" ] ; do 805 | echo $ECHOOPT"." 806 | sleep 1 807 | getstatus 808 | counter=`expr $counter - 1` 809 | done 810 | if [ "X$ECHOOPT" != "X" ] ; then 811 | echo "" 812 | fi 813 | } 814 | 815 | startwait() { 816 | if [ $WAIT_FOR_STARTED_STATUS = true ] 817 | then 818 | waitforjavastartup 819 | fi 820 | # Sleep for a few seconds to allow for intialization if required 821 | # then test to make sure we're still running. 822 | # 823 | i=0 824 | while [ $i -lt $WAIT_AFTER_STARTUP ] 825 | do 826 | sleep 1 827 | echo $ECHOOPT"." 828 | i=`expr $i + 1` 829 | done 830 | if [ $WAIT_AFTER_STARTUP -gt 0 -o $WAIT_FOR_STARTED_STATUS = true ] 831 | then 832 | getpid 833 | if [ "X$pid" = "X" ] 834 | then 835 | eval echo `gettext ' WARNING: $APP_LONG_NAME may have failed to start.'` 836 | exit 1 837 | else 838 | eval echo `gettext ' running: PID:$pid'` 839 | fi 840 | else 841 | echo "" 842 | fi 843 | } 844 | 845 | macosxstart() { 846 | # The daemon has been installed. 847 | eval echo `gettext 'Starting $APP_LONG_NAME. Detected Mac OSX and installed launchd daemon.'` 848 | if [ `id | sed 's/^uid=//;s/(.*$//'` != "0" ] ; then 849 | eval echo `gettext 'Must be root to perform this action.'` 850 | exit 1 851 | fi 852 | 853 | getpid 854 | if [ "X$pid" != "X" ] ; then 855 | eval echo `gettext '$APP_LONG_NAME is already running.'` 856 | exit 1 857 | fi 858 | 859 | # If the daemon was just installed, it may not be loaded. 860 | LOADED_PLIST=`launchctl list | grep ${APP_PLIST_BASE}` 861 | if [ "X${LOADED_PLIST}" = "X" ] ; then 862 | launchctl load /Library/LaunchDaemons/${APP_PLIST} 863 | fi 864 | # If launchd is set to run the daemon already at Load, we don't need to call start 865 | getpid 866 | if [ "X$pid" == "X" ] ; then 867 | launchctl start ${APP_PLIST_BASE} 868 | fi 869 | 870 | startwait 871 | } 872 | 873 | upstartstart() { 874 | # The daemon has been installed. 875 | eval echo `gettext 'Starting $APP_LONG_NAME. Detected Linux and installed upstart.'` 876 | if [ `id | sed 's/^uid=//;s/(.*$//'` != "0" ] ; then 877 | eval echo `gettext 'Must be root to perform this action.'` 878 | exit 1 879 | fi 880 | 881 | getpid 882 | if [ "X$pid" != "X" ] ; then 883 | eval echo `gettext '$APP_LONG_NAME is already running.'` 884 | exit 1 885 | fi 886 | 887 | /sbin/start ${APP_NAME} 888 | 889 | startwait 890 | } 891 | 892 | start() { 893 | eval echo `gettext 'Starting $APP_LONG_NAME...'` 894 | getpid 895 | if [ "X$pid" = "X" ] 896 | then 897 | prepAdditionalParams "$@" 898 | 899 | # The string passed to eval must handles spaces in paths correctly. 900 | COMMAND_LINE="$CMDNICE \"$WRAPPER_CMD\" \"$WRAPPER_CONF\" wrapper.syslog.ident=\"$APP_NAME\" wrapper.pidfile=\"$PIDFILE\" wrapper.name=\"$APP_NAME\" wrapper.displayname=\"$APP_LONG_NAME\" wrapper.daemonize=TRUE $ANCHORPROP $IGNOREPROP $STATUSPROP $COMMANDPROP $LOCKPROP $ADDITIONAL_PARA" 901 | eval $COMMAND_LINE 902 | else 903 | eval echo `gettext '$APP_LONG_NAME is already running.'` 904 | exit 1 905 | fi 906 | 907 | startwait 908 | } 909 | 910 | stopit() { 911 | # $1 exit if down flag 912 | 913 | eval echo `gettext 'Stopping $APP_LONG_NAME...'` 914 | getpid 915 | if [ "X$pid" = "X" ] 916 | then 917 | eval echo `gettext '$APP_LONG_NAME was not running.'` 918 | if [ "X$1" = "X1" ] 919 | then 920 | exit 1 921 | fi 922 | else 923 | if [ "X$IGNORE_SIGNALS" = "X" ] 924 | then 925 | # Running so try to stop it. 926 | kill $pid 927 | if [ $? -ne 0 ] 928 | then 929 | # An explanation for the failure should have been given 930 | eval echo `gettext 'Unable to stop $APP_LONG_NAME.'` 931 | exit 1 932 | fi 933 | else 934 | rm -f "$ANCHORFILE" 935 | if [ -f "$ANCHORFILE" ] 936 | then 937 | # An explanation for the failure should have been given 938 | eval echo `gettext 'Unable to stop $APP_LONG_NAME.'` 939 | exit 1 940 | fi 941 | fi 942 | 943 | # We can not predict how long it will take for the wrapper to 944 | # actually stop as it depends on settings in wrapper.conf. 945 | # Loop until it does. 946 | savepid=$pid 947 | CNT=0 948 | TOTCNT=0 949 | while [ "X$pid" != "X" ] 950 | do 951 | # Show a waiting message every 5 seconds. 952 | if [ "$CNT" -lt "5" ] 953 | then 954 | CNT=`expr $CNT + 1` 955 | else 956 | eval echo `gettext 'Waiting for $APP_LONG_NAME to exit...'` 957 | CNT=0 958 | fi 959 | TOTCNT=`expr $TOTCNT + 1` 960 | 961 | sleep 1 962 | 963 | testpid 964 | done 965 | 966 | pid=$savepid 967 | testpid 968 | if [ "X$pid" != "X" ] 969 | then 970 | eval echo `gettext 'Failed to stop $APP_LONG_NAME.'` 971 | exit 1 972 | else 973 | eval echo `gettext 'Stopped $APP_LONG_NAME.'` 974 | fi 975 | fi 976 | } 977 | 978 | pause() { 979 | eval echo `gettext 'Pausing $APP_LONG_NAME.'` 980 | } 981 | 982 | resume() { 983 | eval echo `gettext 'Resuming $APP_LONG_NAME.'` 984 | } 985 | 986 | status() { 987 | getpid 988 | if [ "X$pid" = "X" ] 989 | then 990 | eval echo `gettext '$APP_LONG_NAME is not running.'` 991 | exit 1 992 | else 993 | if [ "X$DETAIL_STATUS" = "X" ] 994 | then 995 | eval echo `gettext '$APP_LONG_NAME is running: PID:$pid'` 996 | else 997 | getstatus 998 | eval echo `gettext '$APP_LONG_NAME is running: PID:$pid, Wrapper:$STATUS, Java:$JAVASTATUS'` 999 | fi 1000 | exit 0 1001 | fi 1002 | } 1003 | 1004 | installUpstart() { 1005 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon using upstart..'` 1006 | if [ -f "${APP_NAME}.conf" ] ; then 1007 | eval echo `gettext ' a custom upstart conf file ${APP_NAME}.conf found'` 1008 | cp "${REALDIR}/${APP_NAME}.install" "/etc/init/${APP_NAME}.conf" 1009 | else 1010 | eval echo `gettext ' creating default upstart conf file..'` 1011 | echo "# ${APP_NAME} - ${APP_LONG_NAME}" > "/etc/init/${APP_NAME}.conf" 1012 | echo "description \"${APP_LONG_NAME}\"" >> "/etc/init/${APP_NAME}.conf" 1013 | echo "author \"Tanuki Software Ltd. \"" >> "/etc/init/${APP_NAME}.conf" 1014 | echo "start on runlevel [2345]" >> "/etc/init/${APP_NAME}.conf" 1015 | echo "stop on runlevel [!2345]" >> "/etc/init/${APP_NAME}.conf" 1016 | echo "env LANG=${LANG}" >> "/etc/init/${APP_NAME}.conf" 1017 | echo "exec \"${REALPATH}\" launchdinternal" >> "/etc/init/${APP_NAME}.conf" 1018 | fi 1019 | } 1020 | 1021 | installdaemon() { 1022 | if [ `id | sed 's/^uid=//;s/(.*$//'` != "0" ] ; then 1023 | eval echo `gettext 'Must be root to perform this action.'` 1024 | exit 1 1025 | else 1026 | APP_NAME_LOWER=`echo "$APP_NAME" | tr "[A-Z]" "[a-z]"` 1027 | if [ "$DIST_OS" = "solaris" ] ; then 1028 | eval echo `gettext 'Detected Solaris:'` 1029 | if [ -f /etc/init.d/$APP_NAME ] ; then 1030 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1031 | exit 1 1032 | else 1033 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1034 | ln -s "$REALPATH" "/etc/init.d/$APP_NAME" 1035 | ln -s "/etc/init.d/$APP_NAME" "/etc/rc3.d/K20$APP_NAME_LOWER" 1036 | ln -s "/etc/init.d/$APP_NAME" "/etc/rc3.d/S20$APP_NAME_LOWER" 1037 | fi 1038 | elif [ "$DIST_OS" = "linux" ] ; then 1039 | if [ -f /etc/redhat-release -o -f /etc/redhat_version -o -f /etc/fedora-release ] ; then 1040 | eval echo `gettext 'Detected RHEL or Fedora:'` 1041 | if [ -f "/etc/init.d/$APP_NAME" -o -f "/etc/init/${APP_NAME}.conf" ] ; then 1042 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1043 | exit 1 1044 | else 1045 | if [ -n "$USE_UPSTART" -a -d "/etc/init" ] ; then 1046 | installUpstart 1047 | else 1048 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1049 | ln -s "$REALPATH" "/etc/init.d/$APP_NAME" 1050 | /sbin/chkconfig --add "$APP_NAME" 1051 | /sbin/chkconfig "$APP_NAME" on 1052 | fi 1053 | fi 1054 | elif [ -f /etc/SuSE-release ] ; then 1055 | eval echo `gettext 'Detected SuSE or SLES:'` 1056 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1057 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1058 | exit 1 1059 | else 1060 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1061 | ln -s "$REALPATH" "/etc/init.d/$APP_NAME" 1062 | insserv "/etc/init.d/$APP_NAME" 1063 | fi 1064 | elif [ -f /etc/lsb-release ] ; then 1065 | eval echo `gettext 'Detected Ubuntu:'` 1066 | if [ -f "/etc/init.d/$APP_NAME" -o -f "/etc/init/${APP_NAME}.conf" ] ; then 1067 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1068 | exit 1 1069 | else 1070 | if [ -n "$USE_UPSTART" -a -d "/etc/init" ] ; then 1071 | installUpstart 1072 | else 1073 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon using init.d..'` 1074 | ln -s "$REALPATH" "/etc/init.d/$APP_NAME" 1075 | update-rc.d "$APP_NAME" defaults 1076 | fi 1077 | fi 1078 | else 1079 | eval echo `gettext 'Detected Linux:'` 1080 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1081 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1082 | exit 1 1083 | else 1084 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1085 | ln -s "$REALPATH" /etc/init.d/$APP_NAME 1086 | ln -s "/etc/init.d/$APP_NAME" "/etc/rc3.d/K20$APP_NAME_LOWER" 1087 | ln -s "/etc/init.d/$APP_NAME" "/etc/rc3.d/S20$APP_NAME_LOWER" 1088 | ln -s "/etc/init.d/$APP_NAME" "/etc/rc5.d/S20$APP_NAME_LOWER" 1089 | ln -s "/etc/init.d/$APP_NAME" "/etc/rc5.d/K20$APP_NAME_LOWER" 1090 | fi 1091 | fi 1092 | elif [ "$DIST_OS" = "hpux" ] ; then 1093 | eval echo `gettext 'Detected HP-UX:'` 1094 | if [ -f "/sbin/init.d/$APP_NAME" ] ; then 1095 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1096 | exit 1 1097 | else 1098 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1099 | ln -s "$REALPATH" "/sbin/init.d/$APP_NAME" 1100 | ln -s "/sbin/init.d/$APP_NAME" "/sbin/rc3.d/K20$APP_NAME_LOWER" 1101 | ln -s "/sbin/init.d/$APP_NAME" "/sbin/rc3.d/S20$APP_NAME_LOWER" 1102 | fi 1103 | elif [ "$DIST_OS" = "aix" ] ; then 1104 | eval echo `gettext 'Detected AIX:'` 1105 | if [ -f "/etc/rc.d/init.d/$APP_NAME" ] ; then 1106 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed as rc.d script.'` 1107 | exit 1 1108 | elif [ -n "`/usr/sbin/lsitab $APP_NAME`" -a -n "`/usr/bin/lssrc -S -s $APP_NAME`" ] ; then 1109 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed as SRC service.'` 1110 | exit 1 1111 | else 1112 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1113 | if [ -n "`/usr/sbin/lsitab install_assist`" ] ; then 1114 | eval echo `gettext ' The task /usr/sbin/install_assist was found in the inittab, this might cause problems for all subsequent tasks to launch at this process is known to block the init task. Please make sure this task is not needed anymore and remove/deactivate it.'` 1115 | fi 1116 | /usr/bin/mkssys -s "$APP_NAME" -p "$REALPATH" -a "launchdinternal" -u 0 -f 9 -n 15 -S 1117 | /usr/sbin/mkitab "$APP_NAME":2:once:"/usr/bin/startsrc -s \"${APP_NAME}\" >/dev/console 2>&1" 1118 | 1119 | fi 1120 | elif [ "$DIST_OS" = "freebsd" ] ; then 1121 | eval echo `gettext 'Detected FreeBSD:'` 1122 | if [ -f "/etc/rc.d/$APP_NAME" ] ; then 1123 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1124 | exit 1 1125 | else 1126 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1127 | sed -i .bak "/${APP_NAME}_enable=\"YES\"/d" /etc/rc.conf 1128 | if [ -f "${REALDIR}/${APP_NAME}.install" ] ; then 1129 | ln -s "${REALDIR}/${APP_NAME}.install" "/etc/rc.d/$APP_NAME" 1130 | else 1131 | echo '#!/bin/sh' > "/etc/rc.d/$APP_NAME" 1132 | echo "#" >> "/etc/rc.d/$APP_NAME" 1133 | echo "# PROVIDE: $APP_NAME" >> "/etc/rc.d/$APP_NAME" 1134 | echo "# REQUIRE: NETWORKING" >> "/etc/rc.d/$APP_NAME" 1135 | echo "# KEYWORD: shutdown" >> "/etc/rc.d/$APP_NAME" 1136 | echo ". /etc/rc.subr" >> "/etc/rc.d/$APP_NAME" 1137 | echo "name=\"$APP_NAME\"" >> "/etc/rc.d/$APP_NAME" 1138 | echo "rcvar=\`set_rcvar\`" >> "/etc/rc.d/$APP_NAME" 1139 | echo "command=\"${REALDIR}/${APP_NAME}\"" >> "/etc/rc.d/$APP_NAME" 1140 | echo 'start_cmd="${name}_start"' >> "/etc/rc.d/$APP_NAME" 1141 | echo 'load_rc_config $name' >> "/etc/rc.d/$APP_NAME" 1142 | echo 'status_cmd="${name}_status"' >> "/etc/rc.d/$APP_NAME" 1143 | echo 'stop_cmd="${name}_stop"' >> "/etc/rc.d/$APP_NAME" 1144 | echo "${APP_NAME}_status() {" >> "/etc/rc.d/$APP_NAME" 1145 | echo '${command} status' >> "/etc/rc.d/$APP_NAME" 1146 | echo '}' >> "/etc/rc.d/$APP_NAME" 1147 | echo "${APP_NAME}_stop() {" >> "/etc/rc.d/$APP_NAME" 1148 | echo '${command} stop' >> "/etc/rc.d/$APP_NAME" 1149 | echo '}' >> "/etc/rc.d/$APP_NAME" 1150 | echo "${APP_NAME}_start() {" >> "/etc/rc.d/$APP_NAME" 1151 | echo '${command} start' >> "/etc/rc.d/$APP_NAME" 1152 | echo '}' >> "/etc/rc.d/$APP_NAME" 1153 | echo 'run_rc_command "$1"' >> "/etc/rc.d/$APP_NAME" 1154 | fi 1155 | echo "${APP_NAME}_enable=\"YES\"" >> /etc/rc.conf 1156 | chmod 555 "/etc/rc.d/$APP_NAME" 1157 | fi 1158 | elif [ "$DIST_OS" = "macosx" ] ; then 1159 | eval echo `gettext 'Detected Mac OSX:'` 1160 | if [ -f "/Library/LaunchDaemons/${APP_PLIST}" ] ; then 1161 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1162 | exit 1 1163 | else 1164 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1165 | if [ -f "${REALDIR}/${APP_PLIST}" ] ; then 1166 | ln -s "${REALDIR}/${APP_PLIST}" "/Library/LaunchDaemons/${APP_PLIST}" 1167 | else 1168 | echo "" > "/Library/LaunchDaemons/${APP_PLIST}" 1169 | echo "> "/Library/LaunchDaemons/${APP_PLIST}" 1170 | echo "\"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">" >> "/Library/LaunchDaemons/${APP_PLIST}" 1171 | echo "" >> "/Library/LaunchDaemons/${APP_PLIST}" 1172 | echo " " >> "/Library/LaunchDaemons/${APP_PLIST}" 1173 | echo " Label" >> "/Library/LaunchDaemons/${APP_PLIST}" 1174 | echo " ${APP_PLIST_BASE}" >> "/Library/LaunchDaemons/${APP_PLIST}" 1175 | echo " ProgramArguments" >> "/Library/LaunchDaemons/${APP_PLIST}" 1176 | echo " " >> "/Library/LaunchDaemons/${APP_PLIST}" 1177 | echo " ${REALDIR}/${APP_NAME}" >> "/Library/LaunchDaemons/${APP_PLIST}" 1178 | echo " launchdinternal" >> "/Library/LaunchDaemons/${APP_PLIST}" 1179 | echo " " >> "/Library/LaunchDaemons/${APP_PLIST}" 1180 | echo " OnDemand" >> "/Library/LaunchDaemons/${APP_PLIST}" 1181 | echo " " >> "/Library/LaunchDaemons/${APP_PLIST}" 1182 | echo " RunAtLoad" >> "/Library/LaunchDaemons/${APP_PLIST}" 1183 | echo " " >> "/Library/LaunchDaemons/${APP_PLIST}" 1184 | if [ "X$RUN_AS_USER" != "X" ] ; then 1185 | echo " UserName" >> "/Library/LaunchDaemons/${APP_PLIST}" 1186 | echo " ${RUN_AS_USER}" >> "/Library/LaunchDaemons/${APP_PLIST}" 1187 | fi 1188 | echo " " >> "/Library/LaunchDaemons/${APP_PLIST}" 1189 | echo "" >> "/Library/LaunchDaemons/${APP_PLIST}" 1190 | fi 1191 | chmod 555 "/Library/LaunchDaemons/${APP_PLIST}" 1192 | fi 1193 | elif [ "$DIST_OS" = "zos" ] ; then 1194 | eval echo `gettext 'Detected z/OS:'` 1195 | if [ -f /etc/rc.bak ] ; then 1196 | eval echo `gettext ' The $APP_LONG_NAME daemon is already installed.'` 1197 | exit 1 1198 | else 1199 | eval echo `gettext ' Installing the $APP_LONG_NAME daemon..'` 1200 | cp /etc/rc /etc/rc.bak 1201 | sed "s:echo /etc/rc script executed, \`date\`::g" /etc/rc.bak > /etc/rc 1202 | echo "_BPX_JOBNAME='${APP_NAME}' \"${REALDIR}/${APP_NAME}\" start" >>/etc/rc 1203 | echo '/etc/rc script executed, `date`' >>/etc/rc 1204 | fi 1205 | else 1206 | eval echo `gettext 'Install not currently supported for $DIST_OS'` 1207 | exit 1 1208 | fi 1209 | fi 1210 | } 1211 | 1212 | removedaemon() { 1213 | if [ `id | sed 's/^uid=//;s/(.*$//'` != "0" ] ; then 1214 | eval echo `gettext 'Must be root to perform this action.'` 1215 | exit 1 1216 | else 1217 | stopit "0" 1218 | APP_NAME_LOWER=`echo "$APP_NAME" | tr "[A-Z]" "[a-z]"` 1219 | if [ "$DIST_OS" = "solaris" ] ; then 1220 | eval echo `gettext 'Detected Solaris:'` 1221 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1222 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1223 | for i in "/etc/rc3.d/S20$APP_NAME_LOWER" "/etc/rc3.d/K20$APP_NAME_LOWER" "/etc/init.d/$APP_NAME" 1224 | do 1225 | rm -f $i 1226 | done 1227 | else 1228 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1229 | exit 1 1230 | fi 1231 | elif [ "$DIST_OS" = "linux" ] ; then 1232 | if [ -f /etc/redhat-release -o -f /etc/redhat_version -o -f /etc/fedora-release ] ; then 1233 | eval echo `gettext 'Detected RHEL or Fedora:'` 1234 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1235 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1236 | /sbin/chkconfig "$APP_NAME" off 1237 | /sbin/chkconfig --del "$APP_NAME" 1238 | rm -f "/etc/init.d/$APP_NAME" 1239 | elif [ -f "/etc/init/${APP_NAME}.conf" ] ; then 1240 | eval echo `gettext ' Removing $APP_LONG_NAME daemon from upstart...'` 1241 | rm "/etc/init/${APP_NAME}.conf" 1242 | else 1243 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1244 | exit 1 1245 | fi 1246 | elif [ -f /etc/SuSE-release ] ; then 1247 | eval echo `gettext 'Detected SuSE or SLES:'` 1248 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1249 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1250 | insserv -r "/etc/init.d/$APP_NAME" 1251 | rm -f "/etc/init.d/$APP_NAME" 1252 | else 1253 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1254 | exit 1 1255 | fi 1256 | elif [ -f /etc/lsb-release ] ; then 1257 | eval echo `gettext 'Detected Ubuntu:'` 1258 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1259 | eval echo `gettext ' Removing $APP_LONG_NAME daemon from init.d...'` 1260 | update-rc.d -f "$APP_NAME" remove 1261 | rm -f "/etc/init.d/$APP_NAME" 1262 | elif [ -f "/etc/init/${APP_NAME}.conf" ] ; then 1263 | eval echo `gettext ' Removing $APP_LONG_NAME daemon from upstart...'` 1264 | rm "/etc/init/${APP_NAME}.conf" 1265 | else 1266 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1267 | exit 1 1268 | fi 1269 | else 1270 | eval echo `gettext 'Detected Linux:'` 1271 | if [ -f "/etc/init.d/$APP_NAME" ] ; then 1272 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1273 | for i in "/etc/rc3.d/K20$APP_NAME_LOWER" "/etc/rc5.d/K20$APP_NAME_LOWER" "/etc/rc3.d/S20$APP_NAME_LOWER" "/etc/init.d/$APP_NAME" "/etc/rc5.d/S20$APP_NAME_LOWER" 1274 | do 1275 | rm -f $i 1276 | done 1277 | else 1278 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1279 | exit 1 1280 | fi 1281 | fi 1282 | elif [ "$DIST_OS" = "hpux" ] ; then 1283 | eval echo `gettext 'Detected HP-UX:'` 1284 | if [ -f "/sbin/init.d/$APP_NAME" ] ; then 1285 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1286 | for i in "/sbin/rc3.d/K20$APP_NAME_LOWER" "/sbin/rc3.d/S20$APP_NAME_LOWER" "/sbin/init.d/$APP_NAME" 1287 | do 1288 | rm -f $i 1289 | done 1290 | else 1291 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1292 | exit 1 1293 | fi 1294 | elif [ "$DIST_OS" = "aix" ] ; then 1295 | eval echo `gettext 'Detected AIX:'` 1296 | if [ -f "/etc/rc.d/init.d/$APP_NAME" -o -n "`/usr/sbin/lsitab $APP_NAME`" -o -n "`/usr/bin/lssrc -S -s $APP_NAME`" ] ; then 1297 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1298 | if [ -f "/etc/rc.d/init.d/$APP_NAME" ] ; then 1299 | for i in "/etc/rc.d/rc2.d/S20$APP_NAME_LOWER" "/etc/rc.d/rc2.d/K20$APP_NAME_LOWER" "/etc/rc.d/init.d/$APP_NAME" 1300 | do 1301 | rm -f $i 1302 | done 1303 | fi 1304 | if [ -n "`/usr/sbin/lsitab $APP_NAME`" -o -n "`/usr/bin/lssrc -S -s $APP_NAME`" ] ; then 1305 | /usr/sbin/rmitab $APP_NAME 1306 | /usr/bin/rmssys -s $APP_NAME 1307 | fi 1308 | else 1309 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1310 | exit 1 1311 | fi 1312 | elif [ "$DIST_OS" = "freebsd" ] ; then 1313 | eval echo `gettext 'Detected FreeBSD:'` 1314 | if [ -f "/etc/rc.d/$APP_NAME" ] ; then 1315 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1316 | for i in "/etc/rc.d/$APP_NAME" 1317 | do 1318 | rm -f $i 1319 | done 1320 | sed -i .bak "/${APP_NAME}_enable=\"YES\"/d" /etc/rc.conf 1321 | else 1322 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1323 | exit 1 1324 | fi 1325 | elif [ "$DIST_OS" = "macosx" ] ; then 1326 | eval echo `gettext 'Detected Mac OSX:'` 1327 | if [ -f "/Library/LaunchDaemons/${APP_PLIST}" ] ; then 1328 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1329 | # Make sure the plist is installed 1330 | LOADED_PLIST=`launchctl list | grep ${APP_PLIST_BASE}` 1331 | if [ "X${LOADED_PLIST}" != "X" ] ; then 1332 | launchctl unload "/Library/LaunchDaemons/${APP_PLIST}" 1333 | fi 1334 | rm -f "/Library/LaunchDaemons/${APP_PLIST}" 1335 | else 1336 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1337 | exit 1 1338 | fi 1339 | elif [ "$DIST_OS" = "zos" ] ; then 1340 | eval echo `gettext 'Detected z/OS:'` 1341 | if [ -f /etc/rc.bak ] ; then 1342 | eval echo `gettext ' Removing $APP_LONG_NAME daemon...'` 1343 | cp /etc/rc /etc/rc.bak 1344 | sed "s/_BPX_JOBNAME=\'APP_NAME\'.*//g" /etc/rc.bak > /etc/rc 1345 | rm /etc/rc.bak 1346 | else 1347 | eval echo `gettext ' The $APP_LONG_NAME daemon is not currently installed.'` 1348 | exit 1 1349 | fi 1350 | else 1351 | eval echo `gettext 'Remove not currently supported for $DIST_OS'` 1352 | exit 1 1353 | fi 1354 | fi 1355 | } 1356 | 1357 | dump() { 1358 | eval echo `gettext 'Dumping $APP_LONG_NAME...'` 1359 | getpid 1360 | if [ "X$pid" = "X" ] 1361 | then 1362 | eval echo `gettext '$APP_LONG_NAME was not running.'` 1363 | else 1364 | kill -3 $pid 1365 | 1366 | if [ $? -ne 0 ] 1367 | then 1368 | eval echo `gettext 'Failed to dump $APP_LONG_NAME.'` 1369 | exit 1 1370 | else 1371 | eval echo `gettext 'Dumped $APP_LONG_NAME.'` 1372 | fi 1373 | fi 1374 | } 1375 | 1376 | # Used by HP-UX init scripts. 1377 | startmsg() { 1378 | getpid 1379 | if [ "X$pid" = "X" ] 1380 | then 1381 | eval echo `gettext 'Starting $APP_LONG_NAME... Wrapper:Stopped'` 1382 | else 1383 | if [ "X$DETAIL_STATUS" = "X" ] 1384 | then 1385 | eval echo `gettext 'Starting $APP_LONG_NAME... Wrapper:Running'` 1386 | else 1387 | getstatus 1388 | eval echo `gettext 'Starting $APP_LONG_NAME... Wrapper:$STATUS, Java:$JAVASTATUS'` 1389 | fi 1390 | fi 1391 | } 1392 | 1393 | # Used by HP-UX init scripts. 1394 | stopmsg() { 1395 | getpid 1396 | if [ "X$pid" = "X" ] 1397 | then 1398 | eval echo `gettext 'Stopping $APP_LONG_NAME... Wrapper:Stopped'` 1399 | else 1400 | if [ "X$DETAIL_STATUS" = "X" ] 1401 | then 1402 | eval echo `gettext 'Stopping $APP_LONG_NAME... Wrapper:Running'` 1403 | else 1404 | getstatus 1405 | eval echo `gettext 'Stopping $APP_LONG_NAME... Wrapper:$STATUS, Java:$JAVASTATUS'` 1406 | fi 1407 | fi 1408 | } 1409 | 1410 | showUsage() { 1411 | # $1 bad command 1412 | 1413 | if [ -n "$1" ] 1414 | then 1415 | eval echo `gettext 'Unexpected command: $1'` 1416 | echo ""; 1417 | fi 1418 | 1419 | eval MSG=`gettext 'Usage: '` 1420 | if [ -n "$FIXED_COMMAND" ] ; then 1421 | if [ -n "$PASS_THROUGH" ] ; then 1422 | echo "${MSG} $0 {JavaAppArgs}" 1423 | else 1424 | echo "${MSG} $0" 1425 | fi 1426 | else 1427 | if [ -n "$PAUSABLE" ] ; then 1428 | if [ -n "$PASS_THROUGH" ] ; then 1429 | echo "${MSG} $0 [ console {JavaAppArgs} | start {JavaAppArgs} | stop | restart {JavaAppArgs} | condrestart {JavaAppArgs} | pause | resume | status | install | remove | dump ]" 1430 | else 1431 | echo "${MSG} $0 [ console | start | stop | restart | condrestart | pause | resume | status | install | remove | dump ]" 1432 | fi 1433 | else 1434 | if [ -n "$PASS_THROUGH" ] ; then 1435 | echo "${MSG} $0 [ console {JavaAppArgs} | start {JavaAppArgs} | stop | restart {JavaAppArgs} | condrestart {JavaAppArgs} | status | install | remove | dump ]" 1436 | else 1437 | echo "${MSG} $0 [ console | start | stop | restart | condrestart | status | install | remove | dump ]" 1438 | fi 1439 | fi 1440 | fi 1441 | 1442 | if [ ! -n "$BRIEF_USAGE" ] 1443 | then 1444 | echo ""; 1445 | if [ ! -n "$FIXED_COMMAND" ] ; then 1446 | echo "`gettext 'Commands:'`" 1447 | echo "`gettext ' console Launch in the current console.'`" 1448 | echo "`gettext ' start Start in the background as a daemon process.'`" 1449 | echo "`gettext ' stop Stop if running as a daemon or in another console.'`" 1450 | echo "`gettext ' restart Stop if running and then start.'`" 1451 | echo "`gettext ' condrestart Restart only if already running.'`" 1452 | if [ -n "$PAUSABLE" ] ; then 1453 | echo "`gettext ' pause Pause if running.'`" 1454 | echo "`gettext ' resume Resume if paused.'`" 1455 | fi 1456 | echo "`gettext ' status Query the current status.'`" 1457 | echo "`gettext ' install Install to start automatically when system boots.'`" 1458 | echo "`gettext ' remove Uninstall.'`" 1459 | echo "`gettext ' dump Request a Java thread dump if running.'`" 1460 | echo ""; 1461 | fi 1462 | if [ -n "$PASS_THROUGH" ] ; then 1463 | echo "`gettext 'JavaAppArgs: Zero or more arguments which will be passed to the Java application.'`" 1464 | echo ""; 1465 | fi 1466 | fi 1467 | 1468 | exit 1 1469 | } 1470 | 1471 | docommand() { 1472 | case "$COMMAND" in 1473 | 'console') 1474 | checkUser touchlock "$@" 1475 | if [ ! -n "$FIXED_COMMAND" ] ; then 1476 | shift 1477 | fi 1478 | console "$@" 1479 | ;; 1480 | 1481 | 'start') 1482 | if [ "$DIST_OS" = "macosx" -a -f "/Library/LaunchDaemons/${APP_PLIST}" ] ; then 1483 | macosxstart 1484 | elif [ "$DIST_OS" = "linux" -a -f "/etc/init/${APP_NAME}.conf" ] ; then 1485 | checkUser touchlock "$@" 1486 | upstartstart 1487 | else 1488 | checkUser touchlock "$@" 1489 | if [ ! -n "$FIXED_COMMAND" ] ; then 1490 | shift 1491 | fi 1492 | start "$@" 1493 | fi 1494 | ;; 1495 | 1496 | 'stop') 1497 | checkUser "" "$COMMAND" 1498 | stopit "0" 1499 | ;; 1500 | 1501 | 'restart') 1502 | checkUser touchlock "$COMMAND" 1503 | if [ ! -n "$FIXED_COMMAND" ] ; then 1504 | shift 1505 | fi 1506 | stopit "0" 1507 | start "$@" 1508 | ;; 1509 | 1510 | 'condrestart') 1511 | checkUser touchlock "$COMMAND" 1512 | if [ ! -n "$FIXED_COMMAND" ] ; then 1513 | shift 1514 | fi 1515 | stopit "1" 1516 | start "$@" 1517 | ;; 1518 | 1519 | 'pause') 1520 | if [ -n "$PAUSABLE" ] 1521 | then 1522 | pause 1523 | else 1524 | showUsage "$COMMAND" 1525 | fi 1526 | ;; 1527 | 1528 | 'resume') 1529 | if [ -n "$PAUSABLE" ] 1530 | then 1531 | resume 1532 | else 1533 | showUsage "$COMMAND" 1534 | fi 1535 | ;; 1536 | 1537 | 'status') 1538 | checkUser "" "$COMMAND" 1539 | status 1540 | ;; 1541 | 1542 | 'install') 1543 | installdaemon 1544 | ;; 1545 | 1546 | 'remove') 1547 | removedaemon 1548 | ;; 1549 | 1550 | 'dump') 1551 | checkUser "" "$COMMAND" 1552 | dump 1553 | ;; 1554 | 1555 | 'start_msg') 1556 | # Internal command called by launchd on HP-UX. 1557 | checkUser "" "$COMMAND" 1558 | startmsg 1559 | ;; 1560 | 1561 | 'stop_msg') 1562 | # Internal command called by launchd on HP-UX. 1563 | checkUser "" "$COMMAND" 1564 | stopmsg 1565 | ;; 1566 | 1567 | 'launchdinternal') 1568 | # Internal command called by launchd on Max OSX. 1569 | # We do not want to call checkUser here as it is handled in the launchd plist file. Doing it here would confuse launchd. 1570 | if [ ! -n "$FIXED_COMMAND" ] ; then 1571 | shift 1572 | fi 1573 | launchdinternal "$@" 1574 | ;; 1575 | 1576 | *) 1577 | showUsage "$COMMAND" 1578 | ;; 1579 | esac 1580 | } 1581 | 1582 | docommand "$@" 1583 | 1584 | exit 0 1585 | -------------------------------------------------------------------------------- /bin/wrapper-linux-x86-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-linux-x86-32 -------------------------------------------------------------------------------- /bin/wrapper-linux-x86-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-linux-x86-64 -------------------------------------------------------------------------------- /bin/wrapper-macosx-arm-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-macosx-arm-64 -------------------------------------------------------------------------------- /bin/wrapper-macosx-universal-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-macosx-universal-32 -------------------------------------------------------------------------------- /bin/wrapper-macosx-universal-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-macosx-universal-64 -------------------------------------------------------------------------------- /bin/wrapper-windows-x86-32.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-windows-x86-32.dll -------------------------------------------------------------------------------- /bin/wrapper-windows-x86-32.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-windows-x86-32.exe -------------------------------------------------------------------------------- /bin/wrapper-windows-x86-64.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-windows-x86-64.dll -------------------------------------------------------------------------------- /bin/wrapper-windows-x86-64.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper-windows-x86-64.exe -------------------------------------------------------------------------------- /bin/wrapper.conf: -------------------------------------------------------------------------------- 1 | #******************************************************************** 2 | # Wrapper Properties 3 | # 4 | # NOTE - Please use src/conf/wrapper.conf.in as a template for your 5 | # own application rather than the values used for the 6 | # TestWrapper sample. 7 | #******************************************************************** 8 | 9 | #environment variable 10 | set.psqueue.home=. 11 | 12 | #Advanced Configuration Properties 13 | wrapper.ignore_sequence_gaps=TRUE 14 | wrapper.shutdown.timeout=60 15 | wrapper.jvm_exit.timeout=30 16 | wrapper.ping.timeout=300 17 | 18 | # Java Application 19 | wrapper.java.command=%JAVA_HOME%/bin/java 20 | wrapper.working.dir=../ 21 | 22 | # Java Main class. This class must implement the WrapperListener interface 23 | # or guarantee that the WrapperManager class is initialized. Helper 24 | # classes are provided to do this for you. See the Integration section 25 | # of the documentation for details. 26 | wrapper.java.mainclass=wjw.psqueue.server.Wrapper 27 | 28 | # Java Classpath (include wrapper.jar) Add class path elements as 29 | # needed starting from 1 30 | wrapper.java.classpath.1=bin/wrapper.jar 31 | wrapper.java.classpath.2=./lib/*.jar 32 | wrapper.java.classpath.3=./classes 33 | 34 | # Java Library Path (location of Wrapper.DLL or libwrapper.so) 35 | wrapper.java.library.path.1=bin 36 | wrapper.java.library.path.2=. 37 | 38 | # Java Additional Parameters 39 | wrapper.java.additional.1=-Dfile.encoding=UTF-8 40 | wrapper.java.additional.2=-Duser.timezone=GMT+8 41 | wrapper.java.additional.3=-Duser.language=zh 42 | wrapper.java.additional.4=-Duser.country=CN 43 | wrapper.java.additional.5=-Djava.net.preferIPv4Stack=true 44 | 45 | wrapper.java.additional.6=-Dorg.tanukisoftware.wrapper.WrapperManager.mbean=true 46 | wrapper.java.additional.7=-Dorg.tanukisoftware.wrapper.WrapperManager.mbean.testing=false 47 | wrapper.java.additional.8=-DJEMonitor=true 48 | 49 | wrapper.java.additional.9=-server 50 | wrapper.java.additional.10=-Xms1g 51 | wrapper.java.additional.11=-Xmx1g 52 | wrapper.java.additional.12=-XX:+UseG1GC 53 | wrapper.java.additional.13=-XX:MaxGCPauseMillis=10 54 | wrapper.java.additional.14=-XX:GCPauseIntervalMillis=200 55 | 56 | # Initial Java Heap Size (in MB) 57 | #wrapper.java.initmemory=3 58 | 59 | # Maximum Java Heap Size (in MB) 60 | #wrapper.java.maxmemory=128 61 | 62 | # Application parameters. Add parameters as needed starting from 1 63 | wrapper.app.parameter.1=wjw.psqueue.server.App 64 | 65 | #******************************************************************** 66 | # Wrapper Logging Properties 67 | #******************************************************************** 68 | # Format of output for the console. (See docs for formats) 69 | wrapper.console.format=PM 70 | 71 | # Log Level for console output. (See docs for log levels) 72 | wrapper.console.loglevel=INFO 73 | 74 | # Log file to use for wrapper output logging. 75 | wrapper.logfile=%psqueue.home%/log/wrapper-psqueue.log 76 | 77 | # Format of output for the log file. (See docs for formats) 78 | wrapper.logfile.format=LPTM 79 | 80 | # Log Level for log file output. (See docs for log levels) 81 | wrapper.logfile.loglevel=INFO 82 | 83 | # Maximum size that the log file will be allowed to grow to before 84 | # the log is rolled. Size is specified in bytes. The default value 85 | # of 0, disables log rolling. May abbreviate with the 'k' (kb) or 86 | # 'm' (mb) suffix. For example: 10m = 10 megabytes. 87 | #This property is ignored unless the wrapper.logfile.rollmode property has a value of SIZE, SIZE_OR_WRAPPER, or SIZE_OR_JVM. 88 | wrapper.logfile.maxsize=100m 89 | 90 | # Controls the roll mode of the log file. (See docs for formats) 91 | wrapper.logfile.rollmode=SIZE 92 | 93 | # Maximum number of rolled log files which will be allowed before old 94 | # files are deleted. The default value of 0 implies no limit. 95 | wrapper.logfile.maxfiles=10 96 | 97 | # Log Level for sys/event log output. (See docs for log levels) 98 | wrapper.syslog.loglevel=NONE 99 | 100 | #******************************************************************** 101 | # Wrapper Windows Properties 102 | #******************************************************************** 103 | # Title to use when running as a console 104 | wrapper.console.title=psqueue server 105 | 106 | #******************************************************************** 107 | # Wrapper Windows NT/2000/XP Service Properties 108 | #******************************************************************** 109 | # WARNING - Do not modify any of these properties when an application 110 | # using this configuration file has been installed as a service. 111 | # Please uninstall the service before modifying this section. The 112 | # service can then be reinstalled. 113 | 114 | # Name of the service 115 | wrapper.ntservice.name=psqueue 116 | 117 | # Display name of the service 118 | wrapper.ntservice.displayname=psqueue 119 | 120 | # Description of the service 121 | wrapper.ntservice.description=psqueue 122 | 123 | # Service dependencies. Add dependencies as needed starting from 1 124 | wrapper.ntservice.dependency.1= 125 | 126 | # Mode in which the service is installed. AUTO_START or DEMAND_START 127 | wrapper.ntservice.starttype=AUTO_START 128 | 129 | # Allow the service to interact with the desktop. 130 | wrapper.ntservice.interactive=false 131 | -------------------------------------------------------------------------------- /bin/wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/bin/wrapper.jar -------------------------------------------------------------------------------- /conf/conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | * 4 | 1818 5 | 200 6 | 60 7 | UTF-8 8 | 30 9 | admin 10 | 123456 11 | 1819 12 | 13 | -------------------------------------------------------------------------------- /conf/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /help-doc/Test-URL.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/help-doc/Test-URL.txt -------------------------------------------------------------------------------- /lib/EasyFastJson-2.7.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/EasyFastJson-2.7.2.jar -------------------------------------------------------------------------------- /lib/istack-commons-runtime.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/istack-commons-runtime.jar -------------------------------------------------------------------------------- /lib/javax.activation-api.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/javax.activation-api.jar -------------------------------------------------------------------------------- /lib/jaxb-api-2.3.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/jaxb-api-2.3.1.jar -------------------------------------------------------------------------------- /lib/jaxb-runtime.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/jaxb-runtime.jar -------------------------------------------------------------------------------- /lib/log4j-1.2.17.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/log4j-1.2.17.jar -------------------------------------------------------------------------------- /lib/netty-all-4.1.7.Final.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/netty-all-4.1.7.Final.jar -------------------------------------------------------------------------------- /lib/slf4j-api.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/slf4j-api.jar -------------------------------------------------------------------------------- /lib/slf4j-log4j.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/lib/slf4j-log4j.jar -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/BigArrayImpl.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.math.BigInteger; 6 | import java.nio.ByteBuffer; 7 | import java.util.concurrent.atomic.AtomicLong; 8 | import java.util.concurrent.locks.Lock; 9 | import java.util.concurrent.locks.ReadWriteLock; 10 | import java.util.concurrent.locks.ReentrantLock; 11 | import java.util.concurrent.locks.ReentrantReadWriteLock; 12 | 13 | import com.leansoft.bigqueue.page.IMappedPage; 14 | import com.leansoft.bigqueue.page.IMappedPageFactory; 15 | import com.leansoft.bigqueue.page.MappedPageFactoryImpl; 16 | import com.leansoft.bigqueue.utils.Calculator; 17 | import com.leansoft.bigqueue.utils.FileUtil; 18 | 19 | /** 20 | * A big array implementation supporting sequential append and random read. 21 | * 22 | * Main features: 23 | * 1. FAST : close to the speed of direct memory access, extremely fast in append only and sequential read modes, 24 | * sequential append and read are close to O(1) memory access, random read is close to O(1) memory access if 25 | * data is in cache and is close to O(1) disk access if data is not in cache. 26 | * 2. MEMORY-EFFICIENT : automatic paging & swapping algorithm, only most-recently accessed data is kept in memory. 27 | * 3. THREAD-SAFE : multiple threads can concurrently read/append the array without data corruption. 28 | * 4. PERSISTENT - all array data is persisted on disk, and is crash resistant. 29 | * 5. BIG(HUGE) - the total size of the array data is only limited by the available disk space. 30 | * 31 | * 32 | * @author bulldog 33 | * 34 | */ 35 | public class BigArrayImpl implements IBigArray { 36 | 37 | // folder name for index page 38 | final static String INDEX_PAGE_FOLDER = "index"; 39 | // folder name for data page 40 | final static String DATA_PAGE_FOLDER = "data"; 41 | // folder name for meta data page 42 | final static String META_DATA_PAGE_FOLDER = "meta_data"; 43 | 44 | // 2 ^ 20 = 1024 * 1024 45 | final static int INDEX_ITEMS_PER_PAGE_BITS = 20; // 1024 * 1024 46 | // number of items per page 47 | final static int INDEX_ITEMS_PER_PAGE = 1 << INDEX_ITEMS_PER_PAGE_BITS; 48 | // 2 ^ 5 = 32 49 | final static int INDEX_ITEM_LENGTH_BITS = 5; 50 | // length in bytes of an index item 51 | final static int INDEX_ITEM_LENGTH = 1 << INDEX_ITEM_LENGTH_BITS; 52 | // size in bytes of an index page 53 | final static int INDEX_PAGE_SIZE = INDEX_ITEM_LENGTH * INDEX_ITEMS_PER_PAGE; 54 | 55 | // size in bytes of a data page 56 | final int DATA_PAGE_SIZE; 57 | 58 | // default size in bytes of a data page 59 | public final static int DEFAULT_DATA_PAGE_SIZE = 128 * 1024 * 1024; 60 | // minimum size in bytes of a data page 61 | public final static int MINIMUM_DATA_PAGE_SIZE = 32 * 1024 * 1024; 62 | // seconds, time to live for index page cached in memory 63 | final static int INDEX_PAGE_CACHE_TTL = 1000; 64 | // seconds, time to live for data page cached in memory 65 | final static int DATA_PAGE_CACHE_TTL = 1000; 66 | // 2 ^ 4 = 16 67 | final static int META_DATA_ITEM_LENGTH_BITS = 4; 68 | // size in bytes of a meta data page 69 | final static int META_DATA_PAGE_SIZE = 1 << META_DATA_ITEM_LENGTH_BITS; 70 | 71 | // private final static int INDEX_ITEM_DATA_PAGE_INDEX_OFFSET = 0; 72 | // private final static int INDEX_ITEM_DATA_ITEM_OFFSET_OFFSET = 8; 73 | private final static int INDEX_ITEM_DATA_ITEM_LENGTH_OFFSET = 12; 74 | // timestamp offset of an data item within an index item 75 | final static int INDEX_ITEM_DATA_ITEM_TIMESTAMP_OFFSET = 16; 76 | 77 | // directory to persist array data 78 | String arrayDirectory; 79 | 80 | // factory for index page management(acquire, release, cache) 81 | IMappedPageFactory indexPageFactory; 82 | // factory for data page management(acquire, release, cache) 83 | IMappedPageFactory dataPageFactory; 84 | // factory for meta data page management(acquire, release, cache) 85 | IMappedPageFactory metaPageFactory; 86 | 87 | // only use the first page 88 | static final long META_DATA_PAGE_INDEX = 0; 89 | 90 | // head index of the big array, this is the read write barrier. 91 | // readers can only read items before this index, and writes can write this index or after 92 | final AtomicLong arrayHeadIndex = new AtomicLong(); 93 | // tail index of the big array, 94 | // readers can't read items before this tail 95 | final AtomicLong arrayTailIndex = new AtomicLong(); 96 | 97 | // head index of the data page, this is the to be appended data page index 98 | long headDataPageIndex; 99 | // head offset of the data page, this is the to be appended data offset 100 | int headDataItemOffset; 101 | 102 | // lock for appending state management 103 | final Lock appendLock = new ReentrantLock(); 104 | 105 | // global lock for array read and write management 106 | final ReadWriteLock arrayReadWritelock = new ReentrantReadWriteLock(); 107 | final Lock arrayReadLock = arrayReadWritelock.readLock(); 108 | final Lock arrayWriteLock = arrayReadWritelock.writeLock(); 109 | 110 | /** 111 | * 112 | * A big array implementation supporting sequential write and random read, 113 | * use default back data file size per page, see {@link #DEFAULT_DATA_PAGE_SIZE}. 114 | * 115 | * @param arrayDir directory for array data store 116 | * @param arrayName the name of the array, will be appended as last part of the array directory 117 | * @throws IOException exception throws during array initialization 118 | */ 119 | public BigArrayImpl(String arrayDir, String arrayName) throws IOException { 120 | this(arrayDir, arrayName, DEFAULT_DATA_PAGE_SIZE); 121 | } 122 | 123 | /** 124 | * A big array implementation supporting sequential write and random read. 125 | * 126 | * @param arrayDir directory for array data store 127 | * @param arrayName the name of the array, will be appended as last part of the array directory 128 | * @param pageSize the back data file size per page in bytes, see minimum allowed {@link #MINIMUM_DATA_PAGE_SIZE}. 129 | * @throws IOException exception throws during array initialization 130 | */ 131 | public BigArrayImpl(String arrayDir, String arrayName, int pageSize) throws IOException { 132 | arrayDirectory = arrayDir; 133 | if (!arrayDirectory.endsWith(File.separator)) { 134 | arrayDirectory += File.separator; 135 | } 136 | // append array name as part of the directory 137 | arrayDirectory = arrayDirectory + arrayName + File.separator; 138 | 139 | // validate directory 140 | if (!FileUtil.isFilenameValid(arrayDirectory)) { 141 | throw new IllegalArgumentException("invalid array directory : " + arrayDirectory); 142 | } 143 | 144 | if (pageSize < MINIMUM_DATA_PAGE_SIZE) { 145 | throw new IllegalArgumentException("invalid page size, allowed minimum is : " + MINIMUM_DATA_PAGE_SIZE + " bytes."); 146 | } 147 | 148 | DATA_PAGE_SIZE = pageSize; 149 | 150 | this.commonInit(); 151 | } 152 | 153 | public String getArrayDirectory() { 154 | return this.arrayDirectory; 155 | } 156 | 157 | 158 | void commonInit() throws IOException { 159 | // initialize page factories 160 | this.indexPageFactory = new MappedPageFactoryImpl(INDEX_PAGE_SIZE, 161 | this.arrayDirectory + INDEX_PAGE_FOLDER, 162 | INDEX_PAGE_CACHE_TTL); 163 | this.dataPageFactory = new MappedPageFactoryImpl(DATA_PAGE_SIZE, 164 | this.arrayDirectory + DATA_PAGE_FOLDER, 165 | DATA_PAGE_CACHE_TTL); 166 | // the ttl does not matter here since meta data page is always cached 167 | this.metaPageFactory = new MappedPageFactoryImpl(META_DATA_PAGE_SIZE, 168 | this.arrayDirectory + META_DATA_PAGE_FOLDER, 169 | 10 * 1000/*does not matter*/); 170 | 171 | // initialize array indexes 172 | initArrayIndex(); 173 | 174 | // initialize data page indexes 175 | initDataPageIndex(); 176 | } 177 | 178 | @Override 179 | public void removeAll() throws IOException { 180 | try { 181 | arrayWriteLock.lock(); 182 | this.indexPageFactory.deleteAllPages(); 183 | this.dataPageFactory.deleteAllPages(); 184 | this.metaPageFactory.deleteAllPages(); 185 | //FileUtil.deleteDirectory(new File(this.arrayDirectory)); 186 | 187 | this.commonInit(); 188 | } finally { 189 | arrayWriteLock.unlock(); 190 | } 191 | } 192 | 193 | @Override 194 | public void removeBeforeIndex(long index) throws IOException { 195 | try { 196 | arrayWriteLock.lock(); 197 | 198 | validateIndex(index); 199 | 200 | long indexPageIndex = Calculator.div(index, INDEX_ITEMS_PER_PAGE_BITS); 201 | 202 | ByteBuffer indexItemBuffer = this.getIndexItemBuffer(index); 203 | long dataPageIndex = indexItemBuffer.getLong(); 204 | 205 | if (indexPageIndex > 0L) { 206 | this.indexPageFactory.deletePagesBeforePageIndex(indexPageIndex); 207 | } 208 | if (dataPageIndex > 0L) { 209 | this.dataPageFactory.deletePagesBeforePageIndex(dataPageIndex); 210 | } 211 | 212 | // advance the tail to index 213 | this.arrayTailIndex.set(index); 214 | } finally { 215 | arrayWriteLock.unlock(); 216 | } 217 | } 218 | 219 | 220 | 221 | @Override 222 | public void removeBefore(long timestamp) throws IOException { 223 | try { 224 | arrayWriteLock.lock(); 225 | long firstIndexPageIndex = this.indexPageFactory.getFirstPageIndexBefore(timestamp); 226 | if (firstIndexPageIndex >= 0) { 227 | // long nextIndexPageIndex = firstIndexPageIndex; 228 | // if (nextIndexPageIndex == Long.MAX_VALUE) { //wrap 229 | // nextIndexPageIndex = 0L; 230 | // } else { 231 | // nextIndexPageIndex++; 232 | // } 233 | long toRemoveBeforeIndex = Calculator.mul(firstIndexPageIndex, INDEX_ITEMS_PER_PAGE_BITS); 234 | removeBeforeIndex(toRemoveBeforeIndex); 235 | } 236 | } catch (IndexOutOfBoundsException ex) { 237 | // ignore 238 | } finally { 239 | arrayWriteLock.unlock(); 240 | } 241 | } 242 | 243 | // find out array head/tail from the meta data 244 | void initArrayIndex() throws IOException { 245 | IMappedPage metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); 246 | ByteBuffer metaBuf = metaDataPage.getLocal(0); 247 | long head = metaBuf.getLong(); 248 | long tail = metaBuf.getLong(); 249 | 250 | arrayHeadIndex.set(head); 251 | arrayTailIndex.set(tail); 252 | } 253 | 254 | // find out data page head index and offset 255 | void initDataPageIndex() throws IOException { 256 | 257 | if (this.isEmpty()) { 258 | headDataPageIndex = 0L; 259 | headDataItemOffset = 0; 260 | } else { 261 | IMappedPage previousIndexPage = null; 262 | long previousIndexPageIndex = -1; 263 | try { 264 | long previousIndex = this.arrayHeadIndex.get() - 1; 265 | if (previousIndex < 0) { 266 | previousIndex = Long.MAX_VALUE; // wrap 267 | } 268 | previousIndexPageIndex = Calculator.div(previousIndex, INDEX_ITEMS_PER_PAGE_BITS); // shift optimization 269 | previousIndexPage = this.indexPageFactory.acquirePage(previousIndexPageIndex); 270 | int previousIndexPageOffset = (int) (Calculator.mul(Calculator.mod(previousIndex, INDEX_ITEMS_PER_PAGE_BITS), INDEX_ITEM_LENGTH_BITS)); 271 | ByteBuffer previousIndexItemBuffer = previousIndexPage.getLocal(previousIndexPageOffset); 272 | long previousDataPageIndex = previousIndexItemBuffer.getLong(); 273 | int previousDataItemOffset = previousIndexItemBuffer.getInt(); 274 | int perviousDataItemLength = previousIndexItemBuffer.getInt(); 275 | 276 | headDataPageIndex = previousDataPageIndex; 277 | headDataItemOffset = previousDataItemOffset + perviousDataItemLength; 278 | } finally { 279 | if (previousIndexPage != null) { 280 | this.indexPageFactory.releasePage(previousIndexPageIndex); 281 | } 282 | } 283 | } 284 | } 285 | 286 | /** 287 | * Append the data into the head of the array 288 | */ 289 | public long append(byte[] data) throws IOException { 290 | try { 291 | arrayReadLock.lock(); 292 | IMappedPage toAppendDataPage = null; 293 | IMappedPage toAppendIndexPage = null; 294 | long toAppendIndexPageIndex = -1L; 295 | long toAppendDataPageIndex = -1L; 296 | 297 | long toAppendArrayIndex = -1L; 298 | 299 | try { 300 | appendLock.lock(); // only one thread can append 301 | 302 | if (this.isFull()) { // end of the world check:) 303 | throw new IOException("ring space of java long type used up, the end of the world!!!"); 304 | } 305 | 306 | // prepare the data pointer 307 | if (this.headDataItemOffset + data.length > DATA_PAGE_SIZE) { // not enough space 308 | if (this.headDataPageIndex == Long.MAX_VALUE) { 309 | this.headDataPageIndex = 0L; // wrap 310 | } else { 311 | this.headDataPageIndex++; 312 | } 313 | this.headDataItemOffset = 0; 314 | } 315 | 316 | toAppendDataPageIndex = this.headDataPageIndex; 317 | int toAppendDataItemOffset = this.headDataItemOffset; 318 | 319 | toAppendArrayIndex = this.arrayHeadIndex.get(); 320 | 321 | // append data 322 | toAppendDataPage = this.dataPageFactory.acquirePage(toAppendDataPageIndex); 323 | ByteBuffer toAppendDataPageBuffer = toAppendDataPage.getLocal(toAppendDataItemOffset); 324 | toAppendDataPageBuffer.put(data); 325 | toAppendDataPage.setDirty(true); 326 | // update to next 327 | this.headDataItemOffset += data.length; 328 | 329 | toAppendIndexPageIndex = Calculator.div(toAppendArrayIndex, INDEX_ITEMS_PER_PAGE_BITS); // shift optimization 330 | toAppendIndexPage = this.indexPageFactory.acquirePage(toAppendIndexPageIndex); 331 | int toAppendIndexItemOffset = (int) (Calculator.mul(Calculator.mod(toAppendArrayIndex, INDEX_ITEMS_PER_PAGE_BITS), INDEX_ITEM_LENGTH_BITS)); 332 | 333 | // update index 334 | ByteBuffer toAppendIndexPageBuffer = toAppendIndexPage.getLocal(toAppendIndexItemOffset); 335 | toAppendIndexPageBuffer.putLong(toAppendDataPageIndex); 336 | toAppendIndexPageBuffer.putInt(toAppendDataItemOffset); 337 | toAppendIndexPageBuffer.putInt(data.length); 338 | long currentTime = System.currentTimeMillis(); 339 | toAppendIndexPageBuffer.putLong(currentTime); 340 | toAppendIndexPage.setDirty(true); 341 | 342 | // advance the head 343 | this.arrayHeadIndex.incrementAndGet(); 344 | 345 | // update meta data 346 | IMappedPage metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); 347 | ByteBuffer metaDataBuf = metaDataPage.getLocal(0); 348 | metaDataBuf.putLong(this.arrayHeadIndex.get()); 349 | metaDataBuf.putLong(this.arrayTailIndex.get()); 350 | metaDataPage.setDirty(true); 351 | 352 | } finally { 353 | 354 | appendLock.unlock(); 355 | 356 | if (toAppendDataPage != null) { 357 | this.dataPageFactory.releasePage(toAppendDataPageIndex); 358 | } 359 | if (toAppendIndexPage != null) { 360 | this.indexPageFactory.releasePage(toAppendIndexPageIndex); 361 | } 362 | } 363 | 364 | return toAppendArrayIndex; 365 | 366 | } finally { 367 | arrayReadLock.unlock(); 368 | } 369 | } 370 | 371 | @Override 372 | public void flush() { 373 | try { 374 | arrayReadLock.lock(); 375 | 376 | // try { 377 | // appendLock.lock(); // make flush and append mutually exclusive 378 | 379 | this.metaPageFactory.flush(); 380 | this.indexPageFactory.flush(); 381 | this.dataPageFactory.flush(); 382 | 383 | // } finally { 384 | // appendLock.unlock(); 385 | // } 386 | 387 | } finally { 388 | arrayReadLock.unlock(); 389 | } 390 | 391 | } 392 | 393 | public byte[] get(long index) throws IOException { 394 | try { 395 | arrayReadLock.lock(); 396 | validateIndex(index); 397 | 398 | IMappedPage dataPage = null; 399 | long dataPageIndex = -1L; 400 | try { 401 | ByteBuffer indexItemBuffer = this.getIndexItemBuffer(index); 402 | dataPageIndex = indexItemBuffer.getLong(); 403 | int dataItemOffset = indexItemBuffer.getInt(); 404 | int dataItemLength = indexItemBuffer.getInt(); 405 | dataPage = this.dataPageFactory.acquirePage(dataPageIndex); 406 | byte[] data = dataPage.getLocal(dataItemOffset, dataItemLength); 407 | return data; 408 | } finally { 409 | if (dataPage != null) { 410 | this.dataPageFactory.releasePage(dataPageIndex); 411 | } 412 | } 413 | } finally { 414 | arrayReadLock.unlock(); 415 | } 416 | } 417 | 418 | public long getTimestamp(long index) throws IOException { 419 | try { 420 | arrayReadLock.lock(); 421 | validateIndex(index); 422 | 423 | ByteBuffer indexItemBuffer = this.getIndexItemBuffer(index); 424 | // position to the timestamp 425 | int position = indexItemBuffer.position(); 426 | indexItemBuffer.position(position + INDEX_ITEM_DATA_ITEM_TIMESTAMP_OFFSET); 427 | long ts = indexItemBuffer.getLong(); 428 | return ts; 429 | } finally { 430 | arrayReadLock.unlock(); 431 | } 432 | } 433 | 434 | ByteBuffer getIndexItemBuffer(long index) throws IOException { 435 | 436 | IMappedPage indexPage = null; 437 | long indexPageIndex = -1L; 438 | try { 439 | indexPageIndex = Calculator.div(index, INDEX_ITEMS_PER_PAGE_BITS); // shift optimization 440 | indexPage = this.indexPageFactory.acquirePage(indexPageIndex); 441 | int indexItemOffset = (int) (Calculator.mul(Calculator.mod(index, INDEX_ITEMS_PER_PAGE_BITS), INDEX_ITEM_LENGTH_BITS)); 442 | 443 | ByteBuffer indexItemBuffer = indexPage.getLocal(indexItemOffset); 444 | return indexItemBuffer; 445 | } finally { 446 | if (indexPage != null) { 447 | this.indexPageFactory.releasePage(indexPageIndex); 448 | } 449 | } 450 | } 451 | 452 | void validateIndex(long index) { 453 | if (this.arrayTailIndex.get() <= this.arrayHeadIndex.get()) { 454 | if (index < this.arrayTailIndex.get() || index >= this.arrayHeadIndex.get()) { 455 | throw new IndexOutOfBoundsException(); 456 | } 457 | } else { 458 | if (index < this.arrayTailIndex.get() && index >= this.arrayHeadIndex.get()) { 459 | throw new IndexOutOfBoundsException(); 460 | } 461 | } 462 | } 463 | 464 | public long size() { 465 | try { 466 | arrayReadLock.lock(); 467 | if (this.arrayTailIndex.get() <= this.arrayHeadIndex.get()) { 468 | return (this.arrayHeadIndex.get() - this.arrayTailIndex.get()); 469 | } else { 470 | return Long.MAX_VALUE - this.arrayTailIndex.get() + 1 + this.arrayHeadIndex.get(); 471 | } 472 | } finally { 473 | arrayReadLock.unlock(); 474 | } 475 | } 476 | 477 | public long getHeadIndex() { 478 | try { 479 | arrayReadLock.lock(); 480 | return arrayHeadIndex.get(); 481 | } finally { 482 | arrayReadLock.unlock(); 483 | } 484 | } 485 | 486 | public long getTailIndex() { 487 | try { 488 | arrayReadLock.lock(); 489 | return arrayTailIndex.get(); 490 | } finally { 491 | arrayReadLock.unlock(); 492 | } 493 | } 494 | 495 | @Override 496 | public boolean isEmpty() { 497 | try { 498 | arrayReadLock.lock(); 499 | return this.arrayHeadIndex.get() == this.arrayTailIndex.get(); 500 | } finally { 501 | arrayReadLock.unlock(); 502 | } 503 | } 504 | 505 | @Override 506 | public boolean isFull() { 507 | try { 508 | arrayReadLock.lock(); 509 | long currentIndex = this.arrayHeadIndex.get(); 510 | 511 | long nextIndex = currentIndex == Long.MAX_VALUE ? 0 : currentIndex + 1; 512 | return nextIndex == this.arrayTailIndex.get(); 513 | } finally { 514 | arrayReadLock.unlock(); 515 | } 516 | } 517 | 518 | @Override 519 | public void close() throws IOException { 520 | try { 521 | arrayWriteLock.lock(); 522 | if (this.metaPageFactory != null) { 523 | this.metaPageFactory.releaseCachedPages(); 524 | } 525 | if (this.indexPageFactory != null) { 526 | this.indexPageFactory.releaseCachedPages(); 527 | } 528 | if (this.dataPageFactory != null) { 529 | this.dataPageFactory.releaseCachedPages(); 530 | } 531 | } finally { 532 | arrayWriteLock.unlock(); 533 | } 534 | } 535 | 536 | @Override 537 | public int getDataPageSize() { 538 | return DATA_PAGE_SIZE; 539 | } 540 | 541 | @Override 542 | public long findClosestIndex(long timestamp) throws IOException { 543 | try { 544 | arrayReadLock.lock(); 545 | long closestIndex = NOT_FOUND; 546 | long tailIndex = this.arrayTailIndex.get(); 547 | long headIndex = this.arrayHeadIndex.get(); 548 | if (tailIndex == headIndex) return closestIndex; // empty 549 | long lastIndex = headIndex - 1; 550 | if (lastIndex < 0) { 551 | lastIndex = Long.MAX_VALUE; 552 | } 553 | if (tailIndex < lastIndex) { 554 | closestIndex = closestBinarySearch(tailIndex, lastIndex, timestamp); 555 | } else { 556 | long lowPartClosestIndex = closestBinarySearch(0L, lastIndex, timestamp); 557 | long highPartClosetIndex = closestBinarySearch(tailIndex, Long.MAX_VALUE, timestamp); 558 | 559 | long lowPartTimestamp = this.getTimestamp(lowPartClosestIndex); 560 | long highPartTimestamp = this.getTimestamp(highPartClosetIndex); 561 | 562 | closestIndex = Math.abs(timestamp - lowPartTimestamp) < Math.abs(timestamp - highPartTimestamp) 563 | ? lowPartClosestIndex : highPartClosetIndex; 564 | } 565 | 566 | return closestIndex; 567 | } finally { 568 | arrayReadLock.unlock(); 569 | } 570 | } 571 | 572 | private long closestBinarySearch(long low, long high, long timestamp) throws IOException { 573 | long mid; 574 | long sum = low + high; 575 | if (sum < 0) { // overflow 576 | BigInteger bigSum = BigInteger.valueOf(low); 577 | bigSum = bigSum.add(BigInteger.valueOf(high)); 578 | mid = bigSum.shiftRight(1).longValue(); 579 | } else { 580 | mid = sum / 2; 581 | } 582 | 583 | long midTimestamp = this.getTimestamp(mid); 584 | 585 | if (midTimestamp < timestamp) { 586 | long nextLow = mid + 1; 587 | if (nextLow >= high) { 588 | return high; 589 | } 590 | return closestBinarySearch(nextLow, high, timestamp); 591 | } else if (midTimestamp > timestamp) { 592 | long nextHigh = mid - 1; 593 | if (nextHigh <= low) { 594 | return low; 595 | } 596 | return closestBinarySearch(low, nextHigh, timestamp); 597 | } else { 598 | return mid; 599 | } 600 | } 601 | 602 | @Override 603 | public long getBackFileSize() throws IOException { 604 | try { 605 | arrayReadLock.lock(); 606 | 607 | return this._getBackFileSize(); 608 | 609 | } finally { 610 | arrayReadLock.unlock(); 611 | } 612 | } 613 | 614 | @Override 615 | public void limitBackFileSize(long sizeLimit) throws IOException { 616 | if (sizeLimit < INDEX_PAGE_SIZE + DATA_PAGE_SIZE) { 617 | return; // ignore, one index page + one data page are minimum for big array to work correctly 618 | } 619 | 620 | long backFileSize = this.getBackFileSize(); 621 | if (backFileSize <= sizeLimit) return; // nothing to do 622 | 623 | long toTruncateSize = backFileSize - sizeLimit; 624 | if (toTruncateSize < DATA_PAGE_SIZE) { 625 | return; // can't do anything 626 | } 627 | 628 | try { 629 | arrayWriteLock.lock(); 630 | 631 | // double check 632 | backFileSize = this._getBackFileSize(); 633 | if (backFileSize <= sizeLimit) return; // nothing to do 634 | 635 | toTruncateSize = backFileSize - sizeLimit; 636 | if (toTruncateSize < DATA_PAGE_SIZE) { 637 | return; // can't do anything 638 | } 639 | 640 | long tailIndex = this.arrayTailIndex.get(); 641 | long headIndex = this.arrayHeadIndex.get(); 642 | long totalLength = 0L; 643 | while(true) { 644 | if (tailIndex == headIndex) break; 645 | totalLength += this.getDataItemLength(tailIndex); 646 | if (totalLength > toTruncateSize) break; 647 | 648 | if (tailIndex == Long.MAX_VALUE) { 649 | tailIndex = 0; 650 | } else { 651 | tailIndex++; 652 | } 653 | if (Calculator.mod(tailIndex, INDEX_ITEMS_PER_PAGE_BITS) == 0) { // take index page into account 654 | totalLength += INDEX_PAGE_SIZE; 655 | } 656 | } 657 | this.removeBeforeIndex(tailIndex); 658 | } finally { 659 | arrayWriteLock.unlock(); 660 | } 661 | 662 | } 663 | 664 | @Override 665 | public int getItemLength(long index) throws IOException { 666 | try { 667 | arrayReadLock.lock(); 668 | validateIndex(index); 669 | 670 | return getDataItemLength(index); 671 | 672 | } finally { 673 | arrayReadLock.unlock(); 674 | } 675 | } 676 | 677 | private int getDataItemLength(long index) throws IOException { 678 | 679 | ByteBuffer indexItemBuffer = this.getIndexItemBuffer(index); 680 | // position to the data item length 681 | int position = indexItemBuffer.position(); 682 | indexItemBuffer.position(position + INDEX_ITEM_DATA_ITEM_LENGTH_OFFSET); 683 | int length = indexItemBuffer.getInt(); 684 | return length; 685 | } 686 | 687 | // inner getBackFileSize 688 | private long _getBackFileSize() throws IOException { 689 | return this.indexPageFactory.getBackPageFileSize() + this.dataPageFactory.getBackPageFileSize(); 690 | } 691 | } 692 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/BigQueueImpl.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue; 2 | 3 | import java.io.IOException; 4 | import java.nio.ByteBuffer; 5 | import java.util.concurrent.atomic.AtomicLong; 6 | import java.util.concurrent.locks.Lock; 7 | import java.util.concurrent.locks.ReentrantLock; 8 | 9 | import com.leansoft.bigqueue.page.IMappedPage; 10 | import com.leansoft.bigqueue.page.IMappedPageFactory; 11 | import com.leansoft.bigqueue.page.MappedPageFactoryImpl; 12 | 13 | 14 | /** 15 | * A big, fast and persistent queue implementation. 16 | * 17 | * Main features: 18 | * 1. FAST : close to the speed of direct memory access, both enqueue and dequeue are close to O(1) memory access. 19 | * 2. MEMORY-EFFICIENT : automatic paging & swapping algorithm, only most-recently accessed data is kept in memory. 20 | * 3. THREAD-SAFE : multiple threads can concurrently enqueue and dequeue without data corruption. 21 | * 4. PERSISTENT - all data in queue is persisted on disk, and is crash resistant. 22 | * 5. BIG(HUGE) - the total size of the queued data is only limited by the available disk space. 23 | * 24 | * 25 | * @author bulldog 26 | * 27 | */ 28 | public class BigQueueImpl implements IBigQueue { 29 | 30 | final IBigArray innerArray; 31 | 32 | // 2 ^ 3 = 8 33 | final static int QUEUE_FRONT_INDEX_ITEM_LENGTH_BITS = 3; 34 | // size in bytes of queue front index page 35 | final static int QUEUE_FRONT_INDEX_PAGE_SIZE = 1 << QUEUE_FRONT_INDEX_ITEM_LENGTH_BITS; 36 | // only use the first page 37 | static final long QUEUE_FRONT_PAGE_INDEX = 0; 38 | 39 | // folder name for queue front index page 40 | final static String QUEUE_FRONT_INDEX_PAGE_FOLDER = "front_index"; 41 | 42 | // front index of the big queue, 43 | final AtomicLong queueFrontIndex = new AtomicLong(); 44 | 45 | // factory for queue front index page management(acquire, release, cache) 46 | IMappedPageFactory queueFrontIndexPageFactory; 47 | 48 | // locks for queue front write management 49 | final Lock queueFrontWriteLock = new ReentrantLock(); 50 | 51 | /** 52 | * A big, fast and persistent queue implementation, 53 | * use default back data page size, see {@link BigArrayImpl#DEFAULT_DATA_PAGE_SIZE} 54 | * 55 | * @param queueDir the directory to store queue data 56 | * @param queueName the name of the queue, will be appended as last part of the queue directory 57 | * @throws IOException exception throws if there is any IO error during queue initialization 58 | */ 59 | public BigQueueImpl(String queueDir, String queueName) throws IOException { 60 | this(queueDir, queueName, BigArrayImpl.DEFAULT_DATA_PAGE_SIZE); 61 | } 62 | 63 | /** 64 | * A big, fast and persistent queue implementation. 65 | * 66 | * @param queueDir the directory to store queue data 67 | * @param queueName the name of the queue, will be appended as last part of the queue directory 68 | * @param pageSize the back data file size per page in bytes, see minimum allowed {@link BigArrayImpl#MINIMUM_DATA_PAGE_SIZE} 69 | * @throws IOException exception throws if there is any IO error during queue initialization 70 | */ 71 | public BigQueueImpl(String queueDir, String queueName, int pageSize) throws IOException { 72 | innerArray = new BigArrayImpl(queueDir, queueName, pageSize); 73 | 74 | // the ttl does not matter here since queue front index page is always cached 75 | this.queueFrontIndexPageFactory = new MappedPageFactoryImpl(QUEUE_FRONT_INDEX_PAGE_SIZE, 76 | ((BigArrayImpl)innerArray).getArrayDirectory() + QUEUE_FRONT_INDEX_PAGE_FOLDER, 77 | 10 * 1000/*does not matter*/); 78 | IMappedPage queueFrontIndexPage = this.queueFrontIndexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 79 | 80 | ByteBuffer queueFrontIndexBuffer = queueFrontIndexPage.getLocal(0); 81 | long front = queueFrontIndexBuffer.getLong(); 82 | queueFrontIndex.set(front); 83 | } 84 | 85 | @Override 86 | public boolean isEmpty() { 87 | return this.queueFrontIndex.get() == this.innerArray.getHeadIndex(); 88 | } 89 | 90 | @Override 91 | public void enqueue(byte[] data) throws IOException { 92 | this.innerArray.append(data); 93 | } 94 | 95 | @Override 96 | public byte[] dequeue() throws IOException { 97 | long queueFrontIndex = -1L; 98 | try { 99 | queueFrontWriteLock.lock(); 100 | if (this.isEmpty()) { 101 | return null; 102 | } 103 | queueFrontIndex = this.queueFrontIndex.get(); 104 | byte[] data = this.innerArray.get(queueFrontIndex); 105 | long nextQueueFrontIndex = queueFrontIndex; 106 | if (nextQueueFrontIndex == Long.MAX_VALUE) { 107 | nextQueueFrontIndex = 0L; // wrap 108 | } else { 109 | nextQueueFrontIndex++; 110 | } 111 | this.queueFrontIndex.set(nextQueueFrontIndex); 112 | // persist the queue front 113 | IMappedPage queueFrontIndexPage = this.queueFrontIndexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 114 | ByteBuffer queueFrontIndexBuffer = queueFrontIndexPage.getLocal(0); 115 | queueFrontIndexBuffer.putLong(nextQueueFrontIndex); 116 | queueFrontIndexPage.setDirty(true); 117 | return data; 118 | } finally { 119 | queueFrontWriteLock.unlock(); 120 | } 121 | 122 | } 123 | 124 | @Override 125 | public void removeAll() throws IOException { 126 | try { 127 | queueFrontWriteLock.lock(); 128 | this.innerArray.removeAll(); 129 | this.queueFrontIndex.set(0L); 130 | IMappedPage queueFrontIndexPage = this.queueFrontIndexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 131 | ByteBuffer queueFrontIndexBuffer = queueFrontIndexPage.getLocal(0); 132 | queueFrontIndexBuffer.putLong(0L); 133 | queueFrontIndexPage.setDirty(true); 134 | } finally { 135 | queueFrontWriteLock.unlock(); 136 | } 137 | } 138 | 139 | @Override 140 | public byte[] peek() throws IOException { 141 | if (this.isEmpty()) { 142 | return null; 143 | } 144 | byte[] data = this.innerArray.get(this.queueFrontIndex.get()); 145 | return data; 146 | } 147 | 148 | /** 149 | * apply an implementation of a ItemIterator interface for each queue item 150 | * 151 | * @param iterator 152 | * @throws IOException 153 | */ 154 | @Override 155 | public void applyForEach(ItemIterator iterator) throws IOException { 156 | try { 157 | queueFrontWriteLock.lock(); 158 | if (this.isEmpty()) { 159 | return; 160 | } 161 | 162 | long index = this.queueFrontIndex.get(); 163 | for (long i=index; i queueFrontMap = new ConcurrentHashMap(); 48 | 49 | /** 50 | * A big, fast and persistent queue implementation with fandout support. 51 | * 52 | * @param queueDir the directory to store queue data 53 | * @param queueName the name of the queue, will be appended as last part of the queue directory 54 | * @param pageSize the back data file size per page in bytes, see minimum allowed {@link BigArrayImpl#MINIMUM_DATA_PAGE_SIZE} 55 | * @throws IOException exception throws if there is any IO error during queue initialization 56 | */ 57 | public FanOutQueueImpl(String queueDir, String queueName, int pageSize) 58 | throws IOException { 59 | innerArray = new BigArrayImpl(queueDir, queueName, pageSize); 60 | } 61 | 62 | /** 63 | * A big, fast and persistent queue implementation with fanout support, 64 | * use default back data page size, see {@link BigArrayImpl#DEFAULT_DATA_PAGE_SIZE} 65 | * 66 | * @param queueDir the directory to store queue data 67 | * @param queueName the name of the queue, will be appended as last part of the queue directory 68 | * @throws IOException exception throws if there is any IO error during queue initialization 69 | */ 70 | public FanOutQueueImpl(String queueDir, String queueName) throws IOException { 71 | this(queueDir, queueName, BigArrayImpl.DEFAULT_DATA_PAGE_SIZE); 72 | } 73 | 74 | QueueFront getQueueFront(String fanoutId) throws IOException { 75 | QueueFront qf = this.queueFrontMap.get(fanoutId); 76 | if (qf == null) { // not in cache, need to create one 77 | qf = new QueueFront(fanoutId); 78 | QueueFront found = this.queueFrontMap.putIfAbsent(fanoutId, qf); 79 | if (found != null) { 80 | qf.indexPageFactory.releaseCachedPages(); 81 | qf = found; 82 | } 83 | } 84 | 85 | return qf; 86 | } 87 | 88 | @Override 89 | public boolean isEmpty(String fanoutId) throws IOException { 90 | try { 91 | this.innerArray.arrayReadLock.lock(); 92 | 93 | QueueFront qf = this.getQueueFront(fanoutId); 94 | return qf.index.get() == innerArray.getHeadIndex(); 95 | 96 | } finally { 97 | this.innerArray.arrayReadLock.unlock(); 98 | } 99 | } 100 | 101 | @Override 102 | public boolean isEmpty() { 103 | return this.innerArray.isEmpty(); 104 | } 105 | 106 | @Override 107 | public long enqueue(byte[] data) throws IOException { 108 | return innerArray.append(data); 109 | } 110 | 111 | @Override 112 | public byte[] dequeue(String fanoutId) throws IOException { 113 | try { 114 | this.innerArray.arrayReadLock.lock(); 115 | 116 | QueueFront qf = this.getQueueFront(fanoutId); 117 | try { 118 | qf.writeLock.lock(); 119 | 120 | if (qf.index.get() == innerArray.arrayHeadIndex.get()) { 121 | return null; // empty 122 | } 123 | 124 | byte[] data = innerArray.get(qf.index.get()); 125 | qf.incrementIndex(); 126 | 127 | return data; 128 | } catch (IndexOutOfBoundsException ex) { 129 | ex.printStackTrace(); 130 | qf.resetIndex(); // maybe the back array has been truncated to limit size 131 | 132 | byte[] data = innerArray.get(qf.index.get()); 133 | qf.incrementIndex(); 134 | 135 | return data; 136 | 137 | } finally { 138 | qf.writeLock.unlock(); 139 | } 140 | 141 | } finally { 142 | this.innerArray.arrayReadLock.unlock(); 143 | } 144 | } 145 | 146 | @Override 147 | public byte[] peek(String fanoutId) throws IOException { 148 | try { 149 | this.innerArray.arrayReadLock.lock(); 150 | 151 | QueueFront qf = this.getQueueFront(fanoutId); 152 | if (qf.index.get() == innerArray.getHeadIndex()) { 153 | return null; // empty 154 | } 155 | 156 | return innerArray.get(qf.index.get()); 157 | 158 | } finally { 159 | this.innerArray.arrayReadLock.unlock(); 160 | } 161 | } 162 | 163 | @Override 164 | public int peekLength(String fanoutId) throws IOException { 165 | try { 166 | this.innerArray.arrayReadLock.lock(); 167 | 168 | QueueFront qf = this.getQueueFront(fanoutId); 169 | if (qf.index.get() == innerArray.getHeadIndex()) { 170 | return -1; // empty 171 | } 172 | return innerArray.getItemLength(qf.index.get()); 173 | 174 | } finally { 175 | this.innerArray.arrayReadLock.unlock(); 176 | } 177 | } 178 | 179 | @Override 180 | public long peekTimestamp(String fanoutId) throws IOException { 181 | try { 182 | this.innerArray.arrayReadLock.lock(); 183 | 184 | QueueFront qf = this.getQueueFront(fanoutId); 185 | if (qf.index.get() == innerArray.getHeadIndex()) { 186 | return -1; // empty 187 | } 188 | return innerArray.getTimestamp(qf.index.get()); 189 | 190 | } finally { 191 | this.innerArray.arrayReadLock.unlock(); 192 | } 193 | } 194 | 195 | 196 | @Override 197 | public byte[] get(long index) throws IOException { 198 | return this.innerArray.get(index); 199 | } 200 | 201 | @Override 202 | public int getLength(long index) throws IOException { 203 | return this.innerArray.getItemLength(index); 204 | } 205 | 206 | @Override 207 | public long getTimestamp(long index) throws IOException { 208 | return this.innerArray.getTimestamp(index); 209 | } 210 | 211 | @Override 212 | public void removeBefore(long timestamp) throws IOException { 213 | try { 214 | this.innerArray.arrayWriteLock.lock(); 215 | 216 | this.innerArray.removeBefore(timestamp); 217 | for(QueueFront qf : this.queueFrontMap.values()) { 218 | try { 219 | qf.writeLock.lock(); 220 | qf.validateAndAdjustIndex(); 221 | } finally { 222 | qf.writeLock.unlock(); 223 | } 224 | } 225 | } finally { 226 | this.innerArray.arrayWriteLock.unlock(); 227 | } 228 | } 229 | 230 | @Override 231 | public void limitBackFileSize(long sizeLimit) throws IOException { 232 | try { 233 | this.innerArray.arrayWriteLock.lock(); 234 | 235 | this.innerArray.limitBackFileSize(sizeLimit); 236 | 237 | for(QueueFront qf : this.queueFrontMap.values()) { 238 | try { 239 | qf.writeLock.lock(); 240 | qf.validateAndAdjustIndex(); 241 | } finally { 242 | qf.writeLock.unlock(); 243 | } 244 | } 245 | 246 | } finally { 247 | this.innerArray.arrayWriteLock.unlock(); 248 | } 249 | } 250 | 251 | @Override 252 | public long getBackFileSize() throws IOException { 253 | return this.innerArray.getBackFileSize(); 254 | } 255 | 256 | @Override 257 | public long findClosestIndex(long timestamp) throws IOException { 258 | try { 259 | this.innerArray.arrayReadLock.lock(); 260 | 261 | if (timestamp == LATEST) { 262 | return this.innerArray.getHeadIndex(); 263 | } 264 | if (timestamp == EARLIEST) { 265 | return this.innerArray.getTailIndex(); 266 | } 267 | return this.innerArray.findClosestIndex(timestamp); 268 | 269 | } finally { 270 | this.innerArray.arrayReadLock.unlock(); 271 | } 272 | } 273 | 274 | @Override 275 | public void resetQueueFrontIndex(String fanoutId, long index) throws IOException { 276 | try { 277 | this.innerArray.arrayReadLock.lock(); 278 | 279 | QueueFront qf = this.getQueueFront(fanoutId); 280 | 281 | try { 282 | qf.writeLock.lock(); 283 | 284 | if (index != this.innerArray.getHeadIndex()) { // ok to set index to array head index 285 | this.innerArray.validateIndex(index); 286 | } 287 | 288 | qf.index.set(index); 289 | qf.persistIndex(); 290 | 291 | } finally { 292 | qf.writeLock.unlock(); 293 | } 294 | 295 | } finally { 296 | this.innerArray.arrayReadLock.unlock(); 297 | } 298 | } 299 | 300 | @Override 301 | public long size(String fanoutId) throws IOException { 302 | try { 303 | this.innerArray.arrayReadLock.lock(); 304 | 305 | QueueFront qf = this.getQueueFront(fanoutId); 306 | long qFront = qf.index.get(); 307 | long qRear = innerArray.getHeadIndex(); 308 | if (qFront <= qRear) { 309 | return (qRear - qFront); 310 | } else { 311 | return Long.MAX_VALUE - qFront + 1 + qRear; 312 | } 313 | 314 | } finally { 315 | this.innerArray.arrayReadLock.unlock(); 316 | } 317 | } 318 | 319 | @Override 320 | public long size() { 321 | return this.innerArray.size(); 322 | } 323 | 324 | @Override 325 | public void flush() { 326 | try { 327 | this.innerArray.arrayReadLock.lock(); 328 | 329 | for(QueueFront qf : this.queueFrontMap.values()) { 330 | try { 331 | qf.writeLock.lock(); 332 | qf.indexPageFactory.flush(); 333 | } finally { 334 | qf.writeLock.unlock(); 335 | } 336 | } 337 | innerArray.flush(); 338 | 339 | } finally { 340 | this.innerArray.arrayReadLock.unlock(); 341 | } 342 | } 343 | 344 | @Override 345 | public void close() throws IOException { 346 | try { 347 | this.innerArray.arrayWriteLock.lock(); 348 | 349 | for(QueueFront qf : this.queueFrontMap.values()) { 350 | qf.indexPageFactory.releaseCachedPages(); 351 | } 352 | 353 | innerArray.close(); 354 | } finally { 355 | this.innerArray.arrayWriteLock.unlock(); 356 | } 357 | } 358 | 359 | @Override 360 | public void removeAll() throws IOException { 361 | try { 362 | this.innerArray.arrayWriteLock.lock(); 363 | 364 | for(QueueFront qf : this.queueFrontMap.values()) { 365 | try { 366 | qf.writeLock.lock(); 367 | qf.index.set(0L); 368 | qf.persistIndex(); 369 | } finally { 370 | qf.writeLock.unlock(); 371 | } 372 | } 373 | innerArray.removeAll(); 374 | 375 | } finally { 376 | this.innerArray.arrayWriteLock.unlock(); 377 | } 378 | } 379 | 380 | // Queue front wrapper 381 | class QueueFront { 382 | 383 | // fanout index 384 | final String fanoutId; 385 | 386 | // front index of the fanout queue 387 | final AtomicLong index = new AtomicLong(); 388 | 389 | // factory for queue front index page management(acquire, release, cache) 390 | final IMappedPageFactory indexPageFactory; 391 | 392 | // lock for queue front write management 393 | final Lock writeLock = new ReentrantLock(); 394 | 395 | QueueFront(String fanoutId) throws IOException { 396 | try { 397 | FolderNameValidator.validate(fanoutId); 398 | } catch (IllegalArgumentException ex) { 399 | throw new IllegalArgumentException("invalid fanout identifier", ex); 400 | } 401 | this.fanoutId = fanoutId; 402 | // the ttl does not matter here since queue front index page is always cached 403 | this.indexPageFactory = new MappedPageFactoryImpl(QUEUE_FRONT_INDEX_PAGE_SIZE, 404 | innerArray.arrayDirectory + QUEUE_FRONT_INDEX_PAGE_FOLDER_PREFIX + fanoutId, 405 | 10 * 1000/*does not matter*/); 406 | 407 | IMappedPage indexPage = this.indexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 408 | 409 | ByteBuffer indexBuffer = indexPage.getLocal(0); 410 | index.set(indexBuffer.getLong()); 411 | validateAndAdjustIndex(); 412 | } 413 | 414 | void validateAndAdjustIndex() throws IOException { 415 | if (index.get() != innerArray.arrayHeadIndex.get()) { // ok that index is equal to array head index 416 | try { 417 | innerArray.validateIndex(index.get()); 418 | } catch (IndexOutOfBoundsException ex) { // maybe the back array has been truncated to limit size 419 | resetIndex(); 420 | } 421 | } 422 | } 423 | 424 | // reset queue front index to the tail of array 425 | void resetIndex() throws IOException { 426 | index.set(innerArray.arrayTailIndex.get()); 427 | 428 | this.persistIndex(); 429 | } 430 | 431 | void incrementIndex() throws IOException { 432 | long nextIndex = index.get(); 433 | if (nextIndex == Long.MAX_VALUE) { 434 | nextIndex = 0L; // wrap 435 | } else { 436 | nextIndex++; 437 | } 438 | index.set(nextIndex); 439 | 440 | this.persistIndex(); 441 | } 442 | 443 | void persistIndex() throws IOException { 444 | // persist index 445 | IMappedPage indexPage = this.indexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 446 | ByteBuffer indexBuffer = indexPage.getLocal(0); 447 | indexBuffer.putLong(index.get()); 448 | indexPage.setDirty(true); 449 | } 450 | } 451 | 452 | @Override 453 | public long getFrontIndex() { 454 | return this.innerArray.getTailIndex(); 455 | } 456 | 457 | @Override 458 | public long getRearIndex() { 459 | return this.innerArray.getHeadIndex(); 460 | } 461 | 462 | @Override 463 | public long getFrontIndex(String fanoutId) throws IOException { 464 | try { 465 | this.innerArray.arrayReadLock.lock(); 466 | 467 | QueueFront qf = this.getQueueFront(fanoutId); 468 | return qf.index.get(); 469 | 470 | } finally { 471 | this.innerArray.arrayReadLock.unlock(); 472 | } 473 | } 474 | } 475 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/FanOutQueueImplEx.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/src/com/leansoft/bigqueue/FanOutQueueImplEx.java -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/IBigArray.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | 6 | /** 7 | * Append Only Big Array ADT 8 | * 9 | * @author bulldog 10 | * 11 | */ 12 | public interface IBigArray extends Closeable { 13 | 14 | public static final long NOT_FOUND = -1; 15 | 16 | /** 17 | * Append the data into the head of the array 18 | * 19 | * @param data binary data to append 20 | * @return appended index 21 | * @throws IOException if there is any IO error 22 | */ 23 | long append(byte[] data) throws IOException; 24 | 25 | 26 | /** 27 | * Get the data at specific index 28 | * 29 | * @param index valid data index 30 | * @return binary data if the index is valid 31 | * @throws IOException if there is any IO error 32 | */ 33 | byte[] get(long index) throws IOException; 34 | 35 | /** 36 | * Get the timestamp of data at specific index, 37 | * 38 | * this is the timestamp when the data was appended. 39 | * 40 | * @param index valid data index 41 | * @return timestamp when the data was appended 42 | * @throws IOException if there is any IO error 43 | */ 44 | long getTimestamp(long index) throws IOException; 45 | 46 | /** 47 | * The total number of items has been appended into the array 48 | * 49 | * @return total number 50 | */ 51 | long size(); 52 | 53 | 54 | /** 55 | * Get the back data file size per page. 56 | * 57 | * @return size per page 58 | */ 59 | int getDataPageSize(); 60 | 61 | /** 62 | * The head of the array. 63 | * 64 | * This is the next to append index, the index of the last appended data 65 | * is [headIndex - 1] if the array is not empty. 66 | * 67 | * @return an index 68 | */ 69 | long getHeadIndex(); 70 | 71 | /** 72 | * The tail of the array. 73 | * 74 | * The is the index of the first appended data 75 | * 76 | * @return an index 77 | */ 78 | long getTailIndex(); 79 | 80 | /** 81 | * Check if the array is empty or not 82 | * 83 | * @return true if empty false otherwise 84 | */ 85 | boolean isEmpty(); 86 | 87 | /** 88 | * Check if the ring space of java long type has all been used up. 89 | * 90 | * can always assume false, if true, the world is end:) 91 | * 92 | * @return array full or not 93 | */ 94 | boolean isFull(); 95 | 96 | /** 97 | * Remove all data in this array, this will empty the array and delete all back page files. 98 | * 99 | */ 100 | void removeAll() throws IOException; 101 | 102 | /** 103 | * Remove all data before specific index, this will advance the array tail to index and 104 | * delete back page files before index. 105 | * 106 | * @param index an index 107 | * @throws IOException exception thrown if there was any IO error during the removal operation 108 | */ 109 | void removeBeforeIndex(long index) throws IOException; 110 | 111 | /** 112 | * Remove all data before specific timestamp, this will advance the array tail and delete back page files 113 | * accordingly. 114 | * 115 | * @param timestamp a timestamp 116 | * @throws IOException exception thrown if there was any IO error during the removal operation 117 | */ 118 | void removeBefore(long timestamp) throws IOException; 119 | 120 | /** 121 | * Force to persist newly appended data, 122 | * 123 | * normally, you don't need to flush explicitly since: 124 | * 1.) BigArray will automatically flush a cached page when it is replaced out, 125 | * 2.) BigArray uses memory mapped file technology internally, and the OS will flush the changes even your process crashes, 126 | * 127 | * call this periodically only if you need transactional reliability and you are aware of the cost to performance. 128 | */ 129 | void flush(); 130 | 131 | /** 132 | * Find an index closest to the specific timestamp when the corresponding item was appended 133 | * 134 | * @param timestamp when the corresponding item was appended 135 | * @return an index 136 | * @throws IOException exception thrown if there was any IO error during the getClosestIndex operation 137 | */ 138 | long findClosestIndex(long timestamp) throws IOException; 139 | 140 | 141 | /** 142 | * Get total size of back files(index and data files) of the big array 143 | * 144 | * @return total size of back files 145 | * @throws IOException exception thrown if there was any IO error during the getBackFileSize operation 146 | */ 147 | long getBackFileSize() throws IOException; 148 | 149 | /** 150 | * limit the back file size, truncate back file and advance array tail index accordingly, 151 | * Note, this is a best effort call, exact size limit can't be guaranteed 152 | * 153 | * @param sizeLimit the size to limit 154 | * @throws IOException exception thrown if there was any IO error during the limitBackFileSize operation 155 | */ 156 | void limitBackFileSize(long sizeLimit) throws IOException; 157 | 158 | 159 | /** 160 | * Get the data item length at specific index 161 | * 162 | * @param index valid data index 163 | * @return the length of binary data if the index is valid 164 | * @throws IOException if there is any IO error 165 | */ 166 | int getItemLength(long index) throws IOException; 167 | } 168 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/IBigQueue.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | 6 | /** 7 | * Queue ADT 8 | * 9 | * @author bulldog 10 | * 11 | */ 12 | public interface IBigQueue extends Closeable { 13 | 14 | /** 15 | * Determines whether a queue is empty 16 | * 17 | * @return ture if empty, false otherwise 18 | */ 19 | public boolean isEmpty(); 20 | 21 | /** 22 | * Adds an item at the back of a queue 23 | * 24 | * @param data to be enqueued data 25 | * @throws IOException exception throws if there is any IO error during enqueue operation. 26 | */ 27 | public void enqueue(byte[] data) throws IOException; 28 | 29 | /** 30 | * Retrieves and removes the front of a queue 31 | * 32 | * @return data at the front of a queue 33 | * @throws IOException exception throws if there is any IO error during dequeue operation. 34 | */ 35 | public byte[] dequeue() throws IOException; 36 | 37 | /** 38 | * Removes all items of a queue, this will empty the queue and delete all back data files. 39 | * 40 | * @throws IOException exception throws if there is any IO error during dequeue operation. 41 | */ 42 | public void removeAll() throws IOException; 43 | 44 | /** 45 | * Retrieves the item at the front of a queue 46 | * 47 | * @return data at the front of a queue 48 | * @throws IOException exception throws if there is any IO error during peek operation. 49 | */ 50 | public byte[] peek() throws IOException; 51 | 52 | /** 53 | * apply an implementation of a ItemIterator interface for each queue item 54 | * 55 | * @param iterator 56 | * @throws IOException 57 | */ 58 | public void applyForEach(ItemIterator iterator) throws IOException; 59 | 60 | /** 61 | * Delete all used data files to free disk space. 62 | * 63 | * BigQueue will persist enqueued data in disk files, these data files will remain even after 64 | * the data in them has been dequeued later, so your application is responsible to periodically call 65 | * this method to delete all used data files and free disk space. 66 | * 67 | * @throws IOException exception throws if there is any IO error during gc operation. 68 | */ 69 | public void gc() throws IOException; 70 | 71 | /** 72 | * Force to persist current state of the queue, 73 | * 74 | * normally, you don't need to flush explicitly since: 75 | * 1.) BigQueue will automatically flush a cached page when it is replaced out, 76 | * 2.) BigQueue uses memory mapped file technology internally, and the OS will flush the changes even your process crashes, 77 | * 78 | * call this periodically only if you need transactional reliability and you are aware of the cost to performance. 79 | */ 80 | public void flush(); 81 | 82 | /** 83 | * Total number of items available in the queue. 84 | * @return total number 85 | */ 86 | public long size(); 87 | 88 | /** 89 | * Item iterator interface 90 | */ 91 | public static interface ItemIterator { 92 | /** 93 | * Method to be executed for each queue item 94 | * 95 | * @param item queue item 96 | * @throws IOException 97 | */ 98 | public void forEach(byte[] item) throws IOException; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/IFanOutQueue.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | 6 | /** 7 | * FanOut queue ADT 8 | * 9 | * @author bulldog 10 | * 11 | */ 12 | public interface IFanOutQueue extends Closeable { 13 | 14 | /** 15 | * Constant represents earliest timestamp 16 | */ 17 | public static final long EARLIEST = -1; 18 | /** 19 | * Constant represents latest timestamp 20 | */ 21 | public static final long LATEST = -2; 22 | 23 | /** 24 | * Determines whether a fan out queue is empty 25 | * 26 | * @param fanoutId the fanout identifier 27 | * @return true if empty, false otherwise 28 | */ 29 | public boolean isEmpty(String fanoutId) throws IOException; 30 | 31 | /** 32 | * Determines whether the queue is empty 33 | * 34 | * @return true if empty, false otherwise 35 | */ 36 | public boolean isEmpty(); 37 | 38 | /** 39 | * Adds an item at the back of the queue 40 | * 41 | * @param data to be enqueued data 42 | * @return index where the item was appended 43 | * @throws IOException exception throws if there is any IO error during enqueue operation. 44 | */ 45 | public long enqueue(byte[] data) throws IOException; 46 | 47 | /** 48 | * Retrieves and removes the front of a fan out queue 49 | * 50 | * @param fanoutId the fanout identifier 51 | * @return data at the front of a queue 52 | * @throws IOException exception throws if there is any IO error during dequeue operation. 53 | */ 54 | public byte[] dequeue(String fanoutId) throws IOException; 55 | 56 | /** 57 | * Peek the item at the front of a fanout queue, without removing it from the queue 58 | * 59 | * @param fanoutId the fanout identifier 60 | * @return data at the front of a queue 61 | * @throws IOException exception throws if there is any IO error during peek operation. 62 | */ 63 | public byte[] peek(String fanoutId) throws IOException; 64 | 65 | 66 | /** 67 | * Peek the length of the item at the front of a fan out queue 68 | * 69 | * @param fanoutId the fanout identifier 70 | * @return data at the front of a queue 71 | * @throws IOException exception throws if there is any IO error during peek operation. 72 | */ 73 | public int peekLength(String fanoutId) throws IOException; 74 | 75 | /** 76 | * Peek the timestamp of the item at the front of a fan out queue 77 | * 78 | * @param fanoutId the fanout identifier 79 | * @return data at the front of a queue 80 | * @throws IOException exception throws if there is any IO error during peek operation. 81 | */ 82 | public long peekTimestamp(String fanoutId) throws IOException; 83 | 84 | /** 85 | * Retrieves data item at the specific index of the queue 86 | * 87 | * @param index data item index 88 | * @return data at index 89 | * @throws IOException exception throws if there is any IO error during fetch operation. 90 | */ 91 | public byte[] get(long index) throws IOException; 92 | 93 | 94 | /** 95 | * Get length of data item at specific index of the queue 96 | * 97 | * @param index data item index 98 | * @return length of data item 99 | * @throws IOException exception throws if there is any IO error during fetch operation. 100 | */ 101 | public int getLength(long index) throws IOException; 102 | 103 | /** 104 | * Get timestamp of data item at specific index of the queue, this is the timestamp when corresponding item was appended into the queue. 105 | * 106 | * @param index data item index 107 | * @return timestamp of data item 108 | * @throws IOException exception throws if there is any IO error during fetch operation. 109 | */ 110 | public long getTimestamp(long index) throws IOException; 111 | 112 | /** 113 | * Total number of items remaining in the fan out queue 114 | * 115 | * @param fanoutId the fanout identifier 116 | * @return total number 117 | */ 118 | public long size(String fanoutId) throws IOException ; 119 | 120 | 121 | /** 122 | * Total number of items remaining in the queue. 123 | * 124 | * @return total number 125 | */ 126 | public long size(); 127 | 128 | /** 129 | * Force to persist current state of the queue, 130 | * 131 | * normally, you don't need to flush explicitly since: 132 | * 1.) FanOutQueue will automatically flush a cached page when it is replaced out, 133 | * 2.) FanOutQueue uses memory mapped file technology internally, and the OS will flush the changes even your process crashes, 134 | * 135 | * call this periodically only if you need transactional reliability and you are aware of the cost to performance. 136 | */ 137 | public void flush(); 138 | 139 | /** 140 | * Remove all data before specific timestamp, truncate back files and advance the queue front if necessary. 141 | * 142 | * @param timestamp a timestamp 143 | * @throws IOException exception thrown if there was any IO error during the removal operation 144 | */ 145 | void removeBefore(long timestamp) throws IOException; 146 | 147 | /** 148 | * Limit the back file size of this queue, truncate back files and advance the queue front if necessary. 149 | * 150 | * Note, this is a best effort call, exact size limit can't be guaranteed 151 | * 152 | * @param sizeLmit size limit 153 | * @throws IOException exception thrown if there was any IO error during the operation 154 | */ 155 | void limitBackFileSize(long sizeLmit) throws IOException; 156 | 157 | /** 158 | * Current total size of the back files of this queue 159 | * 160 | * @return total back file size 161 | * @throws IOException exception thrown if there was any IO error during the operation 162 | */ 163 | long getBackFileSize() throws IOException; 164 | 165 | 166 | /** 167 | * Find an index closest to the specific timestamp when the corresponding item was enqueued. 168 | * to find latest index, use {@link #LATEST} as timestamp. 169 | * to find earliest index, use {@link #EARLIEST} as timestamp. 170 | * 171 | * @param timestamp when the corresponding item was appended 172 | * @return an index 173 | * @throws IOException exception thrown during the operation 174 | */ 175 | long findClosestIndex(long timestamp) throws IOException; 176 | 177 | /** 178 | * Reset the front index of a fanout queue. 179 | * 180 | * @param fandoutId fanout identifier 181 | * @param index target index 182 | * @throws IOException exception thrown during the operation 183 | */ 184 | void resetQueueFrontIndex(String fanoutId, long index) throws IOException; 185 | 186 | /** 187 | * Removes all items of a queue, this will empty the queue and delete all back data files. 188 | * 189 | * @throws IOException exception throws if there is any IO error during dequeue operation. 190 | */ 191 | public void removeAll() throws IOException; 192 | 193 | /** 194 | * Get the queue front index, this is the earliest appended index 195 | * 196 | * @return an index 197 | */ 198 | public long getFrontIndex(); 199 | 200 | /** 201 | * Get front index of specific fanout queue 202 | * 203 | * @param fanoutId fanout identifier 204 | * @return an index 205 | */ 206 | public long getFrontIndex(String fanoutId) throws IOException; 207 | 208 | /** 209 | * Get the queue rear index, this is the next to be appended index 210 | * 211 | * @return an index 212 | */ 213 | public long getRearIndex(); 214 | 215 | } 216 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/cache/ILRUCache.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.cache; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | import java.util.Collection; 6 | 7 | /** 8 | * LRU cache ADT 9 | * 10 | * @author bulldog 11 | * 12 | * @param the key 13 | * @param the value 14 | */ 15 | public interface ILRUCache { 16 | 17 | /** 18 | * Put a keyed resource with specific ttl into the cache 19 | * 20 | * This call will increment the reference counter of the keyed resource. 21 | * 22 | * @param key the key of the cached resource 23 | * @param value the to be cached resource 24 | * @param ttlInMilliSeconds time to live in milliseconds 25 | */ 26 | public void put(final K key, final V value, final long ttlInMilliSeconds); 27 | 28 | /** 29 | * Put a keyed resource with default ttl into the cache 30 | * 31 | * This call will increment the reference counter of the keyed resource. 32 | * 33 | * @param key the key of the cached resource 34 | * @param value the to be cached resource 35 | */ 36 | public void put(final K key, final V value); 37 | 38 | 39 | /** 40 | * Get a cached resource with specific key 41 | * 42 | * This call will increment the reference counter of the keyed resource. 43 | * 44 | * @param key the key of the cached resource 45 | * @return cached resource if exists 46 | */ 47 | public V get(final K key); 48 | 49 | /** 50 | * Release the cached resource with specific key 51 | * 52 | * This call will decrement the reference counter of the keyed resource. 53 | * 54 | * @param key 55 | */ 56 | public void release(final K key); 57 | 58 | /** 59 | * Remove the resource with specific key from the cache and close it synchronously afterwards. 60 | * 61 | * @param key the key of the cached resource 62 | * @return the removed resource if exists 63 | * @throws IOException exception thrown if there is any IO error 64 | */ 65 | public V remove(final K key) throws IOException; 66 | 67 | /** 68 | * Remove all cached resource from the cache and close them asynchronously afterwards. 69 | * 70 | * @throws IOException exception thrown if there is any IO error 71 | */ 72 | public void removeAll() throws IOException; 73 | 74 | /** 75 | * The size of the cache, equals to current total number of cached resources. 76 | * 77 | * @return the size of the cache 78 | */ 79 | public int size(); 80 | 81 | /** 82 | * All values cached 83 | * @return a collection 84 | */ 85 | public Collection getValues(); 86 | } 87 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/cache/LRUCacheImpl.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.cache; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.Collection; 7 | import java.util.HashMap; 8 | import java.util.HashSet; 9 | import java.util.Map; 10 | import java.util.Set; 11 | import java.util.concurrent.ExecutorService; 12 | import java.util.concurrent.Executors; 13 | import java.util.concurrent.atomic.AtomicLong; 14 | import java.util.concurrent.locks.Lock; 15 | import java.util.concurrent.locks.ReadWriteLock; 16 | import java.util.concurrent.locks.ReentrantReadWriteLock; 17 | 18 | import org.apache.log4j.Logger; 19 | 20 | /** 21 | * Simple and thread-safe LRU cache implementation, 22 | * supporting time to live and reference counting for entry. 23 | * 24 | * in current implementation, entry expiration and purge(mark&sweep) is triggered by put operation, 25 | * and resource closing after mark&sweep is done in async way. 26 | * 27 | * @author bulldog 28 | * 29 | * @param key 30 | * @param value 31 | */ 32 | public class LRUCacheImpl implements ILRUCache { 33 | 34 | private final static Logger logger = Logger.getLogger(LRUCacheImpl.class); 35 | 36 | public static final long DEFAULT_TTL = 10 * 1000; // milliseconds 37 | 38 | private final Map map; 39 | private final Map ttlMap; 40 | private final ReadWriteLock lock = new ReentrantReadWriteLock(); 41 | private final Lock readLock = lock.readLock(); 42 | private final Lock writeLock = lock.writeLock(); 43 | 44 | private static final ExecutorService executorService = Executors.newCachedThreadPool(); 45 | 46 | private final Set keysToRemove = new HashSet(); 47 | 48 | public LRUCacheImpl() { 49 | map = new HashMap(); 50 | ttlMap = new HashMap(); 51 | } 52 | 53 | public void put(K key, V value, long ttlInMilliSeconds) { 54 | Collection valuesToClose = null; 55 | try { 56 | writeLock.lock(); 57 | // trigger mark&sweep 58 | valuesToClose = markAndSweep(); 59 | if (valuesToClose != null && valuesToClose.contains(value)) { // just be cautious 60 | valuesToClose.remove(value); 61 | } 62 | map.put(key, value); 63 | TTLValue ttl = new TTLValue(System.currentTimeMillis(), ttlInMilliSeconds); 64 | ttl.refCount.incrementAndGet(); 65 | ttlMap.put(key, ttl); 66 | } finally { 67 | writeLock.unlock(); 68 | } 69 | if (valuesToClose != null && valuesToClose.size() > 0) { 70 | if (logger.isDebugEnabled()) { 71 | int size = valuesToClose.size(); 72 | logger.info("Mark&Sweep found " + size + (size > 1 ? " resources":" resource") + " to close."); 73 | } 74 | // close resource asynchronously 75 | executorService.execute(new ValueCloser(valuesToClose)); 76 | } 77 | } 78 | 79 | public void put(K key, V value) { 80 | this.put(key, value, DEFAULT_TTL); 81 | } 82 | 83 | /** 84 | * A lazy mark and sweep, 85 | * 86 | * a separate thread can also do this. 87 | */ 88 | private Collection markAndSweep() { 89 | Collection valuesToClose = null; 90 | keysToRemove.clear(); 91 | Set keys = ttlMap.keySet(); 92 | long currentTS = System.currentTimeMillis(); 93 | for(K key: keys) { 94 | TTLValue ttl = ttlMap.get(key); 95 | if (ttl.refCount.get() <= 0 && (currentTS - ttl.lastAccessedTimestamp.get()) > ttl.ttl) { // remove object with no reference and expired 96 | keysToRemove.add(key); 97 | } 98 | } 99 | 100 | if (keysToRemove.size() > 0) { 101 | valuesToClose = new HashSet(); 102 | for(K key : keysToRemove) { 103 | V v = map.remove(key); 104 | valuesToClose.add(v); 105 | ttlMap.remove(key); 106 | } 107 | } 108 | 109 | return valuesToClose; 110 | } 111 | 112 | public V get(K key) { 113 | try { 114 | readLock.lock(); 115 | TTLValue ttl = ttlMap.get(key); 116 | if (ttl != null) { 117 | // Since the resource is acquired by calling thread, 118 | // let's update last accessed timestamp and increment reference counting 119 | ttl.lastAccessedTimestamp.set(System.currentTimeMillis()); 120 | ttl.refCount.incrementAndGet(); 121 | } 122 | return map.get(key); 123 | } finally { 124 | readLock.unlock(); 125 | } 126 | } 127 | 128 | private static class TTLValue { 129 | AtomicLong lastAccessedTimestamp; // last accessed time 130 | AtomicLong refCount = new AtomicLong(0); 131 | long ttl; 132 | 133 | public TTLValue(long ts, long ttl) { 134 | this.lastAccessedTimestamp = new AtomicLong(ts); 135 | this.ttl = ttl; 136 | } 137 | } 138 | 139 | private static class ValueCloser implements Runnable { 140 | Collection valuesToClose; 141 | 142 | public ValueCloser(Collection valuesToClose) { 143 | this.valuesToClose = valuesToClose; 144 | } 145 | 146 | public void run() { 147 | int size = valuesToClose.size(); 148 | for(V v : valuesToClose) { 149 | try { 150 | if (v != null) { 151 | v.close(); 152 | } 153 | } catch (IOException e) { 154 | // close quietly 155 | } 156 | } 157 | if (logger.isDebugEnabled()) { 158 | logger.debug("ResourceCloser closed " + size + (size > 1 ? " resources.":" resource.")); 159 | } 160 | } 161 | } 162 | 163 | public void release(K key) { 164 | try { 165 | readLock.lock(); 166 | TTLValue ttl = ttlMap.get(key); 167 | if (ttl != null) { 168 | // since the resource is released by calling thread 169 | // let's decrement the reference counting 170 | ttl.refCount.decrementAndGet(); 171 | } 172 | } finally { 173 | readLock.unlock(); 174 | } 175 | } 176 | 177 | public int size() { 178 | try { 179 | readLock.lock(); 180 | return map.size(); 181 | } finally { 182 | readLock.unlock(); 183 | } 184 | } 185 | 186 | @Override 187 | public void removeAll() throws IOException { 188 | try { 189 | writeLock.lock(); 190 | 191 | Collection valuesToClose = new HashSet(); 192 | valuesToClose.addAll(map.values()); 193 | 194 | if (valuesToClose != null && valuesToClose.size() > 0) { 195 | // close resource synchronously 196 | for(V v : valuesToClose) { 197 | v.close(); 198 | } 199 | } 200 | map.clear(); 201 | ttlMap.clear(); 202 | 203 | } finally { 204 | writeLock.unlock(); 205 | } 206 | 207 | } 208 | 209 | @Override 210 | public V remove(K key) throws IOException { 211 | try { 212 | writeLock.lock(); 213 | ttlMap.remove(key); 214 | V value = map.remove(key); 215 | if (value != null) { 216 | // close synchronously 217 | value.close(); 218 | } 219 | return value; 220 | } finally { 221 | writeLock.unlock(); 222 | } 223 | 224 | } 225 | 226 | @Override 227 | public Collection getValues() { 228 | try { 229 | readLock.lock(); 230 | Collection col = new ArrayList(); 231 | for(V v : map.values()) { 232 | col.add(v); 233 | } 234 | return col; 235 | } finally { 236 | readLock.unlock(); 237 | } 238 | } 239 | 240 | } 241 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/page/IMappedPage.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.page; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | /** 6 | * Memory mapped page file ADT 7 | * 8 | * @author bulldog 9 | * 10 | */ 11 | public interface IMappedPage { 12 | 13 | /** 14 | * Get a thread local copy of the mapped page buffer 15 | * 16 | * @param position start position(relative to the start position of source mapped page buffer) of the thread local buffer 17 | * @return a byte buffer with specific position as start position. 18 | */ 19 | ByteBuffer getLocal(int position); 20 | 21 | /** 22 | * Get data from a thread local copy of the mapped page buffer 23 | * 24 | * @param position start position(relative to the start position of source mapped page buffer) of the thread local buffer 25 | * @param length the length to fetch 26 | * @return byte data 27 | */ 28 | public byte[] getLocal(int position, int length); 29 | 30 | /** 31 | * Check if this mapped page has been closed or not 32 | * 33 | * @return 34 | */ 35 | boolean isClosed(); 36 | 37 | /** 38 | * Set if the mapped page has been changed or not 39 | * 40 | * @param dirty 41 | */ 42 | void setDirty(boolean dirty); 43 | 44 | /** 45 | * The back page file name of the mapped page 46 | * 47 | * @return 48 | */ 49 | String getPageFile(); 50 | 51 | /** 52 | * The index of the mapped page 53 | * 54 | * @return the index 55 | */ 56 | long getPageIndex(); 57 | 58 | /** 59 | * Persist any changes to disk 60 | */ 61 | public void flush(); 62 | } 63 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/page/IMappedPageFactory.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.page; 2 | 3 | import java.io.IOException; 4 | import java.util.Set; 5 | 6 | /** 7 | * Memory mapped page management ADT 8 | * 9 | * @author bulldog 10 | * 11 | */ 12 | public interface IMappedPageFactory { 13 | 14 | /** 15 | * Acquire a mapped page with specific index from the factory 16 | * 17 | * @param index the index of the page 18 | * @return a mapped page 19 | * @throws IOException exception thrown if there was any IO error during the acquire operation 20 | */ 21 | IMappedPage acquirePage(long index) throws IOException; 22 | 23 | /** 24 | * Return the mapped page to the factory, 25 | * calling thread release the page to inform the factory that it has finished with the page, 26 | * so the factory get a chance to recycle the page to save memory. 27 | * 28 | * @param index the index of the page 29 | */ 30 | void releasePage(long index); 31 | 32 | /** 33 | * Current set page size, when creating pages, the factory will 34 | * only create pages with this size. 35 | * 36 | * @return an integer number 37 | */ 38 | int getPageSize(); 39 | 40 | /** 41 | * Current set page directory. 42 | * 43 | * @return 44 | */ 45 | String getPageDir(); 46 | 47 | /** 48 | * delete a mapped page with specific index in this factory, 49 | * this call will remove the page from the cache if it is cached and 50 | * delete back file. 51 | * 52 | * @param index the index of the page 53 | * @throws IOException exception thrown if there was any IO error during the delete operation. 54 | */ 55 | void deletePage(long index) throws IOException; 56 | 57 | /** 58 | * delete mapped pages with a set of specific indexes in this factory, 59 | * this call will remove the pages from the cache if they ware cached and 60 | * delete back files. 61 | * 62 | * @param indexes the indexes of the pages 63 | * @throws IOException 64 | */ 65 | void deletePages(Set indexes) throws IOException; 66 | 67 | /** 68 | * delete all mapped pages currently available in this factory, 69 | * this call will remove all pages from the cache and delete all back files. 70 | * 71 | * @throws IOException exception thrown if there was any IO error during the delete operation. 72 | */ 73 | void deleteAllPages() throws IOException; 74 | 75 | /** 76 | * remove all cached pages from the cache and close resources associated with the cached pages. 77 | * 78 | * @throws IOException exception thrown if there was any IO error during the release operation. 79 | */ 80 | void releaseCachedPages() throws IOException; 81 | 82 | /** 83 | * Get all indexes of pages with last modified timestamp before the specific timestamp. 84 | * 85 | * @param timestamp the timestamp to check 86 | * @return a set of indexes 87 | */ 88 | Set getPageIndexSetBefore(long timestamp); 89 | 90 | /** 91 | * Delete all pages with last modified timestamp before the specific timestamp. 92 | * 93 | * @param timestamp the timestamp to check 94 | * @throws IOException exception thrown if there was any IO error during the delete operation. 95 | */ 96 | void deletePagesBefore(long timestamp) throws IOException; 97 | 98 | /** 99 | * Delete all pages before the specific index 100 | * 101 | * @param pageIndex page file index to check 102 | * @throws IOException exception thrown if there was any IO error during the delete operation. 103 | */ 104 | void deletePagesBeforePageIndex(long pageIndex) throws IOException; 105 | 106 | /** 107 | * Get last modified timestamp of page file index 108 | * 109 | * @param index page index 110 | */ 111 | long getPageFileLastModifiedTime(long index); 112 | 113 | /** 114 | * Get index of a page file with last modified timestamp closest to specific timestamp. 115 | * 116 | * @param timestamp the timestamp to check 117 | * @return a page index 118 | */ 119 | long getFirstPageIndexBefore(long timestamp); 120 | 121 | /** 122 | * For test, get a list of indexes of current existing back files. 123 | * 124 | * @return a set of indexes 125 | */ 126 | Set getExistingBackFileIndexSet(); 127 | 128 | /** 129 | * For test, get current cache size 130 | * 131 | * @return an integer number 132 | */ 133 | int getCacheSize(); 134 | 135 | /** 136 | * Persist any changes in cached mapped pages 137 | */ 138 | void flush(); 139 | 140 | /** 141 | * 142 | * A set of back page file names 143 | * 144 | * @return file name set 145 | */ 146 | Set getBackPageFileSet(); 147 | 148 | 149 | /** 150 | * Total size of all page files 151 | * 152 | * @return total size 153 | */ 154 | long getBackPageFileSize(); 155 | 156 | } 157 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/page/MappedPageFactoryImpl.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.page; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.io.RandomAccessFile; 6 | import java.nio.MappedByteBuffer; 7 | import java.nio.channels.FileChannel; 8 | import java.util.Collection; 9 | import java.util.HashMap; 10 | import java.util.HashSet; 11 | import java.util.Map; 12 | import java.util.Set; 13 | import java.util.TreeSet; 14 | 15 | import org.apache.log4j.Logger; 16 | 17 | import com.leansoft.bigqueue.cache.ILRUCache; 18 | import com.leansoft.bigqueue.cache.LRUCacheImpl; 19 | import com.leansoft.bigqueue.utils.FileUtil; 20 | 21 | import static java.nio.channels.FileChannel.MapMode.READ_WRITE; 22 | 23 | /** 24 | * Mapped mapped page resource manager, 25 | * responsible for the creation, cache, recycle of the mapped pages. 26 | * 27 | * automatic paging and swapping algorithm is leveraged to ensure fast page fetch while 28 | * keep memory usage efficient at the same time. 29 | * 30 | * @author bulldog 31 | * 32 | */ 33 | public class MappedPageFactoryImpl implements IMappedPageFactory { 34 | 35 | private final static Logger logger = Logger.getLogger(MappedPageFactoryImpl.class); 36 | 37 | private int pageSize; 38 | private String pageDir; 39 | private File pageDirFile; 40 | private String pageFile; 41 | private long ttl; 42 | 43 | private final Object mapLock = new Object(); 44 | private final Map pageCreationLockMap = new HashMap(); 45 | 46 | public static final String PAGE_FILE_NAME = "page"; 47 | public static final String PAGE_FILE_SUFFIX = ".dat"; 48 | 49 | private ILRUCache cache; 50 | 51 | public MappedPageFactoryImpl(int pageSize, String pageDir, long cacheTTL) { 52 | this.pageSize = pageSize; 53 | this.pageDir = pageDir; 54 | this.ttl = cacheTTL; 55 | this.pageDirFile = new File(this.pageDir); 56 | if (!pageDirFile.exists()) { 57 | pageDirFile.mkdirs(); 58 | } 59 | if (!this.pageDir.endsWith(File.separator)) { 60 | this.pageDir += File.separator; 61 | } 62 | this.pageFile = this.pageDir + PAGE_FILE_NAME + "-"; 63 | this.cache = new LRUCacheImpl(); 64 | } 65 | 66 | public IMappedPage acquirePage(long index) throws IOException { 67 | MappedPageImpl mpi = cache.get(index); 68 | if (mpi == null) { // not in cache, need to create one 69 | try { 70 | Object lock = null; 71 | synchronized(mapLock) { 72 | if (!pageCreationLockMap.containsKey(index)) { 73 | pageCreationLockMap.put(index, new Object()); 74 | } 75 | lock = pageCreationLockMap.get(index); 76 | } 77 | synchronized(lock) { // only lock the creation of page index 78 | mpi = cache.get(index); // double check 79 | if (mpi == null) { 80 | RandomAccessFile raf = null; 81 | FileChannel channel = null; 82 | try { 83 | String fileName = this.getFileNameByIndex(index); 84 | raf = new RandomAccessFile(fileName, "rw"); 85 | channel = raf.getChannel(); 86 | MappedByteBuffer mbb = channel.map(READ_WRITE, 0, this.pageSize); 87 | mpi = new MappedPageImpl(mbb, fileName, index); 88 | cache.put(index, mpi, ttl); 89 | if (logger.isDebugEnabled()) { 90 | logger.debug("Mapped page for " + fileName + " was just created and cached."); 91 | } 92 | } finally { 93 | if (channel != null) channel.close(); 94 | if (raf != null) raf.close(); 95 | } 96 | } 97 | } 98 | } finally { 99 | synchronized(mapLock) { 100 | pageCreationLockMap.remove(index); 101 | } 102 | } 103 | } else { 104 | if (logger.isDebugEnabled()) { 105 | logger.debug("Hit mapped page " + mpi.getPageFile() + " in cache."); 106 | } 107 | } 108 | 109 | return mpi; 110 | } 111 | 112 | private String getFileNameByIndex(long index) { 113 | return this.pageFile + index + PAGE_FILE_SUFFIX; 114 | } 115 | 116 | 117 | public int getPageSize() { 118 | return pageSize; 119 | } 120 | 121 | public String getPageDir() { 122 | return pageDir; 123 | } 124 | 125 | public void releasePage(long index) { 126 | cache.release(index); 127 | } 128 | 129 | /** 130 | * thread unsafe, caller need synchronization 131 | */ 132 | @Override 133 | public void releaseCachedPages() throws IOException { 134 | cache.removeAll(); 135 | } 136 | 137 | /** 138 | * thread unsafe, caller need synchronization 139 | */ 140 | @Override 141 | public void deleteAllPages() throws IOException { 142 | cache.removeAll(); 143 | Set indexSet = getExistingBackFileIndexSet(); 144 | this.deletePages(indexSet); 145 | if (logger.isDebugEnabled()) { 146 | logger.debug("All page files in dir " + this.pageDir + " have been deleted."); 147 | } 148 | } 149 | 150 | /** 151 | * thread unsafe, caller need synchronization 152 | */ 153 | @Override 154 | public void deletePages(Set indexes) throws IOException { 155 | if (indexes == null) return; 156 | for(long index : indexes) { 157 | this.deletePage(index); 158 | } 159 | } 160 | 161 | /** 162 | * thread unsafe, caller need synchronization 163 | */ 164 | @Override 165 | public void deletePage(long index) throws IOException { 166 | // remove the page from cache first 167 | cache.remove(index); 168 | String fileName = this.getFileNameByIndex(index); 169 | int count = 0; 170 | int maxRound = 10; 171 | boolean deleted = false; 172 | while(count < maxRound) { 173 | try { 174 | FileUtil.deleteFile(new File(fileName)); 175 | deleted = true; 176 | break; 177 | } catch (IllegalStateException ex) { 178 | try { 179 | Thread.sleep(200); 180 | } catch (InterruptedException e) { 181 | } 182 | count++; 183 | if (logger.isDebugEnabled()) { 184 | logger.warn("fail to delete file " + fileName + ", tried round = " + count); 185 | } 186 | } 187 | } 188 | if (deleted) { 189 | logger.info("Page file " + fileName + " was just deleted."); 190 | } else { 191 | logger.warn("fail to delete file " + fileName + " after max " + maxRound + " rounds of try, you may delete it manually."); 192 | } 193 | } 194 | 195 | @Override 196 | public Set getPageIndexSetBefore(long timestamp) { 197 | Set beforeIndexSet = new HashSet(); 198 | File[] pageFiles = this.pageDirFile.listFiles(); 199 | if (pageFiles != null && pageFiles.length > 0) { 200 | for(File pageFile : pageFiles) { 201 | if (pageFile.lastModified() < timestamp) { 202 | String fileName = pageFile.getName(); 203 | if (fileName.endsWith(PAGE_FILE_SUFFIX)) { 204 | long index = this.getIndexByFileName(fileName); 205 | beforeIndexSet.add(index); 206 | } 207 | } 208 | } 209 | } 210 | return beforeIndexSet; 211 | } 212 | 213 | private long getIndexByFileName(String fileName) { 214 | int beginIndex = fileName.lastIndexOf('-'); 215 | beginIndex += 1; 216 | int endIndex = fileName.lastIndexOf(PAGE_FILE_SUFFIX); 217 | String sIndex = fileName.substring(beginIndex, endIndex); 218 | long index = Long.parseLong(sIndex); 219 | return index; 220 | } 221 | 222 | /** 223 | * thread unsafe, caller need synchronization 224 | * @throws IOException 225 | */ 226 | @Override 227 | public void deletePagesBefore(long timestamp) throws IOException { 228 | Set indexSet = this.getPageIndexSetBefore(timestamp); 229 | this.deletePages(indexSet); 230 | if (logger.isDebugEnabled()) { 231 | logger.debug("All page files in dir [" + this.pageDir + "], before [" + timestamp + "] have been deleted."); 232 | } 233 | } 234 | 235 | @Override 236 | public void deletePagesBeforePageIndex(long pageIndex) throws IOException { 237 | Set indexSet = this.getExistingBackFileIndexSet(); 238 | for (Long index : indexSet) { 239 | if (index < pageIndex) { 240 | this.deletePage(index); 241 | } 242 | } 243 | } 244 | 245 | 246 | @Override 247 | public Set getExistingBackFileIndexSet() { 248 | Set indexSet = new HashSet(); 249 | File[] pageFiles = this.pageDirFile.listFiles(); 250 | if (pageFiles != null && pageFiles.length > 0) { 251 | for(File pageFile : pageFiles) { 252 | String fileName = pageFile.getName(); 253 | if (fileName.endsWith(PAGE_FILE_SUFFIX)) { 254 | long index = this.getIndexByFileName(fileName); 255 | indexSet.add(index); 256 | } 257 | } 258 | } 259 | return indexSet; 260 | } 261 | 262 | @Override 263 | public int getCacheSize() { 264 | return cache.size(); 265 | } 266 | 267 | // for testing 268 | int getLockMapSize() { 269 | return this.pageCreationLockMap.size(); 270 | } 271 | 272 | @Override 273 | public long getPageFileLastModifiedTime(long index) { 274 | String pageFileName = this.getFileNameByIndex(index); 275 | File pageFile = new File(pageFileName); 276 | if (!pageFile.exists()) { 277 | return -1L; 278 | } 279 | return pageFile.lastModified(); 280 | } 281 | 282 | @Override 283 | public long getFirstPageIndexBefore(long timestamp) { 284 | Set beforeIndexSet = getPageIndexSetBefore(timestamp); 285 | if (beforeIndexSet.size() == 0) return -1L; 286 | TreeSet sortedIndexSet = new TreeSet(beforeIndexSet); 287 | Long largestIndex = sortedIndexSet.last(); 288 | if (largestIndex != Long.MAX_VALUE) { // no wrap, just return the largest 289 | return largestIndex; 290 | } else { // wrapped case 291 | Long next = 0L; 292 | while(sortedIndexSet.contains(next)) { 293 | next++; 294 | } 295 | if (next == 0L) { 296 | return Long.MAX_VALUE; 297 | } else { 298 | return --next; 299 | } 300 | } 301 | } 302 | 303 | /** 304 | * thread unsafe, caller need synchronization 305 | */ 306 | @Override 307 | public void flush() { 308 | Collection cachedPages = cache.getValues(); 309 | for(IMappedPage mappedPage : cachedPages) { 310 | mappedPage.flush(); 311 | } 312 | } 313 | 314 | @Override 315 | public Set getBackPageFileSet() { 316 | Set fileSet = new HashSet(); 317 | File[] pageFiles = this.pageDirFile.listFiles(); 318 | if (pageFiles != null && pageFiles.length > 0) { 319 | for(File pageFile : pageFiles) { 320 | String fileName = pageFile.getName(); 321 | if (fileName.endsWith(PAGE_FILE_SUFFIX)) { 322 | fileSet.add(fileName); 323 | } 324 | } 325 | } 326 | return fileSet; 327 | } 328 | 329 | @Override 330 | public long getBackPageFileSize() { 331 | long totalSize = 0L; 332 | File[] pageFiles = this.pageDirFile.listFiles(); 333 | if (pageFiles != null && pageFiles.length > 0) { 334 | for(File pageFile : pageFiles) { 335 | String fileName = pageFile.getName(); 336 | if (fileName.endsWith(PAGE_FILE_SUFFIX)) { 337 | totalSize += pageFile.length(); 338 | } 339 | } 340 | } 341 | return totalSize; 342 | } 343 | 344 | } 345 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/page/MappedPageImpl.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.page; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | import java.lang.reflect.Field; 6 | import java.lang.reflect.Method; 7 | import java.nio.Buffer; 8 | import java.nio.ByteBuffer; 9 | import java.nio.MappedByteBuffer; 10 | 11 | import org.apache.log4j.Logger; 12 | 13 | import sun.misc.Unsafe; 14 | 15 | public class MappedPageImpl implements IMappedPage, Closeable { 16 | 17 | private final static Logger logger = Logger.getLogger(MappedPageImpl.class); 18 | 19 | private ThreadLocalByteBuffer threadLocalBuffer; 20 | private volatile boolean dirty = false; 21 | private volatile boolean closed = false; 22 | private String pageFile; 23 | private long index; 24 | 25 | public MappedPageImpl(MappedByteBuffer mbb, String pageFile, long index) { 26 | this.threadLocalBuffer = new ThreadLocalByteBuffer(mbb); 27 | this.pageFile = pageFile; 28 | this.index = index; 29 | } 30 | 31 | public void close() throws IOException { 32 | synchronized (this) { 33 | if (closed) 34 | return; 35 | 36 | flush(); 37 | 38 | MappedByteBuffer srcBuf = (MappedByteBuffer) threadLocalBuffer.getSourceBuffer(); 39 | unmap(srcBuf); 40 | 41 | this.threadLocalBuffer = null; // hint GC 42 | 43 | closed = true; 44 | if (logger.isDebugEnabled()) { 45 | logger.debug("Mapped page for " + this.pageFile + " was just unmapped and closed."); 46 | } 47 | } 48 | } 49 | 50 | @Override 51 | public void setDirty(boolean dirty) { 52 | this.dirty = dirty; 53 | } 54 | 55 | @Override 56 | public void flush() { 57 | synchronized (this) { 58 | if (closed) 59 | return; 60 | if (dirty) { 61 | MappedByteBuffer srcBuf = (MappedByteBuffer) threadLocalBuffer.getSourceBuffer(); 62 | srcBuf.force(); // flush the changes 63 | dirty = false; 64 | if (logger.isDebugEnabled()) { 65 | logger.debug("Mapped page for " + this.pageFile + " was just flushed."); 66 | } 67 | } 68 | } 69 | } 70 | 71 | public byte[] getLocal(int position, int length) { 72 | ByteBuffer buf = this.getLocal(position); 73 | byte[] data = new byte[length]; 74 | buf.get(data); 75 | return data; 76 | } 77 | 78 | @Override 79 | public ByteBuffer getLocal(int position) { 80 | ByteBuffer buf = this.threadLocalBuffer.get(); 81 | ((Buffer) buf).position(position); 82 | return buf; 83 | } 84 | 85 | private static void unmap(MappedByteBuffer buffer) { 86 | Cleaner.clean(buffer); 87 | } 88 | 89 | /** 90 | * Helper class allowing to clean direct buffers. 91 | */ 92 | private static class Cleaner { 93 | //for JDK8 94 | public static final boolean CLEAN_SUPPORTED; 95 | private static final Method directBufferCleaner; 96 | private static final Method directBufferCleanerClean; 97 | static { 98 | Method directBufferCleanerX = null; 99 | Method directBufferCleanerCleanX = null; 100 | boolean v; 101 | try { 102 | directBufferCleanerX = Class.forName("java.nio.DirectByteBuffer").getMethod("cleaner"); 103 | directBufferCleanerX.setAccessible(true); 104 | directBufferCleanerCleanX = Class.forName("sun.misc.Cleaner").getMethod("clean"); 105 | directBufferCleanerCleanX.setAccessible(true); 106 | v = true; 107 | } catch (Exception e) { 108 | v = false; 109 | } 110 | CLEAN_SUPPORTED = v; 111 | directBufferCleaner = directBufferCleanerX; 112 | directBufferCleanerClean = directBufferCleanerCleanX; 113 | } 114 | 115 | //for JDK11 116 | private static Unsafe unsafe; 117 | static { 118 | try { 119 | Field f = Unsafe.class.getDeclaredField("theUnsafe"); 120 | f.setAccessible(true); 121 | unsafe = (Unsafe) f.get(null); 122 | 123 | Unsafe.class.getDeclaredMethod("invokeCleaner", ByteBuffer.class); 124 | } catch (Exception e) { 125 | unsafe = null; 126 | logger.warn("Unsafe Not support " + e.getMessage()); 127 | } 128 | } 129 | 130 | public static void clean(ByteBuffer buffer) { 131 | if (buffer == null) { 132 | return; 133 | } 134 | 135 | if (buffer.isDirect()) { 136 | if (unsafe != null) { //JDK 11 137 | unsafe.invokeCleaner(buffer); 138 | } else { //JDK 8 139 | if (CLEAN_SUPPORTED && buffer.isDirect()) { 140 | try { 141 | Object cleaner = directBufferCleaner.invoke(buffer); 142 | directBufferCleanerClean.invoke(cleaner); 143 | } catch (Exception e) { 144 | // silently ignore exception 145 | } 146 | } 147 | } 148 | } 149 | } 150 | } 151 | 152 | private static class ThreadLocalByteBuffer extends ThreadLocal { 153 | private ByteBuffer _src; 154 | 155 | public ThreadLocalByteBuffer(ByteBuffer src) { 156 | _src = src; 157 | } 158 | 159 | public ByteBuffer getSourceBuffer() { 160 | return _src; 161 | } 162 | 163 | @Override 164 | protected synchronized ByteBuffer initialValue() { 165 | ByteBuffer dup = _src.duplicate(); 166 | return dup; 167 | } 168 | } 169 | 170 | @Override 171 | public boolean isClosed() { 172 | return closed; 173 | } 174 | 175 | public String toString() { 176 | return "Mapped page for " + this.pageFile + ", index = " + this.index + "."; 177 | } 178 | 179 | @Override 180 | public String getPageFile() { 181 | return this.pageFile; 182 | } 183 | 184 | @Override 185 | public long getPageIndex() { 186 | return this.index; 187 | } 188 | } -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/utils/Calculator.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.utils; 2 | 3 | public class Calculator { 4 | 5 | 6 | /** 7 | * mod by shift 8 | * 9 | * @param val 10 | * @param bits 11 | * @return 12 | */ 13 | public static long mod(long val, int bits) { 14 | return val - ((val >> bits) << bits); 15 | } 16 | 17 | /** 18 | * multiply by shift 19 | * 20 | * @param val 21 | * @param bits 22 | * @return 23 | */ 24 | public static long mul(long val, int bits) { 25 | return val << bits; 26 | } 27 | 28 | /** 29 | * divide by shift 30 | * 31 | * @param val 32 | * @param bits 33 | * @return 34 | */ 35 | public static long div(long val, int bits) { 36 | return val >> bits; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/utils/FileUtil.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.utils; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | public class FileUtil { 7 | 8 | /** 9 | * Only check if a given filename is valid according to the OS rules. 10 | * 11 | * You still need to handle other failures when actually creating 12 | * the file (e.g. insufficient permissions, lack of drive space, security restrictions). 13 | * @param file the name of a file 14 | * @return true if the file is valid, false otherwise 15 | */ 16 | public static boolean isFilenameValid(String file) { 17 | File f = new File(file); 18 | try { 19 | f.getCanonicalPath(); 20 | return true; 21 | } catch (IOException e) { 22 | return false; 23 | } 24 | } 25 | 26 | public static void deleteDirectory(File dir) { 27 | if (!dir.exists()) return; 28 | File[] subs = dir.listFiles(); 29 | if (subs != null) { 30 | for (File f : dir.listFiles()) { 31 | if (f.isFile()) { 32 | if(!f.delete()) { 33 | throw new IllegalStateException("delete file failed: "+f); 34 | } 35 | } else { 36 | deleteDirectory(f); 37 | } 38 | } 39 | } 40 | if(!dir.delete()) { 41 | throw new IllegalStateException("delete directory failed: "+dir); 42 | } 43 | } 44 | 45 | public static void deleteFile(File file) { 46 | if (!file.exists() || !file.isFile()) { 47 | return; 48 | } 49 | if (!file.delete()) { 50 | throw new IllegalStateException("delete file failed: "+file); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/com/leansoft/bigqueue/utils/FolderNameValidator.java: -------------------------------------------------------------------------------- 1 | package com.leansoft.bigqueue.utils; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | public class FolderNameValidator { 6 | 7 | private static final String illegalChars = "/" + '\u0000' + '\u0001' + "-" + '\u001F' + '\u007F' + "-" + '\u009F' + '\uD800' + "-" + '\uF8FF' + '\uFFF0' 8 | + "-" + '\uFFFF'; 9 | private static final Pattern p = Pattern.compile("(^\\.{1,2}$)|[" + illegalChars + "]"); 10 | 11 | public static void validate(String name) { 12 | if (name == null || name.length() == 0) { 13 | throw new IllegalArgumentException("folder name is emtpy"); 14 | } 15 | if(name.length() > 255) { 16 | throw new IllegalArgumentException("folder name is too long"); 17 | } 18 | if (p.matcher(name).find()) { 19 | throw new IllegalArgumentException("folder name [" + name + "] is illegal"); 20 | } 21 | } 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/wjw/psqueue/msg/ResAdd.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.msg; 2 | 3 | import java.beans.ConstructorProperties; 4 | import java.io.Serializable; 5 | 6 | public class ResAdd implements Serializable { 7 | public ResultCode status; 8 | public long idx; 9 | 10 | public ResAdd() { 11 | 12 | } 13 | 14 | @ConstructorProperties({ "status" }) 15 | public ResAdd(ResultCode status) { 16 | this.status = status; 17 | this.idx = -1; 18 | } 19 | 20 | @ConstructorProperties({ "status", "idx" }) 21 | public ResAdd(ResultCode status, long idx) { 22 | this.status = status; 23 | this.idx = idx; 24 | } 25 | 26 | public ResultCode getStatus() { 27 | return status; 28 | } 29 | 30 | public long getIdx() { 31 | return idx; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | StringBuilder builder = new StringBuilder(); 37 | builder.append("ResAdd [status=").append(status).append(", idx=").append(idx).append("]"); 38 | return builder.toString(); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/wjw/psqueue/msg/ResData.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.msg; 2 | 3 | import java.beans.ConstructorProperties; 4 | import java.io.Serializable; 5 | 6 | public class ResData implements Serializable { 7 | public ResultCode status; 8 | public String data; 9 | 10 | public ResData() { 11 | 12 | } 13 | 14 | @ConstructorProperties({ "status" }) 15 | public ResData(ResultCode status) { 16 | this.status = status; 17 | this.data = ""; 18 | } 19 | 20 | @ConstructorProperties({ "status", "data" }) 21 | public ResData(ResultCode status, String data) { 22 | this.status = status; 23 | this.data = data; 24 | } 25 | 26 | public ResultCode getStatus() { 27 | return status; 28 | } 29 | 30 | public String getData() { 31 | return data; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | StringBuilder builder = new StringBuilder(); 37 | builder.append("ResData [status=").append(status).append(", data=").append(data).append("]"); 38 | return builder.toString(); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/wjw/psqueue/msg/ResList.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.msg; 2 | 3 | import java.beans.ConstructorProperties; 4 | import java.io.Serializable; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | public class ResList implements Serializable { 9 | public ResultCode status; 10 | public List data; 11 | 12 | public ResList() { 13 | 14 | } 15 | 16 | @ConstructorProperties({ "status" }) 17 | public ResList(ResultCode status) { 18 | this.status = status; 19 | this.data = new ArrayList(); 20 | } 21 | 22 | @ConstructorProperties({ "status", "data" }) 23 | public ResList(ResultCode status, List data) { 24 | this.status = status; 25 | this.data = data; 26 | } 27 | 28 | public ResultCode getStatus() { 29 | return status; 30 | } 31 | 32 | public List getData() { 33 | return data; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | StringBuilder builder = new StringBuilder(); 39 | builder.append("ResList [status=").append(status).append(", data=").append(data).append("]"); 40 | return builder.toString(); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/wjw/psqueue/msg/ResQueueStatus.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.msg; 2 | 3 | import java.beans.ConstructorProperties; 4 | import java.io.Serializable; 5 | 6 | public class ResQueueStatus implements Serializable { 7 | public ResultCode status; 8 | public String queueName; 9 | public long capacity; 10 | public long size; 11 | public long head; 12 | public long tail; 13 | 14 | public ResQueueStatus() { 15 | 16 | } 17 | 18 | @ConstructorProperties({ "status", "queueName" }) 19 | public ResQueueStatus(ResultCode status, String queueName) { 20 | this.status = status; 21 | this.queueName = queueName; 22 | this.capacity = 0; 23 | this.size = 0; 24 | this.head = 0; 25 | this.tail = 0; 26 | } 27 | 28 | @ConstructorProperties({ "status", "queueName", "capacity", "size", "head", "tail" }) 29 | public ResQueueStatus(ResultCode status, String queueName, long capacity, long size, long head, long tail) { 30 | this.status = status; 31 | this.queueName = queueName; 32 | this.capacity = capacity; 33 | this.size = size; 34 | this.head = head; 35 | this.tail = tail; 36 | } 37 | 38 | public ResultCode getStatus() { 39 | return status; 40 | } 41 | 42 | public String getQueueName() { 43 | return queueName; 44 | } 45 | 46 | public long getCapacity() { 47 | return capacity; 48 | } 49 | 50 | public long getSize() { 51 | return size; 52 | } 53 | 54 | public long getHead() { 55 | return head; 56 | } 57 | 58 | public long getTail() { 59 | return tail; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | StringBuilder builder = new StringBuilder(); 65 | builder.append("ResQueueStatus [status=") 66 | .append(status) 67 | .append(", queueName=") 68 | .append(queueName) 69 | .append(", capacity=") 70 | .append(capacity) 71 | .append(", size=") 72 | .append(size) 73 | .append(", head=") 74 | .append(head) 75 | .append(", tail=") 76 | .append(tail) 77 | .append("]"); 78 | return builder.toString(); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/wjw/psqueue/msg/ResSubStatus.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.msg; 2 | 3 | import java.beans.ConstructorProperties; 4 | import java.io.Serializable; 5 | 6 | public class ResSubStatus implements Serializable { 7 | public ResultCode status; 8 | public String queueName; 9 | public long capacity; 10 | public String subName; 11 | public long size; 12 | public long head; 13 | public long tail; 14 | 15 | public ResSubStatus() { 16 | 17 | } 18 | 19 | @ConstructorProperties({ "status", "queueName", "subName" }) 20 | public ResSubStatus(ResultCode status, String queueName, String subName) { 21 | this.status = status; 22 | this.queueName = queueName; 23 | this.subName = subName; 24 | this.capacity = 0; 25 | this.size = 0; 26 | this.head = 0; 27 | this.tail = 0; 28 | } 29 | 30 | @ConstructorProperties({ "status", "queueName", "capacity", "subName", "size", "head", "tail" }) 31 | public ResSubStatus(ResultCode status, String queueName, long capacity, String subName, long size, long head, long tail) { 32 | this.status = status; 33 | this.queueName = queueName; 34 | this.capacity = capacity; 35 | this.subName = subName; 36 | this.size = size; 37 | this.head = head; 38 | this.tail = tail; 39 | } 40 | 41 | public ResultCode getStatus() { 42 | return status; 43 | } 44 | 45 | public String getQueueName() { 46 | return queueName; 47 | } 48 | 49 | public long getCapacity() { 50 | return capacity; 51 | } 52 | 53 | public String getSubName() { 54 | return subName; 55 | } 56 | 57 | public long getSize() { 58 | return size; 59 | } 60 | 61 | public long getHead() { 62 | return head; 63 | } 64 | 65 | public long getTail() { 66 | return tail; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | StringBuilder builder = new StringBuilder(); 72 | builder.append("ResSubStatus [status=") 73 | .append(status) 74 | .append(", queueName=") 75 | .append(queueName) 76 | .append(", capacity=") 77 | .append(capacity) 78 | .append(", subName=") 79 | .append(subName) 80 | .append(", size=") 81 | .append(size) 82 | .append(", head=") 83 | .append(head) 84 | .append(", tail=") 85 | .append(tail) 86 | .append("]"); 87 | return builder.toString(); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/wjw/psqueue/msg/ResultCode.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.msg; 2 | 3 | import java.beans.ConstructorProperties; 4 | 5 | public class ResultCode { 6 | public final static ResultCode SUCCESS = new ResultCode(0, "ok"); 7 | public final static ResultCode CMD_INVALID = new ResultCode(1, "Invalid opt command!"); 8 | public final static ResultCode INTERNAL_ERROR = new ResultCode(2, "internal error"); 9 | public final static ResultCode AUTHENTICATION_FAILURE = new ResultCode(3, "authentication failure"); 10 | public final static ResultCode QUEUE_NOT_EXIST = new ResultCode(4, "queue not exist"); 11 | public final static ResultCode SUB_NOT_EXIST = new ResultCode(5, "Subscriber not exist"); 12 | public final static ResultCode QUEUE_NAME_INVALID = new ResultCode(6, "invalid queue name!queue name not include:? \\ / : | < > * and _META_"); 13 | public final static ResultCode SUB_NAME_INVALID = new ResultCode(7, "invalid Subscriber name!Subscriber name not include:? \\ / : | < > * and _META_"); 14 | public final static ResultCode SUB_IS_EXIST = new ResultCode(8, "Subscriber is already exist"); 15 | public final static ResultCode QUEUE_IS_EXIST = new ResultCode(9, "queue is already exist"); 16 | public final static ResultCode QUEUE_IS_EMPTY = new ResultCode(10, "queue is empty"); 17 | public final static ResultCode QUEUE_CREATE_ERROR = new ResultCode(11, "queue create error"); 18 | public final static ResultCode QUEUE_REMOVE_ERROR = new ResultCode(12, "queue remove error"); 19 | public final static ResultCode SUB_REMOVE_ERROR = new ResultCode(13, "Subscriber remove error"); 20 | public final static ResultCode INDEX_OUT_OF_BOUNDS = new ResultCode(14, "index out of bounds"); 21 | public final static ResultCode QUEUE_ADD_ERROR = new ResultCode(15, "queue add error"); 22 | public final static ResultCode QUEUE_POLL_ERROR = new ResultCode(16, "queue poll error"); 23 | public final static ResultCode ALL_MESSAGE_CONSUMED = new ResultCode(17, "all message consumed"); 24 | public final static ResultCode QUEUE_CAPACITY_INVALID = new ResultCode(18, "queue capacity must >=1000000L AND <=1000000000L"); 25 | public final static ResultCode SUB_TAILPOS_ERROR = new ResultCode(19, "Subscriber set tail pos error"); 26 | 27 | public int code; 28 | public String msg; 29 | 30 | public ResultCode() { 31 | } 32 | 33 | @ConstructorProperties({ "code", "msg" }) 34 | public ResultCode(final int code, final String msg) { 35 | this.code = code; 36 | this.msg = msg; 37 | } 38 | 39 | public int getCode() { 40 | return code; 41 | } 42 | 43 | public String getMsg() { 44 | return msg; 45 | } 46 | 47 | @Override 48 | public String toString() { 49 | StringBuilder builder = new StringBuilder(); 50 | builder.append("ResultCode [code=").append(code).append(", msg=").append(msg).append("]"); 51 | return builder.toString(); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/wjw/psqueue/server/App.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/src/wjw/psqueue/server/App.java -------------------------------------------------------------------------------- /src/wjw/psqueue/server/Conf.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/src/wjw/psqueue/server/Conf.java -------------------------------------------------------------------------------- /src/wjw/psqueue/server/HttpRequestHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/src/wjw/psqueue/server/HttpRequestHandler.java -------------------------------------------------------------------------------- /src/wjw/psqueue/server/HttpServerChannelInitializer.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.server; 2 | 3 | import io.netty.channel.ChannelInitializer; 4 | import io.netty.channel.ChannelPipeline; 5 | import io.netty.channel.socket.SocketChannel; 6 | import io.netty.handler.codec.http.HttpObjectAggregator; 7 | import io.netty.handler.codec.http.HttpRequestDecoder; 8 | import io.netty.handler.codec.http.HttpResponseEncoder; 9 | 10 | public class HttpServerChannelInitializer extends ChannelInitializer { 11 | private App _app; 12 | 13 | public HttpServerChannelInitializer(App app) { 14 | _app = app; 15 | } 16 | 17 | @Override 18 | public void initChannel(SocketChannel ch) throws Exception { 19 | // Create a default pipeline implementation. 20 | ChannelPipeline pipeline = ch.pipeline(); 21 | 22 | // Uncomment the following line if you want HTTPS 23 | //SSLEngine engine = SecureChatSslContextFactory.getServerContext().createSSLEngine(); 24 | //engine.setUseClientMode(false); 25 | //p.addLast("ssl", new SslHandler(engine)); 26 | 27 | pipeline.addLast("decoder", new HttpRequestDecoder()); 28 | pipeline.addLast("aggregator", new HttpObjectAggregator(1048576)); 29 | pipeline.addLast("encoder", new HttpResponseEncoder()); 30 | 31 | // Remove the following line if you don't want automatic content compression. 32 | //pipeline.addLast("deflater", new HttpContentCompressor()); 33 | 34 | pipeline.addLast("handler", new HttpRequestHandler(_app)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/wjw/psqueue/server/Wrapper.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.server; 2 | 3 | import org.tanukisoftware.wrapper.WrapperManager; 4 | import org.tanukisoftware.wrapper.WrapperSimpleApp; 5 | 6 | public class Wrapper extends WrapperSimpleApp { 7 | @Override 8 | public Integer start(String[] args) { 9 | Integer result = super.start(args); 10 | 11 | WrapperManager.log(WrapperManager.WRAPPER_LOG_LEVEL_FATAL, "Started Wrapper PSQueue!"); 12 | 13 | return result; 14 | } 15 | 16 | @Override 17 | public int stop(int exitCode) { 18 | int result = super.stop(exitCode); 19 | 20 | WrapperManager.log(WrapperManager.WRAPPER_LOG_LEVEL_FATAL, "Stoped Wrapper PSQueue!"); 21 | 22 | return result; 23 | } 24 | 25 | protected Wrapper(String[] strings) { 26 | super(strings); 27 | } 28 | 29 | public static void main(String args[]) { 30 | new Wrapper(args); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/wjw/psqueue/server/jmx/AppMXBean.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wjw465150/PSQueueServer/96c568808440c9a609309ac13d58eef4ae16d3f7/src/wjw/psqueue/server/jmx/AppMXBean.java -------------------------------------------------------------------------------- /src/wjw/psqueue/server/jmx/JMXTools.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.server.jmx; 2 | 3 | import java.lang.reflect.Constructor; 4 | import java.lang.reflect.Method; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | public class JMXTools { 9 | private static org.slf4j.Logger _log = org.slf4j.LoggerFactory.getLogger(JMXTools.class); 10 | 11 | public static final Map> mapJMXPrimitive = new HashMap>(); 12 | static { 13 | mapJMXPrimitive.put("byte", byte.class); 14 | mapJMXPrimitive.put("short", short.class); 15 | mapJMXPrimitive.put("int", int.class); 16 | mapJMXPrimitive.put("long", long.class); 17 | mapJMXPrimitive.put("float", float.class); 18 | mapJMXPrimitive.put("double", double.class); 19 | mapJMXPrimitive.put("char", char.class); 20 | mapJMXPrimitive.put("boolean", boolean.class); 21 | } 22 | 23 | private JMXTools() { 24 | } 25 | 26 | /** 27 | * Java 1.5 and above supports the ability to register the WrapperManager 28 | * MBean internally. 29 | */ 30 | @SuppressWarnings("rawtypes") 31 | static public void registerMBean(Object mbean, String name) { 32 | Class classManagementFactory; 33 | Class classMBeanServer; 34 | Class classObjectName; 35 | try { 36 | classManagementFactory = Class.forName("java.lang.management.ManagementFactory"); 37 | classMBeanServer = Class.forName("javax.management.MBeanServer"); 38 | classObjectName = Class.forName("javax.management.ObjectName"); 39 | } catch (ClassNotFoundException e) { 40 | _log.error("Registering MBeans not supported by current JVM:" + name); 41 | return; 42 | } 43 | 44 | try { 45 | // This code uses reflection so it combiles on older JVMs. 46 | // The original code is as follows: 47 | // javax.management.MBeanServer mbs = 48 | // java.lang.management.ManagementFactory.getPlatformMBeanServer(); 49 | // javax.management.ObjectName oName = new javax.management.ObjectName( name ); 50 | // mbs.registerMBean( mbean, oName ); 51 | 52 | // The version of the above code using reflection follows. 53 | Method methodGetPlatformMBeanServer = classManagementFactory.getMethod("getPlatformMBeanServer", (Class[]) null); 54 | Constructor constructorObjectName = classObjectName.getConstructor(new Class[] { String.class }); 55 | Method methodRegisterMBean = classMBeanServer.getMethod("registerMBean", new Class[] { Object.class, classObjectName }); 56 | Object mbs = methodGetPlatformMBeanServer.invoke(null, (Object[]) null); 57 | Object oName = constructorObjectName.newInstance(new Object[] { name }); 58 | methodRegisterMBean.invoke(mbs, new Object[] { mbean, oName }); 59 | 60 | _log.info("Registered MBean with Platform MBean Server:" + name); 61 | } catch (Throwable t) { 62 | if (t instanceof ClassNotFoundException) { 63 | _log.error("Using MBean requires at least a JVM version 1.5."); 64 | } 65 | _log.error("Unable to register the " + name + " MBean."); 66 | t.printStackTrace(); 67 | } 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/wjw/psqueue/server/jmx/annotation/Description.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.server.jmx.annotation; 2 | 3 | import static java.lang.annotation.ElementType.CONSTRUCTOR; 4 | import static java.lang.annotation.ElementType.FIELD; 5 | import static java.lang.annotation.ElementType.METHOD; 6 | import static java.lang.annotation.ElementType.PARAMETER; 7 | import static java.lang.annotation.ElementType.TYPE; 8 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 9 | 10 | import java.lang.annotation.Documented; 11 | import java.lang.annotation.Retention; 12 | import java.lang.annotation.Target; 13 | 14 | @Documented 15 | @Retention(value = RUNTIME) 16 | @Target(value = { CONSTRUCTOR, FIELD, METHOD, PARAMETER, TYPE }) 17 | public @interface Description { 18 | String name() default ""; 19 | 20 | String description() default ""; 21 | } 22 | -------------------------------------------------------------------------------- /src/wjw/psqueue/server/jmx/annotation/MBean.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.server.jmx.annotation; 2 | 3 | import static java.lang.annotation.ElementType.TYPE; 4 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 5 | 6 | import java.lang.annotation.Documented; 7 | import java.lang.annotation.Inherited; 8 | import java.lang.annotation.Retention; 9 | import java.lang.annotation.Target; 10 | 11 | @Documented 12 | @Retention(value = RUNTIME) 13 | @Target(value = TYPE) 14 | @Inherited 15 | public @interface MBean { 16 | String objectName() default ""; 17 | 18 | String description() default ""; 19 | } 20 | -------------------------------------------------------------------------------- /src/wjw/psqueue/server/jmx/annotation/ManagedOperation.java: -------------------------------------------------------------------------------- 1 | package wjw.psqueue.server.jmx.annotation; 2 | 3 | import static java.lang.annotation.ElementType.METHOD; 4 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 5 | 6 | import java.lang.annotation.Documented; 7 | import java.lang.annotation.Retention; 8 | import java.lang.annotation.Target; 9 | 10 | @Documented 11 | @Retention(value = RUNTIME) 12 | @Target(value = { METHOD }) 13 | public @interface ManagedOperation { 14 | String description() default ""; 15 | } 16 | --------------------------------------------------------------------------------