├── .gitignore
├── src
    └── main
    │   ├── java
    │       └── io
    │       │   └── openmessaging
    │       │       ├── io
    │       │           ├── AsyncIO_task.java
    │       │           ├── tests
    │       │           │   └── test_io.java
    │       │           ├── IOReaderThread.java
    │       │           ├── AsyncIO.java
    │       │           └── IOThread.java
    │       │       ├── QueueStore.java
    │       │       ├── utils
    │       │           ├── UnsafeUtils.java
    │       │           ├── backup.md
    │       │           ├── MessageB64Serialization.java
    │       │           └── ChromiumBase64.java
    │       │       ├── DefaultQueueStoreImpl.java
    │       │       └── DemoTester.java
    │   └── resources
    │       └── package.xml
├── LICENSE
├── pom.xml
└── README.md


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | target/
3 | data*
4 | *.iml
5 | *.jar
6 | *tmp*
7 | *PlayGround.java
8 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/io/AsyncIO_task.java:
--------------------------------------------------------------------------------
1 | package io.openmessaging.io;
2 | 
3 | public class AsyncIO_task {
4 |     final long global_offset;
5 |     public AsyncIO_task(long which_chunk){
6 |        this.global_offset = which_chunk;
7 |     }
8 | }
9 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/io/tests/test_io.java:
--------------------------------------------------------------------------------
 1 | package io.openmessaging.io.tests;
 2 | 
 3 | import io.openmessaging.io.AsyncIO;
 4 | 
 5 | public class test_io {
 6 |     public static void main(String args[]){
 7 |         int queue_num_per_file[] = new int[8];
 8 |         int batch_size[] = new int[8];
 9 |         for(int i = 0 ; i < 8; i++){
10 |             queue_num_per_file[i] = 100;
11 |             batch_size[i] = 10;
12 |         }
13 |         AsyncIO asyncIO = new AsyncIO("data", 8, queue_num_per_file, batch_size);
14 |         asyncIO.start();
15 |         asyncIO.waitFinishIO();
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/src/main/resources/package.xml:
--------------------------------------------------------------------------------
 1 | <assembly xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2 |           xsi:schemaLocation="
 3 | 		http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/assembly-1.0.0.xsd
 4 | 	">
 5 |     <id>package</id>
 6 |     <formats>
 7 |         <format>dir</format>
 8 |     </formats>
 9 |     <includeBaseDirectory>true</includeBaseDirectory>
10 |     <fileSets>
11 |         <fileSet>
12 |             <directory>src/main/resources</directory>
13 |             <outputDirectory>conf</outputDirectory>
14 |             <filtered>false</filtered>
15 |         </fileSet>
16 |     </fileSets>
17 |     <dependencySets>
18 |         <dependencySet>
19 |             <outputDirectory>lib</outputDirectory>
20 |             <scope>runtime</scope>
21 |         </dependencySet>
22 |     </dependencySets>
23 | </assembly>


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/QueueStore.java:
--------------------------------------------------------------------------------
 1 | package io.openmessaging;
 2 | 
 3 | import java.util.Collection;
 4 | 
 5 | public abstract class QueueStore {
 6 |     /**
 7 |      * 把一条消息写入一个队列；
 8 |      * 这个接口需要是线程安全的，也即评测程序会并发调用该接口进行put；
 9 |      * 每个queue中的内容，按发送顺序存储消息（可以理解为Java中的List），同时每个消息会有一个索引，索引从0开始；
10 |      * 不同queue中的内容，相互独立，互不影响；
11 |      * @param queueName 代表queue名字，如果是第一次put，则自动生产一个queue
12 |      * @param message message，代表消息的内容，评测时内容会随机产生，大部分长度在64字节左右，会有少量消息在1k左右
13 |      */
14 |     abstract void put(String queueName, byte[] message);
15 | 
16 |     /**
17 |      * 从一个队列中读出一批消息，读出的消息要按照发送顺序来；
18 |      * 这个接口需要是线程安全的，也即评测程序会并发调用该接口进行get；
19 |      * 返回的Collection会被并发读，但不涉及写，因此只需要是线程读安全就可以了；
20 |      * @param queueName 代表队列的名字
21 |      * @param offset 代表消息的在这个队列中的起始消息索引
22 |      * @param num 代表读取的消息的条数，如果消息足够，则返回num条，否则只返回已有的消息即可;没有消息了，则返回一个空的集合
23 |      */
24 |     abstract Collection<byte[]> get(String queueName, long offset, long num);
25 | }
26 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Lipeng WANG and Yulin CHE
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/io/IOReaderThread.java:
--------------------------------------------------------------------------------
 1 | package io.openmessaging.io;
 2 | 
 3 | import io.openmessaging.DefaultQueueStoreImpl;
 4 | 
 5 | import java.io.IOException;
 6 | import java.nio.MappedByteBuffer;
 7 | import java.nio.channels.FileChannel;
 8 | 
 9 | public class IOReaderThread {
10 |     final public static long MAPPED_BLOCK_SIZE = AsyncIO.INDEX_ENTRY_SIZE * DefaultQueueStoreImpl.CLUSTER_SIZE * 1024 * 4 * 50;
11 |     final static long IO_READER_MAX_MAPED_CHUNK_NUM = (30L * 1024 * 1024 * 1024 / MAPPED_BLOCK_SIZE);
12 | 
13 |     IOReaderThread(FileChannel fd) {
14 |         try {
15 |             for (int i = 0; i < IO_READER_MAX_MAPED_CHUNK_NUM; i++) {
16 |                 if (i * MAPPED_BLOCK_SIZE < fd.size()) {
17 |                     long real_map_size = Math.min(MAPPED_BLOCK_SIZE, fd.size() - i * MAPPED_BLOCK_SIZE);
18 |                     index_mapped_buffers[i] = fd.map(FileChannel.MapMode.READ_ONLY, ((long) i) * MAPPED_BLOCK_SIZE, real_map_size);
19 |                 } else {
20 |                     break;
21 |                 }
22 |             }
23 |         } catch (IOException e) {
24 |             e.printStackTrace();
25 |         }
26 |     }
27 |     public MappedByteBuffer index_mapped_buffers[] = new MappedByteBuffer[(int) IO_READER_MAX_MAPED_CHUNK_NUM];
28 | }
29 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>io.openmessaging</groupId>
 8 |     <artifactId>race2018</artifactId>
 9 |     <version>1.0-SNAPSHOT</version>
10 | 
11 | 
12 |     <dependencies>
13 |         <dependency>
14 |             <groupId>org.slf4j</groupId>
15 |             <artifactId>slf4j-log4j12</artifactId>
16 |             <version>1.7.6</version>
17 |         </dependency>
18 |     </dependencies>
19 | 
20 |     <build>
21 |         <plugins>
22 |             <plugin>
23 |                 <artifactId>maven-compiler-plugin</artifactId>
24 |                 <version>2.3.2</version>
25 |                 <configuration>
26 |                     <source>1.8</source>
27 |                     <target>1.8</target>
28 |                     <encoding>utf-8</encoding>
29 |                 </configuration>
30 |             </plugin>
31 |             <plugin>
32 |                 <artifactId>maven-assembly-plugin</artifactId>
33 |                 <version>2.2.1</version>
34 |                 <dependencies>
35 |                     <dependency>
36 |                         <groupId>org.apache.maven</groupId>
37 |                         <artifactId>maven-core</artifactId>
38 |                         <version>2.2.1</version>
39 |                     </dependency>
40 |                 </dependencies>
41 |                 <configuration>
42 |                     <finalName>Race2018</finalName>
43 |                     <appendAssemblyId>false</appendAssemblyId>
44 |                     <descriptors>
45 |                         <descriptor>src/main/resources/package.xml</descriptor>
46 |                     </descriptors>
47 |                 </configuration>
48 |                 <executions>
49 |                     <execution>
50 |                         <id>make-assembly</id>
51 |                         <phase>package</phase>
52 |                         <goals>
53 |                             <goal>single</goal>
54 |                         </goals>
55 |                     </execution>
56 |                 </executions>
57 |             </plugin>
58 |         </plugins>
59 |     </build>
60 | 
61 | </project>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ## Disclaimer
  2 | 
  3 | 此项目仅供学习与交流使用，请遵循MIT协议，如果您在任何项目中使用相关代码，请保留此项目的LICENSE文件。
  4 | 
  5 | >写在前面: 
  6 | > 1. 在开始coding前请仔细阅读以下内容
  7 | 
  8 | 这个是Java Demo, C++ Demo 请查看[链接](https://code.aliyun.com/middlewarerace2018/queue-race-2018-cpp)。
  9 | 提交的时候，代码地址只需要写":"后面的内容，也即省去"git@code.aliyun.com"。
 10 | CPP的提交，请在镜像地址里写"CPP"，Java的无需填写。
 11 | 用户名和密码可以不用填写。
 12 | 
 13 | ```
 14 | -Xms1G -Xmx1G -XX:+UseConcMarkSweepGC -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails
 15 | ```
 16 | 
 17 | 
 18 | ## 1. 题目背景
 19 | Apache RocketMQ作为的一款分布式的消息中间件，历年双十一承载了万亿级的消息流转，为业务方提供高性能低延迟的稳定可靠的消息服务。随着业务的逐步发展和云上的输出，单机队列数量的逐步增加，给RocketMQ带来了新的挑战。复赛的题目要求设计一个单机百万队列以上的存储引擎，单机内存有限，需要充分利用数据结构与存储技术，最大化吞吐量。  
 20 | 
 21 | ## 2. 题目描述
 22 | 
 23 | ### 2.1 题目内容
 24 | 实现一个进程内的队列引擎，单机可支持100万队列以上。
 25 | 
 26 | 
 27 | ### 2.2 语言限定
 28 | JAVA和C++
 29 | 
 30 | 
 31 | 注意：
 32 | Java和C++一起参与排名。
 33 | C++的Demo还在制作中（预计两天后可以出来），其核心逻辑与Java是一致的。选手们可以参考Java先行开始代码编写。
 34 | 
 35 | 
 36 | ## 3.  程序目标
 37 | 
 38 | 仔细阅读demo项目中的QueueStore，DefaultQueueStoreImpl，DemoTester三个类。
 39 | 
 40 | 你的coding目标是重写DefaultQueueStoreImpl，并实现以下接口:  
 41 | abstract void put(String queueName, String message);  
 42 | abstract Collection<String> get(String queueName, long offset, long num);  
 43 | 
 44 | 注：
 45 | 评测时的数据存储路径为：/alidata1/race2018/data。
 46 | 日志请直接打印在控制台标准输出，可以使用System.out.println，如果使用日志框架，请配置为ConsoleAppender。注意不要把日志输出到Error通道（也即不要使用System.err.println，如果使用日志框架，则不要使用log.error）。评测程序会把控制台标准输出的内容搜集出来，放置在OSS上面供用户排错，但是请不要密集打印日志，单次评测，最多不能超过100M。
 47 | 日志下载路径为：http://race2018.oss-cn-beijing.aliyuncs.com/{teamcode}.logs.tgz，只保存最近一次任务的日志。
 48 | 
 49 | 
 50 | ## 4.参赛方法说明
 51 | 1. 在阿里天池找到"中间件性能挑战赛"，并报名参加
 52 | 2. 在code.aliyun.com注册一个账号，并新建一个仓库名，并将大赛官方账号middlewarerace2018添加为项目成员，权限为reporter
 53 | 3. fork或者拷贝本仓库的代码到自己的仓库，并实现自己的逻辑
 54 | 4. 在天池提交成绩的入口，提交自己的仓库git地址，等待评测结果
 55 | 5. 坐等每天10点排名更新
 56 | 
 57 | 
 58 | ## 4. 测试环境描述
 59 | 测试环境为4c8g的ECS，限定使用的最大JVM大小为4GB(-Xmx4g)。带一块300G左右大小的SSD磁盘。
 60 | 
 61 | SSD性能大致如下：
 62 | iops 1w 左右；块读写能力(一次读写4K以上) 在200MB/s 左右。
 63 | 
 64 | ulimit -a:
 65 | 
 66 | ```
 67 | core file size          (blocks, -c) 0
 68 | data seg size           (kbytes, -d) unlimited
 69 | scheduling priority             (-e) 0
 70 | file size               (blocks, -f) unlimited
 71 | pending signals                 (-i) 31404
 72 | max locked memory       (kbytes, -l) 64
 73 | max memory size         (kbytes, -m) unlimited
 74 | open files                      (-n) 6553560
 75 | pipe size            (512 bytes, -p) 8
 76 | POSIX message queues     (bytes, -q) 819200
 77 | real-time priority              (-r) 0
 78 | stack size              (kbytes, -s) 10240
 79 | cpu time               (seconds, -t) unlimited
 80 | max user processes              (-u) 31404
 81 | virtual memory          (kbytes, -v) unlimited
 82 | file locks                      (-x) unlimited
 83 | ```
 84 | 磁盘调度算法是 deadline
 85 | 其它系统参数都是默认的。
 86 | 
 87 | ## 5. 程序校验逻辑
 88 | 
 89 | 校验程序分为三个阶段：
 90 | 1.发送阶段
 91 | 2.索引校验阶段
 92 | 3.顺序消费阶段
 93 | 请详细阅读DemoTester以理解评测程序的逻辑。
 94 | 
 95 | ### 5.1. 程序校验规模说明
 96 | 1.各个阶段线程数在20~30左右
 97 | 2.发送阶段：消息大小在50字节左右，消息条数在20亿条左右，也即发送总数据在100G左右
 98 | 3.索引校验阶段：会对所有队列的索引进行随机校验；平均每个队列会校验1~2次；
 99 | 4.顺序消费阶段：挑选20%的队列进行全部读取和校验；
100 | 5.发送阶段最大耗时不能超过1800s；索引校验阶段和顺序消费阶段加在一起，最大耗时也不能超过1800s；超时会被判断为评测失败。
101 | 
102 | ## 6. 排名规则
103 | 
104 | 在结果校验100%正确的前提下，按照平均tps从高到低来排名
105 | 
106 | 
107 | ## 7. 第二/三方库规约
108 | 
109 | * 仅允许依赖JavaSE 8 包含的lib
110 | * 可以参考别人的实现，拷贝少量的代码
111 | * 我们会对排名靠前的代码进行review，如果发现大量拷贝别人的代码，将扣分
112 | 
113 | ## 8.作弊说明
114 | 
115 | 所有消息都应该进行按实际发送的信息进行存储，可以压缩，但不能伪造。
116 | 如果发现有作弊行为，比如通过hack评测程序，绕过了必须的评测逻辑，则程序无效，且取消参赛资格。
117 | 
118 | 
119 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/utils/UnsafeUtils.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging.utils;
  2 | 
  3 | import sun.misc.Unsafe;
  4 | import sun.nio.ch.DirectBuffer;
  5 | 
  6 | import java.lang.reflect.Field;
  7 | import java.lang.reflect.Modifier;
  8 | import java.nio.Buffer;
  9 | import java.nio.ByteBuffer;
 10 | import java.nio.MappedByteBuffer;
 11 | 
 12 | // Not to be used in our project
 13 | public class UnsafeUtils {
 14 |     private static Unsafe getUnsafe() {
 15 |         Unsafe unsafe = null;
 16 |         try {
 17 |             Field f = Unsafe.class.getDeclaredField("theUnsafe");
 18 |             f.setAccessible(true);
 19 |             unsafe = (Unsafe) f.get(null);
 20 |         } catch (Exception e) {
 21 |         }
 22 |         return unsafe;
 23 |     }
 24 | 
 25 |     public static void main(String[] args) {
 26 |         Unsafe unsafe = getUnsafe();
 27 |         long address = unsafe.allocateMemory(2);
 28 |         short number = 1;
 29 |         unsafe.putShort(address, number);
 30 |         if (unsafe.getByte(address) == 0)
 31 |             System.out.println("Big Endian");
 32 |         else
 33 |             System.out.println("Little Endian");
 34 |         unsafe.freeMemory(address);
 35 |     }
 36 | 
 37 |     //===================start of the unsafe hack========================
 38 |     //https://stackoverflow.com/questions/15409727/missing-some-absolute-methods-on-bytebuffer
 39 |     private static final sun.misc.Unsafe UNSAFE;
 40 | 
 41 |     public static void unmap(MappedByteBuffer buffer) {
 42 |         if(buffer != null) {
 43 |             sun.misc.Cleaner cleaner = ((DirectBuffer) buffer).cleaner();
 44 |             cleaner.clean();
 45 |         }
 46 |     }
 47 | 
 48 |     static {
 49 |         Object result = null;
 50 |         try {
 51 |             Class<?> klass = Class.forName("sun.misc.Unsafe");
 52 |             for (Field field : klass.getDeclaredFields()) {
 53 |                 if (field.getType() == klass &&
 54 |                         (field.getModifiers() & (Modifier.FINAL | Modifier.STATIC)) ==
 55 |                                 (Modifier.FINAL | Modifier.STATIC)) {
 56 |                     field.setAccessible(true);
 57 |                     result = field.get(null);
 58 |                     break;
 59 |                 }
 60 |             }
 61 |         } catch (Throwable ignored) {
 62 |         }
 63 |         UNSAFE = result == null ? null : (sun.misc.Unsafe) result;
 64 |     }
 65 | 
 66 |     private static final Field ADDRESS_FIELD;
 67 | 
 68 |     static {
 69 |         Field f;
 70 |         try {
 71 |             f = Buffer.class.getDeclaredField("address");
 72 |             f.setAccessible(true);
 73 |         } catch (NoSuchFieldException | SecurityException e) {
 74 |             f = null;
 75 |         }
 76 |         ADDRESS_FIELD = f;
 77 |     }
 78 | 
 79 | 
 80 |     public static void absolutePut(ByteBuffer dstBuffer, int dstPosition, ByteBuffer srcBuffer) {
 81 |         if (!srcBuffer.isDirect()) {
 82 |             absolutePut(dstBuffer, dstPosition,
 83 |                     srcBuffer.array(), srcBuffer.arrayOffset() + srcBuffer.position(),
 84 |                     srcBuffer.remaining());
 85 |             return;
 86 |         }
 87 | 
 88 |         if (UNSAFE != null && ADDRESS_FIELD != null && dstBuffer.isDirect()) {
 89 |             try {
 90 |                 long dstAddress = (long) ADDRESS_FIELD.get(dstBuffer) + dstPosition;
 91 |                 long srcAddress = (long) ADDRESS_FIELD.get(srcBuffer) + srcBuffer.position();
 92 |                 UNSAFE.copyMemory(srcAddress, dstAddress, srcBuffer.remaining());
 93 |             } catch (IllegalAccessException e) {
 94 |                 throw new RuntimeException(e);
 95 |             }
 96 |         } else {
 97 |             // fallback to basic loop
 98 |             for (int i = srcBuffer.position(); i < srcBuffer.limit(); i++) {
 99 |                 dstBuffer.put(dstPosition + i, srcBuffer.get(i));
100 |             }
101 |         }
102 |     }
103 | 
104 |     public static void absolutePut(ByteBuffer dstBuffer, int dstPosition, byte[] src, int srcOffset, int length) {
105 |         if (UNSAFE != null && ADDRESS_FIELD != null && dstBuffer.isDirect()) {
106 |             try {
107 |                 long dstAddress = (long) ADDRESS_FIELD.get(dstBuffer) + dstPosition;
108 |                 UNSAFE.copyMemory(
109 |                         src, UNSAFE.arrayBaseOffset(byte[].class) + srcOffset,
110 |                         null, dstAddress,
111 |                         length);
112 |             } catch (IllegalAccessException e) {
113 |                 throw new RuntimeException(e);
114 |             }
115 |         } else {
116 |             // fallback to System.arraycopy
117 |             System.arraycopy(
118 |                     src, srcOffset,
119 |                     dstBuffer.array(), dstBuffer.arrayOffset() + dstPosition,
120 |                     length);
121 |         }
122 |     }
123 |     //===================end of the unsafe hack========================
124 | }


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/io/AsyncIO.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging.io;
  2 | 
  3 | import io.openmessaging.DefaultQueueStoreImpl;
  4 | 
  5 | import java.util.concurrent.BrokenBarrierException;
  6 | import java.util.concurrent.ConcurrentSkipListSet;
  7 | import java.util.concurrent.CyclicBarrier;
  8 | import java.util.concurrent.locks.Lock;
  9 | import java.util.concurrent.locks.ReentrantLock;
 10 | 
 11 | enum IO_thread_status {INITING, RUNNING, CLOSING};
 12 | 
 13 | public class AsyncIO {
 14 |     final public static byte LARGE_MESSAGE_MAGIC_CHAR = '!';
 15 |     final public static int INDEX_ENTRY_SIZE = 43;
 16 |     final public static int RAW_NORMAL_MESSAGE_SIZE = 58;
 17 |     final static int MAX_CONCURRENT_INDEX_MAPPED_BLOCK_NUM = 3;
 18 |     final static int MAX_BLOCKING_IO_TASK = 64;
 19 |     final public static long INDEX_MAPPED_BLOCK_SIZE = (INDEX_ENTRY_SIZE * 1000 * 250 * 10);
 20 |     final public static long INDEX_BLOCK_WRITE_TIMES_TO_FULL = (INDEX_MAPPED_BLOCK_SIZE / INDEX_ENTRY_SIZE);
 21 |     final public static int MAX_MAPED_CHUNK_NUM = (int) (24L * 1024 * 1024 * 1024 / INDEX_MAPPED_BLOCK_SIZE);
 22 | 
 23 |     static boolean mapped_buffer_flags = false;
 24 |     static boolean finished = false;
 25 |     static int finished_thread = 0;
 26 | 
 27 |     static boolean sended_flush_msg = false;
 28 |     static Lock send_lock = new ReentrantLock();
 29 |     static int thread_num;
 30 |     public static IOThread work_threads[];
 31 | 
 32 |     public static IOReaderThread ioReaderThread[] = new IOReaderThread[DefaultQueueStoreImpl.IO_THREAD];
 33 | 
 34 |     public AsyncIO(String file_prefix, int thread_num, int queue_num_per_file[], int batch_size[]) {
 35 |         AsyncIO.thread_num = thread_num;
 36 |         this.work_threads = new IOThread[thread_num];
 37 |         for (int i = 0; i < thread_num; i++) {
 38 |             work_threads[i] = new IOThread(i, file_prefix, queue_num_per_file[i], batch_size[i], MAX_BLOCKING_IO_TASK);
 39 |         }
 40 | 
 41 |     }
 42 | 
 43 |     public void start() {
 44 |         for (int i = 0; i < thread_num; i++) {
 45 |             this.work_threads[i].start();
 46 |         }
 47 |     }
 48 | 
 49 |     public void submitIOTask(int which_thread, AsyncIO_task task) {
 50 |         if (work_threads[which_thread].status == IO_thread_status.RUNNING) {
 51 |             try {
 52 |                 work_threads[which_thread].blockingQueue.put(task);
 53 |             } catch (InterruptedException e) {
 54 |                 e.printStackTrace();
 55 |             }
 56 |         } else {
 57 |             System.out.printf("failed to submit IO task to %d thread\n", which_thread);
 58 |         }
 59 |     }
 60 | 
 61 |     ConcurrentSkipListSet<Long> exitThreadSets = new ConcurrentSkipListSet<>();
 62 |     final Object terminateLock = new Object();
 63 | 
 64 |     public void waitFinishIO() {
 65 |         if (!finished) {
 66 |             send_lock.lock();
 67 |             if (!sended_flush_msg) {
 68 |                 sended_flush_msg = true;
 69 |                 try {
 70 |                     for (int i = 0; i < thread_num; i++) {
 71 |                         AsyncIO_task task = new AsyncIO_task(-1);
 72 |                         work_threads[i].blockingQueue.put(task);
 73 | 
 74 |                     }
 75 |                 } catch (InterruptedException e) {
 76 |                     e.printStackTrace();
 77 |                 }
 78 |             }
 79 |             send_lock.unlock();
 80 | 
 81 |             synchronized (DefaultQueueStoreImpl.indexCheckerStartSync) {
 82 |                 if (AsyncIO.finished_thread < thread_num) {
 83 |                     try {
 84 |                         DefaultQueueStoreImpl.indexCheckerStartSync.wait();
 85 | 
 86 |                     } catch (InterruptedException e) {
 87 |                         e.printStackTrace();
 88 |                     }
 89 |                 }
 90 |             }
 91 |             finished = true;
 92 |         }
 93 |     }
 94 | 
 95 |     private CyclicBarrier barrier1 = new CyclicBarrier(10, () -> {
 96 | //            for(int i = 0 ; i < DefaultQueueStoreImpl.IO_THREAD; i++){
 97 | //                for(int chunkID = 0 ; chunkID < AsyncIO.MAX_MAPED_CHUNK_NUM; chunkID++){
 98 | //                    if(AsyncIO.work_threads[i].index_file_memory_blocks[chunkID] != null){
 99 | //                        AsyncIO.work_threads[i].index_file_memory_blocks[chunkID].clear();
100 | //                        AsyncIO.work_threads[i].index_file_memory_blocks[chunkID] = null;
101 | //                    }
102 | //                }
103 | //            }
104 |         for (int i = 0; i < DefaultQueueStoreImpl.IO_THREAD; i++) {
105 |             ioReaderThread[i] = new IOReaderThread(AsyncIO.work_threads[i].index_file_fd);
106 |         }
107 |     });
108 | 
109 |     public void waitMappedBuffers() {
110 |         if (!mapped_buffer_flags) {
111 |             try {
112 |                 barrier1.await();
113 |             } catch (InterruptedException | BrokenBarrierException e) {
114 |                 e.printStackTrace();
115 |             }
116 |             mapped_buffer_flags = true;
117 |         }
118 |     }
119 | }
120 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/utils/backup.md:
--------------------------------------------------------------------------------
  1 | ```java
  2 |     static int ChromiumBase64Decode(byte[] dest, byte[] src, int len) {
  3 | //        if (len == 0) return 0;
  4 | //
  5 | //        /*
  6 | //         * if padding is used, then the message must be at least
  7 | //         * 4 chars and be a multiple of 4
  8 | //         */
  9 | //        if (len < 4 || (len % 4 != 0)) {
 10 | //            return MODP_B64_ERROR; /* error */
 11 | //        }
 12 | //        /* there can be at most 2 pad chars at the end */
 13 | //        if (src[len - 1] == CHARPAD) {
 14 | //            len--;
 15 | //            if (src[len - 1] == CHARPAD) {
 16 | //                len--;
 17 | //            }
 18 | //        }
 19 | 
 20 |         int i;
 21 | //        int leftover = len % 4;
 22 | //        int chunks = (leftover == 0) ? len / 4 - 1 : len / 4;
 23 |         int chunks = len / 4;
 24 | 
 25 |         int x;
 26 |         int size_p = 0;
 27 | 
 28 |         int offset_y = 0;
 29 |         for (i = 0; i < chunks; ++i, offset_y += 4) {
 30 |             x = d0[src[offset_y]] | d1[src[1 + offset_y]] | d2[src[2 + offset_y]] | d3[src[3 + offset_y]];
 31 | //            if (x >= BADCHAR) return MODP_B64_ERROR;
 32 |             dest[size_p++] = (byte) ((x >>> 0) & 0xff);  //x[0]
 33 |             dest[size_p++] = (byte) ((x >>> 8) & 0xff);  //x[1]
 34 |             dest[size_p++] = (byte) ((x >>> 16) & 0xff);   //x[2]
 35 |         }
 36 | 
 37 | //        switch (leftover) {
 38 | //            case 0:
 39 | //                x = d0[src[offset_y]] | d1[src[offset_y + 1]] | d2[src[offset_y + 2]] | d3[src[offset_y + 3]];
 40 | //
 41 | //                if (x >= BADCHAR) return MODP_B64_ERROR;
 42 | //                dest[size_p++] = (byte) ((x >>> 0) & 0xff);  //x[0]
 43 | //                dest[size_p++] = (byte) ((x >>> 8) & 0xff);  //x[1]
 44 | //                dest[size_p] = (byte) ((x >>> 16) & 0xff);   //x[2]
 45 | //                return (chunks + 1) * 3;
 46 | //            case 1:  /* with padding this is an impossible case */
 47 | //                x = d0[src[offset_y]];
 48 | //                dest[size_p] = (byte) ((x >>> 0) & 0xff);  //x[0]// i.e. first char/byte in int
 49 | //                break;
 50 | //            case 2: // * case 2, 1  output byte */
 51 | //                x = d0[src[offset_y]] | d1[src[offset_y + 1]];
 52 | //                dest[size_p] = (byte) ((x >>> 0) & 0xff);  //x[0] // i.e. first char
 53 | //                break;
 54 | //            default: /* case 3, 2 output bytes */
 55 | //                x = d0[src[offset_y]] | d1[src[offset_y + 1]] | d2[src[offset_y + 2]];  /* 0x3c */
 56 | //                dest[size_p++] = (byte) ((x >>> 0) & 0xff);  //x[0]
 57 | //                dest[size_p] = (byte) ((x >>> 8) & 0xff);  //x[1]
 58 | //                break;
 59 | //        }
 60 | //        if (x >= BADCHAR) return MODP_B64_ERROR;
 61 | //        return 3 * chunks + (6 * leftover) / 8;
 62 |         return 3 * chunks;
 63 |     }
 64 | ```
 65 | 
 66 | ```java
 67 |     static int ChromiumBase64Encode(byte[] dest, byte[] str, int len) {
 68 |         int i = 0;
 69 |         int size_p = 0;
 70 | 
 71 |         /* unsigned here is important! */ // for correct offset loopup
 72 |         byte t1, t2, t3;
 73 | 
 74 |         if (len > 2) {
 75 |             for (; i < len - 2; i += 3) {
 76 |                 t1 = str[i];
 77 |                 t2 = str[i + 1];
 78 |                 t3 = str[i + 2];
 79 |                 dest[size_p++] = e0[t1 & 0xff];
 80 |                 dest[size_p++] = e1[(((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)) & 0xff];
 81 |                 dest[size_p++] = e1[(((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)) & 0xff];
 82 |                 dest[size_p++] = e2[t3 & 0xff];
 83 |             }
 84 |         }
 85 | 
 86 |         switch (len - i) {
 87 |             case 0:
 88 |                 break;
 89 |             case 1:
 90 |                 t1 = str[i];
 91 |                 dest[size_p++] = e0[t1 & 0xff];
 92 |                 dest[size_p++] = e1[((t1 & 0x03) << 4) & 0xff];
 93 |                 dest[size_p++] = CHARPAD;
 94 |                 dest[size_p++] = CHARPAD;
 95 |                 break;
 96 |             default: /* case 2 */
 97 |                 t1 = str[i];
 98 |                 t2 = str[i + 1];
 99 |                 dest[size_p++] = e0[t1 & 0xff];
100 |                 dest[size_p++] = e1[(((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)) & 0xff];
101 |                 dest[size_p++] = e2[((t2 & 0x0F) << 2) & 0xff];
102 |                 dest[size_p++] = CHARPAD;
103 |         }
104 | //        dest[size_p] = '\0';          // no need to use \0 in java
105 |         return size_p;
106 |     }
107 | ```
108 | 
109 | ```java
110 | 
111 |     public static int ChromiumBase6EncodeLen(int A) {
112 |         return ((A + 2) / 3 * 4 + 1);       // 2 for padding, 3 bytes -> 4 bytes, 1 is for '\0'
113 |     }
114 | ```
115 | 
116 | ```java
117 |     static int MODP_B64_ERROR = ((Integer.MAX_VALUE) - 1);
118 |     static byte CHAR62 = '+';
119 |     static byte CHAR63 = '/';
120 |         private static int BADCHAR = 0x01FFFFFF;
121 | 
122 | 
123 | ```
124 | 
125 | ```java
126 | switch (len - i) {
127 |             case 0:
128 |                 break;
129 |             case 1:
130 |                 t1 = str[i];
131 |                 dest[size_p++] = e0[t1 & 0xff];
132 |                 dest[size_p++] = e1[((t1 & 0x03) << 4) & 0xff];
133 |                 dest[size_p++] = CHARPAD;
134 |                 dest[size_p++] = CHARPAD;
135 |                 break;
136 |             default: /* case 2 */
137 |                 t1 = str[i];
138 |                 t2 = str[i + 1];
139 |                 dest[size_p++] = e0[t1 & 0xff];
140 |                 dest[size_p++] = e1[(((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)) & 0xff];
141 |                 dest[size_p++] = e2[((t2 & 0x0F) << 2) & 0xff];
142 |                 dest[size_p++] = CHARPAD;
143 |         }
144 | ```
145 | 
146 | ```java
147 |     static int ChromiumBase64Encode(byte[] dest, byte[] str, int len) {
148 |         int i = 0;
149 |         int size_p = 0;
150 |         /* unsigned here is important! */ // for correct offset loopup
151 |         byte t1, t2, t3;
152 |         if (len > 2) {
153 |             for (; i < len - 2; i += 3) {
154 |                 t1 = str[i];
155 |                 t2 = str[i + 1];
156 |                 t3 = str[i + 2];
157 |                 dest[size_p++] = e0[t1 & 0xff];
158 |                 dest[size_p++] = e1[(((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)) & 0xff];
159 |                 dest[size_p++] = e1[(((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)) & 0xff];
160 |                 dest[size_p++] = e2[t3 & 0xff];
161 |             }
162 |         }
163 |         return size_p;
164 |     }
165 | ```


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/io/IOThread.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging.io;
  2 | 
  3 | import io.openmessaging.DefaultQueueStoreImpl;
  4 | 
  5 | import java.io.File;
  6 | import java.io.IOException;
  7 | import java.nio.ByteBuffer;
  8 | import java.nio.MappedByteBuffer;
  9 | import java.nio.channels.FileChannel;
 10 | import java.nio.file.StandardOpenOption;
 11 | import java.util.concurrent.ArrayBlockingQueue;
 12 | import java.util.concurrent.BlockingQueue;
 13 | import java.util.concurrent.atomic.AtomicIntegerArray;
 14 | import java.util.concurrent.atomic.AtomicLong;
 15 | 
 16 | import static io.openmessaging.utils.UnsafeUtils.absolutePut;
 17 | import static io.openmessaging.utils.UnsafeUtils.unmap;
 18 | 
 19 | public class IOThread extends Thread {
 20 | 
 21 |     private void doIO(AsyncIO_task task) throws IOException {
 22 |         for (; current_index_mapped_start_chunk < AsyncIO.MAX_MAPED_CHUNK_NUM; current_index_mapped_start_chunk++) {
 23 |             if (index_mapped_block_write_counter.get(current_index_mapped_start_chunk) >= AsyncIO.INDEX_BLOCK_WRITE_TIMES_TO_FULL) {
 24 | 
 25 | //                MappedByteBuffer index_file_buf = index_file_fd.map(FileChannel.MapMode.READ_WRITE,
 26 | //                        AsyncIO.INDEX_MAPPED_BLOCK_SIZE * current_index_mapped_start_chunk, AsyncIO.INDEX_MAPPED_BLOCK_SIZE);
 27 | //                absolutePut(index_file_buf, 0, index_file_memory_blocks[current_index_mapped_start_chunk], 0, (int) AsyncIO.INDEX_MAPPED_BLOCK_SIZE);
 28 | //                unmap(index_file_buf);
 29 |                 index_file_fd.write(ByteBuffer.wrap(index_file_memory_blocks[current_index_mapped_start_chunk]), AsyncIO.INDEX_MAPPED_BLOCK_SIZE * current_index_mapped_start_chunk);
 30 | 
 31 |                 int next_chunk = current_index_mapped_start_chunk + AsyncIO.MAX_CONCURRENT_INDEX_MAPPED_BLOCK_NUM;
 32 |                 current_index_mapped_start_offset += AsyncIO.INDEX_MAPPED_BLOCK_SIZE;
 33 |                 current_index_mapped_end_offset += AsyncIO.INDEX_MAPPED_BLOCK_SIZE;
 34 |                 this.index_file_size = this.current_index_mapped_end_offset;
 35 |                 this.index_file_fd.truncate(this.index_file_size);
 36 | 
 37 |                 index_file_memory_blocks[next_chunk] = index_file_memory_blocks[current_index_mapped_start_chunk];
 38 |                 index_file_memory_blocks[current_index_mapped_start_chunk] = null;
 39 |                 synchronized (sync_blocks[next_chunk]) {
 40 |                     sync_blocks[next_chunk].notifyAll();
 41 |                 }
 42 |             } else {
 43 |                 break;
 44 |             }
 45 |         }
 46 |     }
 47 | 
 48 |     @Override
 49 |     public void run() {
 50 |         this.status = IO_thread_status.RUNNING;
 51 |         for (; ; ) {
 52 |             AsyncIO_task task = null;
 53 |             try {
 54 |                 task = this.blockingQueue.take();
 55 |             } catch (InterruptedException e) {
 56 |                 e.printStackTrace();
 57 |             }
 58 |             if (this.status == IO_thread_status.CLOSING || task.global_offset == -1) {
 59 |                 for (int i = current_index_mapped_start_chunk; i < AsyncIO.MAX_MAPED_CHUNK_NUM && index_file_memory_blocks[i] != null; i++) {
 60 |                     try {
 61 | //                        MappedByteBuffer index_file_buf = index_file_fd.map(FileChannel.MapMode.READ_WRITE, i * AsyncIO.INDEX_MAPPED_BLOCK_SIZE, AsyncIO.INDEX_MAPPED_BLOCK_SIZE);
 62 | //                        absolutePut(index_file_buf, 0, index_file_memory_blocks[i], 0, (int) AsyncIO.INDEX_MAPPED_BLOCK_SIZE);
 63 | //                        unmap(index_file_buf);
 64 |                         index_file_fd.write(ByteBuffer.wrap(index_file_memory_blocks[i]), AsyncIO.INDEX_MAPPED_BLOCK_SIZE * i);
 65 |                         index_file_memory_blocks[i] = null;
 66 |                     } catch (IOException e) {
 67 |                         e.printStackTrace();
 68 |                     }
 69 |                 }
 70 |                 // must print out, otherwise incorrect because of compiler jit optmization
 71 |                 synchronized (DefaultQueueStoreImpl.indexCheckerStartSync) {
 72 |                     AsyncIO.finished_thread++;
 73 |                     if (AsyncIO.finished_thread == DefaultQueueStoreImpl.IO_THREAD) {
 74 |                         DefaultQueueStoreImpl.indexCheckerStartSync.notifyAll();
 75 |                     }
 76 |                 }
 77 |                 break;
 78 |             }
 79 |             try {
 80 |                 doIO(task);
 81 |             } catch (IOException e) {
 82 |                 e.printStackTrace();
 83 |             }
 84 |         }
 85 |     }
 86 | 
 87 |     IOThread(int thread_id, String file_prefix, int queue_num_per_file, int batch_size, int blocking_queue_size) {
 88 |         this.status = IO_thread_status.INITING;
 89 |         this.batch_size = batch_size;
 90 |         this.queue_num_per_file = queue_num_per_file;
 91 |         this.thread_id = thread_id;
 92 |         this.chunk_size = queue_num_per_file * batch_size;
 93 |         File data_file_name_f = new File(file_prefix + '_' + thread_id + ".data");
 94 |         File index_file_name_f = new File(file_prefix + '_' + thread_id + ".idx");
 95 |         try {
 96 |             data_file_fd = FileChannel.open(data_file_name_f.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE,
 97 |                     StandardOpenOption.CREATE, StandardOpenOption.SPARSE, StandardOpenOption.TRUNCATE_EXISTING);
 98 |             index_file_fd = FileChannel.open(index_file_name_f.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE,
 99 |                     StandardOpenOption.CREATE, StandardOpenOption.SPARSE, StandardOpenOption.TRUNCATE_EXISTING);
100 |         } catch (IOException e) {
101 |             e.printStackTrace();
102 |         }
103 |         blockingQueue = new ArrayBlockingQueue<>(AsyncIO.MAX_BLOCKING_IO_TASK);
104 |         this.queue_counter = new AtomicIntegerArray(queue_num_per_file);
105 | 
106 |         try {
107 |             this.current_index_mapped_start_offset = 0;
108 |             this.current_index_mapped_start_chunk = 0;
109 |             this.current_index_mapped_end_offset = AsyncIO.MAX_CONCURRENT_INDEX_MAPPED_BLOCK_NUM * AsyncIO.INDEX_MAPPED_BLOCK_SIZE;
110 |             this.index_file_size = this.current_index_mapped_end_offset;
111 | 
112 |             this.index_file_fd.truncate(this.index_file_size);
113 |             for (int i = 0; i < AsyncIO.MAX_CONCURRENT_INDEX_MAPPED_BLOCK_NUM; i++) {
114 |                 index_file_memory_blocks[i] = new byte[(int) AsyncIO.INDEX_MAPPED_BLOCK_SIZE];
115 |             }
116 | 
117 |         } catch (IOException e) {
118 |             e.printStackTrace();
119 |         }
120 |         for (int i = 0; i < AsyncIO.MAX_MAPED_CHUNK_NUM; i++) {
121 |             sync_blocks[i] = new Object();
122 |         }
123 |     }
124 | 
125 |     private final int thread_id;
126 |     final public long chunk_size;
127 |     private final int queue_num_per_file;
128 |     final public int batch_size;
129 |     public AtomicLong data_file_size = new AtomicLong(0);
130 |     public long index_file_size = 0;
131 |     public AtomicIntegerArray queue_counter;
132 | 
133 |     public FileChannel data_file_fd;
134 |     public FileChannel index_file_fd;
135 |     final BlockingQueue<AsyncIO_task> blockingQueue;
136 |     IO_thread_status status;
137 | 
138 |     public long current_index_mapped_start_offset = 0;
139 |     public long current_index_mapped_end_offset = 0;
140 |     public int current_index_mapped_start_chunk = 0;
141 | 
142 |     public byte[][] index_file_memory_blocks = new byte[AsyncIO.MAX_MAPED_CHUNK_NUM][];
143 |     public AtomicIntegerArray index_mapped_block_write_counter = new AtomicIntegerArray(AsyncIO.MAX_MAPED_CHUNK_NUM);
144 |     public Object sync_blocks[] = new Object[AsyncIO.MAX_MAPED_CHUNK_NUM];
145 | }
146 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/DefaultQueueStoreImpl.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging;
  2 | 
  3 | 
  4 | import io.openmessaging.io.AsyncIO;
  5 | import io.openmessaging.io.AsyncIO_task;
  6 | import io.openmessaging.io.IOReaderThread;
  7 | import io.openmessaging.io.IOThread;
  8 | import io.openmessaging.utils.MessageB64Serialization;
  9 | 
 10 | import java.io.IOException;
 11 | import java.nio.ByteBuffer;
 12 | import java.nio.channels.FileChannel;
 13 | import java.util.ArrayList;
 14 | import java.util.Collection;
 15 | import java.util.concurrent.ConcurrentSkipListSet;
 16 | 
 17 | import static io.openmessaging.utils.UnsafeUtils.absolutePut;
 18 | 
 19 | /**
 20 |  * 这是一个简单的基于内存的实现，以方便选手理解题意；
 21 |  * 实际提交时，请维持包名和类名不变，把方法实现修改为自己的内容；
 22 |  */
 23 | public class DefaultQueueStoreImpl extends QueueStore {
 24 |     final static String file_prefix = "/alidata1/race2018/data/data";
 25 |     final static long TOTAL_QUEUE_NUM = 1000000;
 26 |     final public static int IO_THREAD = 4;
 27 |     final public static int CLUSTER_SIZE = 20;
 28 |     final static int queue_nums[] = new int[IO_THREAD];
 29 |     final static int cluster_size[] = new int[IO_THREAD];
 30 | 
 31 |     final private static FileChannel data_file_handles[] = new FileChannel[IO_THREAD];
 32 |     final private static FileChannel index_file_handles[] = new FileChannel[IO_THREAD];
 33 | 
 34 |     final private AsyncIO asyncIO;
 35 |     public static final Object indexCheckerStartSync = new Object();
 36 | 
 37 |     private boolean phase3 = false;
 38 |     private static ConcurrentSkipListSet<Long> tidSet = new ConcurrentSkipListSet<>();
 39 | 
 40 |     private static long getQueueID(String queueName) {
 41 |         long res = 0;
 42 |         long multiplier = 1;
 43 |         for (int i = queueName.length() - 1; i >= 0 && queueName.charAt(i) >= '0' && queueName.charAt(i) <= '9'; i--) {
 44 |             res += (queueName.charAt(i) - '0') * multiplier;
 45 |             multiplier *= 10;
 46 |         }
 47 |         return res;
 48 |     }
 49 | 
 50 |     public DefaultQueueStoreImpl() {
 51 |         int q_ave = (int) TOTAL_QUEUE_NUM / IO_THREAD;
 52 |         for (int i = 0; i < IO_THREAD; i++) {
 53 |             queue_nums[i] = q_ave;
 54 |             cluster_size[i] = CLUSTER_SIZE;
 55 |         }
 56 |         queue_nums[IO_THREAD - 1] = (int) TOTAL_QUEUE_NUM - q_ave * (IO_THREAD - 1);
 57 |         cluster_size[IO_THREAD - 1] = CLUSTER_SIZE;
 58 | 
 59 |         asyncIO = new AsyncIO(file_prefix, IO_THREAD, queue_nums, cluster_size);
 60 |         asyncIO.start();
 61 | 
 62 |         for (int i = 0; i < IO_THREAD; i++) {
 63 |             data_file_handles[i] = AsyncIO.work_threads[i].data_file_fd;
 64 |             index_file_handles[i] = AsyncIO.work_threads[i].index_file_fd;
 65 |         }
 66 |     }
 67 | 
 68 |     public void put(String queueName, byte[] message) {
 69 |         long queueID = getQueueID(queueName);
 70 |         int threadID = (int) (queueID % IO_THREAD);
 71 | 
 72 |         IOThread ioThread = AsyncIO.work_threads[threadID];
 73 | 
 74 |         long which_queue_in_this_io_thread = queueID / IO_THREAD;
 75 |         long queue_offset = ioThread.queue_counter.getAndIncrement((int) which_queue_in_this_io_thread);
 76 | 
 77 |         long chunk_id = ((queue_offset / ioThread.batch_size) * (ioThread.chunk_size) +
 78 |                 (which_queue_in_this_io_thread * ioThread.batch_size) + queue_offset % ioThread.batch_size);
 79 | 
 80 |         long idx_file_offset = AsyncIO.INDEX_ENTRY_SIZE * chunk_id;
 81 |         int which_mapped_chunk = (int) (idx_file_offset / AsyncIO.INDEX_MAPPED_BLOCK_SIZE);
 82 | 
 83 |         long offset_in_mapped_chunk = idx_file_offset % AsyncIO.INDEX_MAPPED_BLOCK_SIZE;
 84 |         while (ioThread.index_file_memory_blocks[which_mapped_chunk] == null) {
 85 | //            System.out.println("I am full...");
 86 |             synchronized (ioThread.sync_blocks[which_mapped_chunk]) {
 87 |                 if (ioThread.index_file_memory_blocks[which_mapped_chunk] == null) {
 88 |                     try {
 89 |                         ioThread.sync_blocks[which_mapped_chunk].wait();
 90 |                     } catch (InterruptedException e) {
 91 |                         e.printStackTrace();
 92 |                     }
 93 |                 }
 94 |             }
 95 |         }
 96 | 
 97 |         if (message.length > AsyncIO.RAW_NORMAL_MESSAGE_SIZE) {
 98 |             ByteBuffer index_record = ByteBuffer.allocate(AsyncIO.INDEX_ENTRY_SIZE);
 99 |             byte[] message_ser_buf = MessageB64Serialization.SerializeBase64DecodingSkipIndex(message, message.length);
100 |             long large_msg_size = message_ser_buf.length;
101 |             long large_msg_offset = ioThread.data_file_size.getAndAdd(large_msg_size);
102 | 
103 |             index_record.putLong(4, large_msg_offset);
104 |             index_record.putLong(12, large_msg_size);
105 |             index_record.put(AsyncIO.INDEX_ENTRY_SIZE - 1, AsyncIO.LARGE_MESSAGE_MAGIC_CHAR);
106 |             try {
107 |                 ioThread.data_file_fd.write(ByteBuffer.wrap(message_ser_buf), large_msg_offset);
108 |             } catch (IOException e) {
109 |                 e.printStackTrace();
110 |             }
111 |             for(int i = 0 ; i < AsyncIO.INDEX_ENTRY_SIZE; i++){
112 |                 ioThread.index_file_memory_blocks[which_mapped_chunk][(int) (offset_in_mapped_chunk + i)] = index_record.get(i);
113 |             }
114 |             //absolutePut(ioThread.index_file_memory_blocks[which_mapped_chunk], (int) offset_in_mapped_chunk, index_record);
115 |         } else {
116 |             MessageB64Serialization.SerializeBase64DecodingSkipIndexOff(message, message.length,
117 |                     ioThread.index_file_memory_blocks[which_mapped_chunk], (int) offset_in_mapped_chunk);
118 |         }
119 | 
120 |         int write_times = ioThread.index_mapped_block_write_counter.incrementAndGet(which_mapped_chunk);
121 | 
122 |         if (write_times == AsyncIO.INDEX_BLOCK_WRITE_TIMES_TO_FULL) {
123 | //            System.out.println("send one remap task");
124 |             AsyncIO_task task = new AsyncIO_task(0);
125 |             asyncIO.submitIOTask(threadID, task);
126 |         }
127 | 
128 |     }
129 | 
130 |     static class MemoryPool {
131 |         Collection<byte[]> results = new ArrayList<>((int) 48);
132 |         byte[][] bytes = new byte[48][];
133 |         ByteBuffer byteBuffer = ByteBuffer.allocate(AsyncIO.INDEX_ENTRY_SIZE * CLUSTER_SIZE);
134 | 
135 |         MemoryPool() {
136 |             for (int i = 0; i < 48; i++) {
137 |                 bytes[i] = new byte[58];
138 |             }
139 |         }
140 |     }
141 | 
142 |     private static MemoryPool threadMemoryPool[] = new MemoryPool[128];
143 | 
144 |     static {
145 |         for (int i = 0; i < threadMemoryPool.length; i++) {
146 |             threadMemoryPool[i] = new MemoryPool();
147 |         }
148 |     }
149 | 
150 |     public Collection<byte[]> get(String queueName, long offset, long num) {
151 |         long currentTid = Thread.currentThread().getId();
152 |         if (!phase3) {
153 |             asyncIO.waitFinishIO();
154 |             asyncIO.waitMappedBuffers();
155 |             if (!tidSet.contains(currentTid)) {
156 |                 tidSet.add(currentTid);
157 |                 if (tidSet.size() > 10) {
158 |                     phase3 = true;
159 |                     System.out.println("start check phase");
160 |                 }
161 |             }
162 |         }
163 | 
164 |         MemoryPool memoryStruct = threadMemoryPool[(int) currentTid];
165 | 
166 | 
167 |         Collection<byte[]> results = memoryStruct.results;
168 |         results.clear();
169 | 
170 |         long queueID = getQueueID(queueName);
171 | 
172 |         int threadID = (int) (queueID % IO_THREAD);
173 |         IOThread ioThread = AsyncIO.work_threads[threadID];
174 |         int batch_size = ioThread.batch_size;
175 |         int which_queue_in_this_io_thread = (int) (queueID / IO_THREAD);
176 | 
177 |         long queue_count = ioThread.queue_counter.get(which_queue_in_this_io_thread);
178 |         long max_offset = Math.min(offset + num, queue_count);
179 | 
180 |         IOReaderThread ioReaderThread = AsyncIO.ioReaderThread[threadID];
181 | 
182 |         ByteBuffer index_record = memoryStruct.byteBuffer;
183 | 
184 |         int result_counter = 0;
185 | 
186 |         long chunk_size = ioThread.chunk_size;
187 |         long chunk_offset = (which_queue_in_this_io_thread * batch_size);
188 |         for (long queue_offset = offset; queue_offset < max_offset; ) {
189 |             long left_num = batch_size - (queue_offset % batch_size);
190 |             if (queue_offset + left_num > max_offset) {
191 |                 left_num = max_offset - queue_offset;
192 |             }
193 |             long chunk_id = ((queue_offset / batch_size) * chunk_size + chunk_offset + queue_offset % batch_size);
194 |             long idx_file_offset = AsyncIO.INDEX_ENTRY_SIZE * chunk_id;
195 | 
196 |             index_record.position(0);
197 |             index_record.limit((int) (left_num * AsyncIO.INDEX_ENTRY_SIZE));
198 | 
199 |             if (phase3) {
200 | 
201 |                 int which_chunk_in_io_file = (int) (idx_file_offset / IOReaderThread.MAPPED_BLOCK_SIZE);
202 | 
203 |                 int index_in_this_mapped_chunk = (int) (idx_file_offset % IOReaderThread.MAPPED_BLOCK_SIZE);
204 |                 for (int i = 0; i < index_record.limit(); i++) {
205 |                     index_record.put(i, ioReaderThread.index_mapped_buffers[which_chunk_in_io_file].get(index_in_this_mapped_chunk + i));
206 |                 }
207 |             } else {
208 |                 try {
209 |                     ioThread.index_file_fd.read(index_record, idx_file_offset);
210 |                 } catch (IOException e) {
211 |                     e.printStackTrace();
212 |                 }
213 |             }
214 | 
215 |             for (int element = 0; element < left_num; element++) {
216 |                 byte output_buf[];
217 |                 if (index_record.get(AsyncIO.INDEX_ENTRY_SIZE * (element + 1) - 1) == AsyncIO.LARGE_MESSAGE_MAGIC_CHAR) {
218 |                     long large_msg_size;
219 |                     long large_msg_offset;
220 |                     large_msg_offset = index_record.getLong(4 + AsyncIO.INDEX_ENTRY_SIZE * element);
221 |                     large_msg_size = index_record.getLong(12 + AsyncIO.INDEX_ENTRY_SIZE * element);
222 |                     ByteBuffer large_msg_buf = ByteBuffer.allocate((int) large_msg_size);
223 |                     try {
224 |                         ioThread.data_file_fd.read(large_msg_buf, large_msg_offset);
225 |                     } catch (IOException e) {
226 |                         e.printStackTrace();
227 |                     }
228 |                     output_buf = MessageB64Serialization.DeserializeBase64EncodingAddIndex(large_msg_buf.array(),
229 |                             0, large_msg_buf.position(), (int) (queue_offset + element));
230 |                     results.add(output_buf);
231 |                 } else {
232 | 
233 |                     MessageB64Serialization.DeserializeBase64EncodingAddIndexNormalSize(index_record.array(),
234 |                             element * AsyncIO.INDEX_ENTRY_SIZE, AsyncIO.INDEX_ENTRY_SIZE, (int) (queue_offset + element), memoryStruct.bytes[result_counter]);
235 |                     results.add(memoryStruct.bytes[result_counter]);
236 |                     result_counter++;
237 |                 }
238 | 
239 |             }
240 |             queue_offset += left_num;
241 |         }
242 |         return results;
243 |     }
244 | }
245 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/utils/MessageB64Serialization.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging.utils;
  2 | 
  3 | import java.nio.ByteBuffer;
  4 | 
  5 | public class MessageB64Serialization {
  6 |     final static int FIXED_PART_LEN = 10;  // assume FIXED_PART_LEN >=2, otherwise program is not correct
  7 |     final static int BASE64_INFO_LEN = 2;
  8 |     final static int INDEX_LEN = 4;
  9 |     final static int VARYING_VERIFY_LEN = 4;
 10 |     final static int COPY_PART_LEN = FIXED_PART_LEN - INDEX_LEN;
 11 | 
 12 |     final static byte[] magicArr = {'B', 'L', 'I', 'N', 'K'};
 13 |     final static int MAX_FIVE_BITS_INT = 0x1f;      // 31, for varyingLen Serialization
 14 | 
 15 |     // mutable: message is not usable later
 16 |     public static byte[] SerializeBase64Decoding(byte[] message, int len) {
 17 |         int serialize_len = len - FIXED_PART_LEN;
 18 |         byte padding_chars = (byte) ((4 - serialize_len % 4) % 4);
 19 |         // 1st: fixed part
 20 |         int serialization_base64_output_length = 3 * (serialize_len / 4 + (serialize_len % 4 == 0 ? 0 : 1));
 21 |         byte[] serialized = new byte[serialization_base64_output_length + FIXED_PART_LEN + 1];
 22 | 
 23 |         System.arraycopy(message, serialize_len, serialized, serialization_base64_output_length, FIXED_PART_LEN);
 24 |         // 2nd: padding for mutable message
 25 |         System.arraycopy(magicArr, 0, message, serialize_len, padding_chars);
 26 |         // 3rd: lookup with chromium base64 decoding
 27 |         ChromiumBase64.ChromiumBase64Decode(serialized, message, serialize_len + padding_chars);
 28 |         // 4th: record padding chars
 29 |         serialized[serialization_base64_output_length + FIXED_PART_LEN] = padding_chars;      // indicate number of padding chars
 30 |         return serialized;
 31 |     }
 32 | 
 33 |     public static byte[] DeserializeBase64Encoding(byte[] serialized, int start_off, int total_serialized_len) {
 34 |         int serialize_len = total_serialized_len - FIXED_PART_LEN - 1;
 35 | 
 36 |         // 1st: must be the accurate size to make it correct later without extra copying
 37 |         byte[] deserialized = new byte[serialize_len / 3 * 4 + FIXED_PART_LEN - serialized[start_off + total_serialized_len - 1]];
 38 |         // 2nd: deserialization
 39 |         int length = ChromiumBase64.ChromiumBase64Encode(deserialized, serialized, start_off, serialize_len);
 40 |         // 3rd: copy the fixed part
 41 |         System.arraycopy(serialized, serialize_len + start_off, deserialized, length - serialized[start_off +
 42 |                 total_serialized_len - 1], FIXED_PART_LEN);
 43 |         return deserialized;
 44 |     }
 45 | 
 46 |     public static void SerializeBase64DecodingSkipIndexOff(byte[] message, int len, byte[] serialized, int offset) {
 47 |         int serialize_len = len - FIXED_PART_LEN;
 48 |         // should be optimized to be always 0
 49 |         byte padding_chars = (byte) ((4 - serialize_len % 4) % 4);
 50 |         // 1st: fixed part
 51 |         int serialization_base64_output_length = 3 * (serialize_len / 4 + (serialize_len % 4 == 0 ? 0 : 1));
 52 | 
 53 |         System.arraycopy(message, serialize_len, serialized, offset + serialization_base64_output_length, BASE64_INFO_LEN);
 54 |         System.arraycopy(message, serialize_len + BASE64_INFO_LEN + INDEX_LEN, serialized,
 55 |                 offset + serialization_base64_output_length + BASE64_INFO_LEN, VARYING_VERIFY_LEN);
 56 | 
 57 | 
 58 |         // 2nd: padding for mutable message
 59 |         System.arraycopy(magicArr, 0, message, serialize_len, padding_chars);
 60 |         // 3rd: lookup with chromium base64 decoding
 61 |         ChromiumBase64.ChromiumBase64DecodeOff(serialized, offset, message, serialize_len + padding_chars);
 62 |         // 4th: record padding chars
 63 |         serialized[serialization_base64_output_length + COPY_PART_LEN] = padding_chars;      // indicate number of padding chars
 64 |     }
 65 | 
 66 |     public static byte[] SerializeBase64DecodingSkipIndex(byte[] message, int len) {
 67 |         int serialize_len = len - FIXED_PART_LEN;
 68 |         // should be optimized to be always 0
 69 |         byte padding_chars = (byte) ((4 - serialize_len % 4) % 4);
 70 |         // 1st: fixed part
 71 |         int serialization_base64_output_length = 3 * (serialize_len / 4 + (serialize_len % 4 == 0 ? 0 : 1));
 72 |         byte[] serialized = new byte[serialization_base64_output_length + COPY_PART_LEN + 1];
 73 | 
 74 |         System.arraycopy(message, serialize_len, serialized, serialization_base64_output_length, BASE64_INFO_LEN);
 75 |         System.arraycopy(message, serialize_len + BASE64_INFO_LEN + INDEX_LEN, serialized,
 76 |                 serialization_base64_output_length + BASE64_INFO_LEN, VARYING_VERIFY_LEN);
 77 | 
 78 | 
 79 |         // 2nd: padding for mutable message
 80 |         System.arraycopy(magicArr, 0, message, serialize_len, padding_chars);
 81 |         // 3rd: lookup with chromium base64 decoding
 82 |         ChromiumBase64.ChromiumBase64Decode(serialized, message, serialize_len + padding_chars);
 83 |         // 4th: record padding chars
 84 |         serialized[serialization_base64_output_length + COPY_PART_LEN] = padding_chars;      // indicate number of padding chars
 85 |         return serialized;
 86 |     }
 87 | 
 88 |     public static byte[] DeserializeBase64EncodingAddIndex(byte[] serialized, int start_off, int total_serialized_len, int index) {
 89 |         int serialize_len = total_serialized_len - COPY_PART_LEN - 1;
 90 | 
 91 |         // 1st: must be the accurate size to make it correct later without extra copying
 92 |         byte[] deserialized = new byte[serialize_len / 3 * 4 + FIXED_PART_LEN - serialized[start_off + total_serialized_len - 1]];
 93 |         // 2nd: deserialization
 94 |         int length = ChromiumBase64.ChromiumBase64Encode(deserialized, serialized, start_off, serialize_len);
 95 |         // 3rd: copy the fixed part
 96 |         int new_off = length - serialized[start_off + total_serialized_len - 1];
 97 |         //  1) remaining base64
 98 |         System.arraycopy(serialized, serialize_len + start_off, deserialized, new_off, BASE64_INFO_LEN);
 99 |         new_off += BASE64_INFO_LEN;
100 |         //  2) index:
101 |         deserialized[new_off] = (byte) (index & 0xff);
102 |         deserialized[new_off + 1] = (byte) ((index >> 8) & 0xff);
103 |         deserialized[new_off + 2] = (byte) ((index >> 16) & 0xff);
104 |         new_off += INDEX_LEN;
105 |         //  3) other verification parts
106 |         System.arraycopy(serialized, serialize_len + start_off + BASE64_INFO_LEN, deserialized, new_off, VARYING_VERIFY_LEN);
107 | 
108 |         return deserialized;
109 |     }
110 | 
111 | 
112 |     // only for normal size message (58 raw bytes)
113 |     public static void DeserializeBase64EncodingAddIndexNormalSize(byte[] serialized, int start_off, int total_serialized_len, int index, byte[] deserialized) {
114 |         int serialize_len = total_serialized_len - COPY_PART_LEN - 1;
115 | 
116 |         // 1st: must be the accurate size to make it correct later without extra copying
117 |         // 2nd: deserialization
118 |         int length = ChromiumBase64.ChromiumBase64Encode(deserialized, serialized, start_off, serialize_len);
119 |         // 3rd: copy the fixed part
120 |         int new_off = length - serialized[start_off + total_serialized_len - 1];
121 |         //  1) remaining base64
122 |         System.arraycopy(serialized, serialize_len + start_off, deserialized, new_off, BASE64_INFO_LEN);
123 |         new_off += BASE64_INFO_LEN;
124 |         //  2) index:
125 |         deserialized[new_off] = (byte) (index & 0xff);
126 |         deserialized[new_off + 1] = (byte) ((index >> 8) & 0xff);
127 |         deserialized[new_off + 2] = (byte) ((index >> 16) & 0xff);
128 |         new_off += INDEX_LEN;
129 |         //  3) other verification parts
130 |         System.arraycopy(serialized, serialize_len + start_off + BASE64_INFO_LEN, deserialized, new_off, VARYING_VERIFY_LEN);
131 |     }
132 | 
133 |     // mutable: message is not usable later
134 |     static int SerializeBase64DecodingByteBuffer(byte[] message, int len, ByteBuffer serialized) {
135 |         int serialize_len = len - FIXED_PART_LEN;
136 |         byte padding_chars = (byte) ((4 - serialize_len % 4) % 4);
137 | 
138 |         // 1st: fixed part
139 |         int serialization_base64_output_length = 3 * (serialize_len / 4 + (serialize_len % 4 == 0 ? 0 : 1));
140 |         for (int i = 0; i < FIXED_PART_LEN; i++) {
141 |             serialized.put(serialization_base64_output_length + i, message[serialize_len + i]);
142 |         }
143 |         // 2nd: padding for mutable message
144 |         System.arraycopy(magicArr, 0, message, serialize_len, padding_chars);
145 |         // 3rd: lookup with chromium base64 decoding
146 |         ChromiumBase64.ChromiumBase64DecodeByteBuffer(serialized, message, serialize_len + padding_chars);
147 |         // 4th: record padding chars
148 |         serialized.position(serialization_base64_output_length + FIXED_PART_LEN);
149 |         serialized.put(padding_chars);      // indicate number of padding chars
150 | 
151 |         serialized.flip();
152 |         return serialization_base64_output_length + FIXED_PART_LEN + 1;
153 |     }
154 | 
155 |     public static byte[] DeserializeBase64EncodingByteBuffer(ByteBuffer serialized, int total_serialized_len) {
156 |         int serialize_len = total_serialized_len - FIXED_PART_LEN - 1;
157 | 
158 |         // 1st: must be the accurate size to make it correct later without extra copying
159 |         byte padding_chars = serialized.get(total_serialized_len - 1);
160 |         byte[] deserialized = new byte[serialize_len / 3 * 4 + FIXED_PART_LEN - padding_chars];
161 |         // 2nd: deserialization
162 |         int length = ChromiumBase64.ChromiumBase64EncodeByteBuffer(deserialized, serialized, serialize_len);
163 |         // 3rd: copy the fixed part
164 |         for (int i = 0; i < FIXED_PART_LEN; i++) {
165 |             deserialized[i + length - padding_chars] = serialized.get(serialize_len + i);
166 |         }
167 |         return deserialized;
168 |     }
169 | 
170 |     // attention: 1) assume serialized allocated the same size as message, worst case same length
171 |     // 2) 2 extra bytes is for length of the raw message
172 |     public static ByteBuffer SerializeVaryingLen(byte[] message, int messageLen) {
173 |         // add the header to indicate raw message varying-length part size, similar to padding chars
174 |         ByteBuffer serializedBuffer = ByteBuffer.allocate(messageLen + 2);    // assume clear at the allocation
175 |         byte[] serialized = serializedBuffer.array();
176 | 
177 |         int serialize_len = messageLen - FIXED_PART_LEN;
178 |         int headerLen;
179 |         if (messageLen < 128) {
180 |             serialized[0] = (byte) (messageLen - FIXED_PART_LEN);
181 |             headerLen = 1;
182 |         } else {
183 |             int tmp = (messageLen - FIXED_PART_LEN);                 // tmp high bits are zeros
184 |             serialized[0] = (byte) ((tmp >>> 7) | 0x80); // assume < 32767
185 |             serialized[1] = (byte) (tmp & 0x7f);               // low 7 bits
186 |             headerLen = 2;
187 |         }
188 |         int next_extra_3bits_idx = 5 * serialize_len;
189 |         int next_5bits_idx = 0;
190 | 
191 |         // attention: message is not usable later
192 |         for (int i = 0; i < serialize_len; i++) {
193 |             message[i] = (byte) (message[i] >= 'a' ? message[i] - 'a' : message[i] - '0' + 26);
194 |         }
195 | 
196 |         // 1) construct the compressed part
197 |         for (int i = 0; i < serialize_len; i++) {
198 |             int cur_uchar = message[i] & 0xff;
199 |             int expand_uchar = cur_uchar < MAX_FIVE_BITS_INT ? (cur_uchar << 11) : (MAX_FIVE_BITS_INT << 11);
200 | 
201 |             int shift_bits = (next_5bits_idx & 0x7);
202 |             expand_uchar >>= shift_bits;
203 |             int idx = (next_5bits_idx >> 3) + headerLen;
204 |             serialized[idx] |= (expand_uchar >> 8);
205 |             serialized[idx + 1] |= (expand_uchar & 0xff);
206 |             next_5bits_idx += 5;
207 | 
208 |             if (cur_uchar >= MAX_FIVE_BITS_INT) {
209 |                 // do extra bits operations
210 |                 expand_uchar = ((cur_uchar - MAX_FIVE_BITS_INT) << 13);
211 |                 shift_bits = (next_extra_3bits_idx & 0x7);
212 |                 expand_uchar >>= shift_bits;
213 |                 // assume little-endian
214 |                 idx = (next_extra_3bits_idx >> 3) + headerLen;
215 |                 serialized[idx] |= (expand_uchar >> 8);
216 |                 serialized[idx + 1] |= (expand_uchar & 0xff);
217 |                 next_extra_3bits_idx += 3;
218 |             }
219 |         }
220 | 
221 |         // 2) left FIXED_PART_LEN, should use memcpy
222 |         int start_copy_byte_idx = headerLen + (next_extra_3bits_idx >> 3) + ((next_extra_3bits_idx & 0x7) != 0 ? 1 : 0);
223 | 
224 |         System.arraycopy(message, serialize_len, serialized, start_copy_byte_idx, FIXED_PART_LEN);
225 |         serializedBuffer.position(start_copy_byte_idx + FIXED_PART_LEN);
226 |         return serializedBuffer;
227 |     }
228 | 
229 |     public static byte[] DeserializeVaryingLen(ByteBuffer serializedBuffer) {
230 |         // get the length of varying part
231 |         byte[] serialized = serializedBuffer.array();
232 |         int varying_byte_len;
233 |         int headerLen;
234 |         if ((serialized[0] & 0x80) == 0) {
235 |             varying_byte_len = serialized[0];
236 |             headerLen = 1;
237 |         } else {
238 |             varying_byte_len = (((serialized[0] & 0x7f) << 0x7) + serialized[1]);
239 |             headerLen = 2;
240 |         }
241 |         int next_extra_3bits_idx = 5 * varying_byte_len;
242 |         int next_5bits_idx = 0;
243 | 
244 |         byte[] deserialized = new byte[varying_byte_len + FIXED_PART_LEN];
245 |         // deserialize
246 |         for (int i = 0; i < varying_byte_len; i++) {
247 |             int idx = (next_5bits_idx >> 3) + headerLen;
248 |             int value = ((serialized[idx] & 0xff) << 8) + (serialized[idx + 1] & 0xff);
249 |             value = (value >> (11 - (next_5bits_idx & 0x7))) & MAX_FIVE_BITS_INT;
250 |             if (value != MAX_FIVE_BITS_INT) {
251 |                 deserialized[i] = (byte) (value < 26 ? 'a' + value : value - 26 + '0');
252 |             } else {
253 |                 idx = (next_extra_3bits_idx >> 3) + headerLen;
254 |                 value = ((serialized[idx] & 0xff) << 8) + (serialized[idx + 1] & 0xff);
255 |                 value = (value >> (13 - (next_extra_3bits_idx & 0x7))) & 0x7;
256 |                 deserialized[i] = (byte) (value + '5');
257 |                 next_extra_3bits_idx += 3;
258 |             }
259 |             next_5bits_idx += 5;
260 |         }
261 | 
262 |         // 2) copy the fixed part
263 |         System.arraycopy(serialized, headerLen +
264 |                         (next_extra_3bits_idx >> 3) + ((next_extra_3bits_idx & 0x7) != 0 ? 1 : 0),
265 |                 deserialized, varying_byte_len, FIXED_PART_LEN);
266 |         return deserialized;
267 |     }
268 | }
269 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/DemoTester.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging;
  2 | 
  3 | import java.util.Collection;
  4 | import java.util.Random;
  5 | import java.util.concurrent.ConcurrentHashMap;
  6 | import java.util.concurrent.ConcurrentMap;
  7 | import java.util.concurrent.atomic.AtomicInteger;
  8 | import java.util.concurrent.atomic.AtomicLong;
  9 | 
 10 | //这是评测程序的一个demo版本，其评测逻辑与实际评测程序基本类似，但是比实际评测简单很多
 11 | //该评测程序主要便于选手在本地优化和调试自己的程序
 12 | 
 13 | public class DemoTester {
 14 |     private static final byte[] messageTemplate = new byte[]{
 15 |             'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C',
 16 |             'C', 'C', 'D', 'D', 'D', 'D', 'E', 'E', 'E', 'E',
 17 |             'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H',
 18 |             'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J',
 19 |             'K', 'K', 'K', 'K', 'L', 'L', 'L', 'L', 'M', 'M',
 20 |             'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O',
 21 |             'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R',
 22 |             'R', 'R', 'S', 'S', 'S', 'S', 'T', 'T', 'T', 'T',
 23 |             'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W',
 24 |             'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y',
 25 |             'Z', 'Z', 'Z', 'Z', 'a', 'a', 'a', 'a', 'b', 'b',
 26 |             'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd',
 27 |             'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g',
 28 |             'g', 'g', 'h', 'h', 'h', 'h', 'i', 'i', 'i', 'i',
 29 |             'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l',
 30 |             'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n',
 31 |             'o', 'o', 'o', 'o', 'p', 'p', 'p', 'p', 'q', 'q',
 32 |             'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's',
 33 |             't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v',
 34 |             'v', 'v', 'w', 'w', 'w', 'w', 'x', 'x', 'x', 'x',
 35 |             'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0',
 36 |             '0', '0', '1', '1', '1', '1', '2', '2', '2', '2',
 37 |             '3', '3', '3', '3', '4', '4', '4', '4', '5', '5',
 38 |             '5', '5', '6', '6', '6', '6', '7', '7', '7', '7',
 39 |             '8', '8', '8', '8', '9', '9', '9', '9',
 40 |     };
 41 | 
 42 |     private static final byte[] bigMessageTemplate;
 43 | 
 44 |     static {
 45 |         bigMessageTemplate = new byte[32768];
 46 |         for (int i = 0; i < bigMessageTemplate.length; i++) {
 47 |             bigMessageTemplate[i] = messageTemplate[i % messageTemplate.length];
 48 |         }
 49 |     }
 50 | 
 51 |     // used by producer, base64 string followed by fixed length bytes
 52 |     private static byte[] produce(int index, int base64Len) {
 53 |         byte[] bytes = new byte[base64Len + 8];
 54 |         System.arraycopy(bigMessageTemplate, 0, bytes, 0, base64Len);
 55 |         // construct index for indexing checking
 56 |         for (int i = 0; i < 4; i++) {
 57 |             bytes[(base64Len + i)] = ((byte) (index >>> 8 * i));
 58 |         }
 59 |         // hash code for correctness checking
 60 |         System.arraycopy(bigMessageTemplate, 0, bytes, base64Len + 4, 4);
 61 |         return bytes;
 62 |     }
 63 | 
 64 |     // used by consumer, index and integrity checking
 65 |     private static boolean verify(byte[] message, int index) {
 66 |         // 1st: for index checking
 67 |         int verifyingIdx = 0;
 68 |         for (int i = 0; i < 4; i++) {
 69 |             byte b = message[(message.length - 8 + i)];
 70 |             verifyingIdx += ((b & 0xFF) << 8 * i);
 71 |         }
 72 |         if (verifyingIdx != index) {
 73 |             System.out.println(verifyingIdx + " , " + index);
 74 |             System.out.println("index checking fail");
 75 |             return false;
 76 |         }
 77 |         // 2nd: for integrity checking
 78 |         int integrityStart = message.length - 4;
 79 |         for (int i = 0; i < 4; i++) {
 80 |             if (message[i] != message[integrityStart + i]) {
 81 |                 System.out.println(message.length);
 82 |                 System.out.println("idx:" + verifyingIdx);
 83 |                 System.out.println(message[i] + message[integrityStart + i]);
 84 |                 System.out.println("integrity checking fail");
 85 |                 return false;
 86 |             }
 87 |         }
 88 |         return true;
 89 |     }
 90 | 
 91 |     public static void main(String args[]) throws Exception {
 92 |         //评测相关配置
 93 |         //发送阶段的发送数量，也即发送阶段必须要在规定时间内把这些消息发送完毕方可
 94 |         int msgNum = 10000000;
 95 |         //发送阶段的最大持续时间，也即在该时间内，如果消息依然没有发送完毕，则退出评测
 96 |         int sendTime = 2000 * 1000;
 97 |         //消费阶段的最大持续时间，也即在该时间内，如果消息依然没有消费完毕，则退出评测
 98 |         int checkTime = 1600 * 1000;
 99 |         //队列的数量
100 |         int queueNum = 1000000; // attention: currently need to be exact 1000000
101 |         //正确性检测的次数
102 |         int checkNum = queueNum;
103 |         //消费阶段的总队列数量
104 |         int checkQueueNum = queueNum / 10;
105 |         //发送的线程数量
106 |         int sendTsNum = 10;
107 |         //消费的线程数量
108 |         int checkTsNum = 10;        // attention: currently should be exact 10, for the synchronization purpose
109 | 
110 |         ConcurrentMap<String, AtomicInteger> queueNumMap = new ConcurrentHashMap<>();
111 |         for (int i = 0; i < queueNum; i++) {
112 |             queueNumMap.put("Queue-" + i, new AtomicInteger(0));
113 |         }
114 | 
115 |         QueueStore queueStore = null;
116 | 
117 |         try {
118 |             Class queueStoreClass = Class.forName("io.openmessaging.DefaultQueueStoreImpl");
119 |             queueStore = (QueueStore) queueStoreClass.newInstance();
120 |         } catch (Throwable t) {
121 |             t.printStackTrace();
122 |             System.exit(-1);
123 |         }
124 | 
125 |         //Step1: 发送消息
126 |         long sendStart = System.currentTimeMillis();
127 |         long maxTimeStamp = System.currentTimeMillis() + sendTime;
128 |         AtomicLong sendCounter = new AtomicLong(0);
129 |         Thread[] sends = new Thread[sendTsNum];
130 |         for (int i = 0; i < sendTsNum; i++) {
131 |             sends[i] = new Thread(new Producer(queueStore, i, maxTimeStamp, msgNum, sendCounter, queueNumMap));
132 |         }
133 |         for (int i = 0; i < sendTsNum; i++) {
134 |             sends[i].start();
135 |         }
136 |         for (int i = 0; i < sendTsNum; i++) {
137 |             sends[i].join();
138 |         }
139 |         long sendSend = System.currentTimeMillis();
140 |         System.out.printf("Send: %d ms Num:%d\n", sendSend - sendStart, sendCounter.get());
141 |         long maxCheckTime = System.currentTimeMillis() + checkTime;
142 | 
143 |         //Step2: 索引的正确性校验
144 |         long indexCheckStart = System.currentTimeMillis();
145 |         AtomicLong indexCheckCounter = new AtomicLong(0);
146 |         Thread[] indexChecks = new Thread[checkTsNum];
147 |         for (int i = 0; i < sendTsNum; i++) {
148 |             indexChecks[i] = new Thread(new IndexChecker(queueStore, i, maxCheckTime, checkNum, indexCheckCounter, queueNumMap));
149 |         }
150 |         for (int i = 0; i < sendTsNum; i++) {
151 |             indexChecks[i].start();
152 |         }
153 |         for (int i = 0; i < sendTsNum; i++) {
154 |             indexChecks[i].join();
155 |         }
156 |         long indexCheckEnd = System.currentTimeMillis();
157 |         System.out.printf("Index Check: %d ms Num:%d\n", indexCheckEnd - indexCheckStart, indexCheckCounter.get());
158 | 
159 |         //Step3: 消费消息，并验证顺序性
160 |         long checkStart = System.currentTimeMillis();
161 |         Random random = new Random();
162 |         AtomicLong checkCounter = new AtomicLong(0);
163 |         Thread[] checks = new Thread[checkTsNum];
164 |         for (int i = 0; i < sendTsNum; i++) {
165 |             int eachCheckQueueNum = checkQueueNum / checkTsNum;
166 |             ConcurrentMap<String, AtomicInteger> offsets = new ConcurrentHashMap<>();
167 |             for (int j = 0; j < eachCheckQueueNum; j++) {
168 |                 String queueName = "Queue-" + random.nextInt(queueNum);
169 |                 while (offsets.containsKey(queueName)) {
170 |                     queueName = "Queue-" + random.nextInt(queueNum);
171 |                 }
172 |                 offsets.put(queueName, queueNumMap.get(queueName));
173 |             }
174 |             checks[i] = new Thread(new Consumer(queueStore, i, maxCheckTime, checkCounter, offsets));
175 |         }
176 |         for (int i = 0; i < sendTsNum; i++) {
177 |             checks[i].start();
178 |         }
179 |         for (int i = 0; i < sendTsNum; i++) {
180 |             checks[i].join();
181 |         }
182 |         long checkEnd = System.currentTimeMillis();
183 |         System.out.printf("Check: %d ms Num: %d\n", checkEnd - checkStart, checkCounter.get());
184 | 
185 |         //评测结果
186 |         System.out.printf("Tps:%f\n", ((sendCounter.get() + checkCounter.get() + indexCheckCounter.get()) + 0.1) * 1000 / ((sendSend - sendStart) + (checkEnd - checkStart) + (indexCheckEnd - indexCheckStart)));
187 |     }
188 | 
189 |     static class Producer implements Runnable {
190 | 
191 |         private AtomicLong counter;
192 |         private ConcurrentMap<String, AtomicInteger> queueCounter;
193 |         private long maxMsgNum;
194 |         private QueueStore queueStore;
195 |         private int number;
196 |         private long maxTimeStamp;
197 | 
198 |         public Producer(QueueStore queueStore, int number, long maxTimeStamp, int maxMsgNum, AtomicLong counter, ConcurrentMap<String, AtomicInteger> queueCounter) {
199 |             this.counter = counter;
200 |             this.maxMsgNum = maxMsgNum;
201 |             this.queueCounter = queueCounter;
202 |             this.number = number;
203 |             this.queueStore = queueStore;
204 |             this.maxTimeStamp = maxTimeStamp;
205 |         }
206 | 
207 |         @Override
208 |         public void run() {
209 |             long count;
210 |             while ((count = counter.getAndIncrement()) < maxMsgNum && System.currentTimeMillis() <= maxTimeStamp) {
211 |                 try {
212 |                     String queueName = "Queue-" + count % queueCounter.size();
213 |                     synchronized (queueCounter.get(queueName)) {
214 |                         int inQueueOff = queueCounter.get(queueName).getAndIncrement();
215 |                         // currently do not support < 50 ...
216 |                         queueStore.put(queueName, produce(inQueueOff, inQueueOff % 100 == 1 ? 1024 : 50));
217 | //                        queueStore.put(queueName, produce(inQueueOff, 50));
218 |                     }
219 |                 } catch (Throwable t) {
220 |                     t.printStackTrace();
221 |                     System.exit(-1);
222 |                 }
223 |             }
224 |         }
225 |     }
226 | 
227 |     static class IndexChecker implements Runnable {
228 | 
229 |         private AtomicLong counter;
230 |         private long maxMsgNum;
231 |         private QueueStore queueStore;
232 |         private long maxTimeStamp;
233 |         private int number;
234 |         private ConcurrentMap<String, AtomicInteger> queueCounter;
235 | 
236 |         public IndexChecker(QueueStore queueStore, int number, long maxTimeStamp, int maxMsgNum, AtomicLong counter, ConcurrentMap<String, AtomicInteger> queueCounter) {
237 |             this.counter = counter;
238 |             this.maxMsgNum = maxMsgNum;
239 |             this.queueStore = queueStore;
240 |             this.number = number;
241 |             this.queueCounter = queueCounter;
242 |             this.maxTimeStamp = maxTimeStamp;
243 |         }
244 | 
245 |         @Override
246 |         public void run() {
247 |             Random random = new Random();
248 |             while (counter.getAndIncrement() < maxMsgNum && System.currentTimeMillis() <= maxTimeStamp) {
249 |                 try {
250 |                     String queueName = "Queue-" + random.nextInt(queueCounter.size());
251 |                     int index = random.nextInt(queueCounter.get(queueName).get()) - 10;
252 |                     if (index < 0) index = 0;
253 |                     Collection<byte[]> msgs = queueStore.get(queueName, index, 10);
254 | 
255 |                     int cnt = 0;
256 |                     for (byte[] msg : msgs) {
257 |                         if (!verify(msg, index + cnt)) {
258 |                             System.out.println("Check error");
259 |                             System.out.println(queueName);
260 |                             System.exit(-1);
261 |                         }
262 |                         cnt++;
263 |                     }
264 |                 } catch (Throwable t) {
265 |                     t.printStackTrace();
266 |                     System.exit(-1);
267 | 
268 |                 }
269 |             }
270 |         }
271 |     }
272 | 
273 |     static class Consumer implements Runnable {
274 | 
275 |         private AtomicLong counter;
276 |         private QueueStore queueStore;
277 |         private ConcurrentMap<String, AtomicInteger> offsets;
278 |         private long maxTimeStamp;
279 |         private int number;
280 | 
281 |         public Consumer(QueueStore queueStore, int number, long maxTimeStamp, AtomicLong counter, ConcurrentMap<String, AtomicInteger> offsets) {
282 |             this.counter = counter;
283 |             this.queueStore = queueStore;
284 |             this.offsets = offsets;
285 |             this.maxTimeStamp = maxTimeStamp;
286 |             this.number = number;
287 |         }
288 | 
289 |         @Override
290 |         public void run() {
291 |             ConcurrentMap<String, AtomicInteger> pullOffsets = new ConcurrentHashMap<>();
292 |             for (String queueName : offsets.keySet()) {
293 |                 pullOffsets.put(queueName, new AtomicInteger(0));
294 |             }
295 |             while (pullOffsets.size() > 0 && System.currentTimeMillis() <= maxTimeStamp) {
296 |                 try {
297 |                     for (String queueName : pullOffsets.keySet()) {
298 |                         int index = pullOffsets.get(queueName).get();
299 |                         Collection<byte[]> msgs = queueStore.get(queueName, index, 10);
300 |                         if (msgs != null && msgs.size() > 0) {
301 |                             pullOffsets.get(queueName).getAndAdd(msgs.size());
302 |                             int cnt = 0;
303 |                             for (byte[] msg : msgs) {
304 |                                 if (!verify(msg, index + cnt)) {
305 |                                     System.out.println("Check error");
306 |                                     System.exit(-1);
307 |                                 }
308 |                                 cnt++;
309 |                             }
310 | 
311 |                             counter.addAndGet(msgs.size());
312 |                         }
313 |                         if (msgs == null || msgs.size() < 10) {
314 |                             if (pullOffsets.get(queueName).get() != offsets.get(queueName).get()) {
315 |                                 System.out.printf("Queue Number Error");
316 |                                 System.exit(-1);
317 |                             }
318 |                             pullOffsets.remove(queueName);
319 |                         }
320 |                     }
321 |                 } catch (Throwable t) {
322 |                     t.printStackTrace();
323 |                     System.exit(-1);
324 |                 }
325 |             }
326 |         }
327 |     }
328 | }
329 | 
330 | 
331 | 


--------------------------------------------------------------------------------
/src/main/java/io/openmessaging/utils/ChromiumBase64.java:
--------------------------------------------------------------------------------
  1 | package io.openmessaging.utils;
  2 | 
  3 | 
  4 | import java.nio.ByteBuffer;
  5 | import java.nio.ByteOrder;
  6 | 
  7 | public class ChromiumBase64 {
  8 |     final private static byte[] e0 = {
  9 |             'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C',
 10 |             'C', 'C', 'D', 'D', 'D', 'D', 'E', 'E', 'E', 'E',
 11 |             'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H',
 12 |             'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J',
 13 |             'K', 'K', 'K', 'K', 'L', 'L', 'L', 'L', 'M', 'M',
 14 |             'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O',
 15 |             'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R',
 16 |             'R', 'R', 'S', 'S', 'S', 'S', 'T', 'T', 'T', 'T',
 17 |             'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W',
 18 |             'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y',
 19 |             'Z', 'Z', 'Z', 'Z', 'a', 'a', 'a', 'a', 'b', 'b',
 20 |             'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd',
 21 |             'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g',
 22 |             'g', 'g', 'h', 'h', 'h', 'h', 'i', 'i', 'i', 'i',
 23 |             'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l',
 24 |             'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n',
 25 |             'o', 'o', 'o', 'o', 'p', 'p', 'p', 'p', 'q', 'q',
 26 |             'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's',
 27 |             't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v',
 28 |             'v', 'v', 'w', 'w', 'w', 'w', 'x', 'x', 'x', 'x',
 29 |             'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0',
 30 |             '0', '0', '1', '1', '1', '1', '2', '2', '2', '2',
 31 |             '3', '3', '3', '3', '4', '4', '4', '4', '5', '5',
 32 |             '5', '5', '6', '6', '6', '6', '7', '7', '7', '7',
 33 |             '8', '8', '8', '8', '9', '9', '9', '9', '+', '+',
 34 |             '+', '+', '/', '/', '/', '/'
 35 |     };
 36 | 
 37 |     final private static byte[] e1 = {
 38 |             'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
 39 |             'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
 40 |             'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
 41 |             'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
 42 |             'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
 43 |             'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
 44 |             '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F',
 45 |             'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
 46 |             'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
 47 |             'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
 48 |             'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
 49 |             'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3',
 50 |             '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B',
 51 |             'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
 52 |             'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
 53 |             'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
 54 |             'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
 55 |             'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
 56 |             '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
 57 |             '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
 58 |             'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
 59 |             'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b',
 60 |             'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
 61 |             'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
 62 |             'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5',
 63 |             '6', '7', '8', '9', '+', '/'
 64 |     };
 65 | 
 66 |     final private static byte e2[] = {
 67 |             'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
 68 |             'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
 69 |             'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
 70 |             'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
 71 |             'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
 72 |             'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
 73 |             '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F',
 74 |             'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
 75 |             'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
 76 |             'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
 77 |             'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
 78 |             'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3',
 79 |             '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B',
 80 |             'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
 81 |             'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
 82 |             'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
 83 |             'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
 84 |             'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
 85 |             '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
 86 |             '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
 87 |             'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
 88 |             'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b',
 89 |             'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
 90 |             'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
 91 |             'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5',
 92 |             '6', '7', '8', '9', '+', '/'
 93 |     };
 94 | 
 95 |     /* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */
 96 |     final private static int[] d0 = {
 97 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
 98 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
 99 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
100 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
101 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
102 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
103 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
104 |             0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc,
105 |             0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
106 |             0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff,
107 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
108 |             0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
109 |             0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
110 |             0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
111 |             0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
112 |             0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
113 |             0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
114 |             0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
115 |             0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
116 |             0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
117 |             0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff,
118 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
119 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
120 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
121 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
122 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
123 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
124 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
125 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
126 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
127 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
128 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
129 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
130 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
131 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
132 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
133 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
134 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
135 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
136 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
137 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
138 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
139 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff
140 |     };
141 | 
142 |     final private static int[] d1 = {
143 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
144 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
145 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
146 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
147 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
148 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
149 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
150 |             0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003,
151 |             0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
152 |             0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff,
153 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
154 |             0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
155 |             0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
156 |             0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
157 |             0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
158 |             0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
159 |             0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
160 |             0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
161 |             0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
162 |             0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
163 |             0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff,
164 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
165 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
166 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
167 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
168 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
169 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
170 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
171 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
172 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
173 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
174 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
175 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
176 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
177 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
178 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
179 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
180 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
181 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
182 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
183 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
184 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
185 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff
186 |     };
187 | 
188 | 
189 |     final private static int[] d2 = {
190 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
191 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
192 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
193 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
194 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
195 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
196 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
197 |             0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00,
198 |             0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
199 |             0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff,
200 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
201 |             0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
202 |             0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
203 |             0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
204 |             0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
205 |             0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
206 |             0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
207 |             0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
208 |             0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
209 |             0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
210 |             0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff,
211 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
212 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
213 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
214 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
215 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
216 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
217 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
218 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
219 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
220 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
221 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
222 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
223 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
224 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
225 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
226 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
227 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
228 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
229 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
230 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
231 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
232 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff
233 |     };
234 | 
235 | 
236 |     final private static int[] d3 = {
237 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
238 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
239 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
240 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
241 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
242 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
243 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
244 |             0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000,
245 |             0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
246 |             0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff,
247 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000,
248 |             0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
249 |             0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
250 |             0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
251 |             0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
252 |             0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
253 |             0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
254 |             0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
255 |             0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
256 |             0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
257 |             0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff,
258 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
259 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
260 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
261 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
262 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
263 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
264 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
265 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
266 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
267 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
268 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
269 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
270 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
271 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
272 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
273 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
274 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
275 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
276 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
277 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
278 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff,
279 |             0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff
280 |     };
281 | 
282 |     static int ChromiumBase64Encode(byte[] dest, byte[] str, int start_off, int len) {
283 |         int size_p = 0;
284 |         /* unsigned here is important! */ // for correct offset loopup
285 |         byte t1, t2, t3;
286 |         for (int i = 0; i < len - 2; i += 3) {
287 |             t1 = str[start_off + i];
288 |             t2 = str[start_off + i + 1];
289 |             t3 = str[start_off + i + 2];
290 |             dest[size_p] = e0[t1 & 0xff];
291 |             dest[size_p + 1] = e1[(((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)) & 0xff];
292 |             dest[size_p + 2] = e1[(((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)) & 0xff];
293 |             dest[size_p + 3] = e2[t3 & 0xff];
294 |             size_p += 4;
295 |         }
296 |         return size_p;
297 |     }
298 | 
299 |     static void ChromiumBase64Decode(byte[] dest, byte[] src, int len) {
300 |         int chunks = len / 4;
301 |         int size_p = 0;
302 |         int offset_y = 0;
303 |         for (int i = 0; i < chunks; ++i, offset_y += 4) {
304 |             int x = d0[src[offset_y]] | d1[src[1 + offset_y]] | d2[src[2 + offset_y]] | d3[src[3 + offset_y]];
305 |             dest[size_p++] = (byte) ((x) & 0xff);  //x[0]
306 |             dest[size_p++] = (byte) ((x >>> 8) & 0xff);  //x[1]
307 |             dest[size_p++] = (byte) ((x >>> 16) & 0xff);   //x[2]
308 |         }
309 |     }
310 | 
311 |     static void ChromiumBase64DecodeOff(byte[] dest, int off, byte[] src, int len) {
312 |         int chunks = len / 4;
313 |         int size_p = off;
314 |         int offset_y = 0;
315 |         for (int i = 0; i < chunks; ++i, offset_y += 4) {
316 |             int x = d0[src[offset_y]] | d1[src[1 + offset_y]] | d2[src[2 + offset_y]] | d3[src[3 + offset_y]];
317 |             dest[size_p++] = (byte) ((x) & 0xff);  //x[0]
318 |             dest[size_p++] = (byte) ((x >>> 8) & 0xff);  //x[1]
319 |             dest[size_p++] = (byte) ((x >>> 16) & 0xff);   //x[2]
320 |         }
321 |     }
322 | 
323 |     // assume reusable bytebuffer and already flipped
324 |     static int ChromiumBase64EncodeByteBuffer(byte[] dest, ByteBuffer srcByteBuffer, int len) {
325 |         int size_p = 0;
326 |         /* unsigned here is important! */ // for correct offset loopup
327 |         byte t1, t2, t3;
328 |         for (int i = 0; i < len - 2; i += 3) {
329 |             t1 = srcByteBuffer.get(i);
330 |             t2 = srcByteBuffer.get(i + 1);
331 |             t3 = srcByteBuffer.get(i + 2);
332 |             dest[size_p] = e0[t1 & 0xff];
333 |             dest[size_p + 1] = e1[(((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)) & 0xff];
334 |             dest[size_p + 2] = e1[(((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)) & 0xff];
335 |             dest[size_p + 3] = e2[t3 & 0xff];
336 |             size_p += 4;
337 |         }
338 |         return size_p;
339 |     }
340 | 
341 |     // assume reuse DirectByteBuffer reusable
342 |     static void ChromiumBase64DecodeByteBuffer(ByteBuffer dest, byte[] src, int len) {
343 |         dest.clear(); // for writing
344 | 
345 |         int chunks = len / 4;
346 |         int offset_y = 0;
347 |         for (int i = 0; i < chunks; ++i, offset_y += 4) {
348 |             int x = d0[src[offset_y]] | d1[src[1 + offset_y]] | d2[src[2 + offset_y]] | d3[src[3 + offset_y]];
349 |             dest.put((byte) ((x >>> 0) & 0xff));
350 |             dest.put((byte) ((x >>> 8) & 0xff));
351 |             dest.put((byte) ((x >>> 16) & 0xff));
352 |         }
353 |     }
354 | 
355 |     private static void TestEndian() {
356 |         int x = 1;
357 |         if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
358 |             System.out.println("Big-endian");
359 |         } else {
360 |             System.out.println("Little-endian");
361 |         }
362 |         for (int i = 0; i < 4; i++) {
363 |             System.out.println("mem addr:[" + i + "]: " + (x >>> ((3 - i) * 8) & 0xff));
364 |         }
365 |     }
366 | 
367 |     private static void TestRawArr(byte[] chars) {
368 |         // 1st: serialization
369 |         byte[] serialized = new byte[chars.length / 4 * 3];
370 |         ChromiumBase64Decode(serialized, chars, chars.length);
371 | 
372 |         System.out.println("serialized: " + serialized.length);
373 |         for (byte aChar : serialized) {
374 |             System.out.print(Integer.toString(aChar & 0xff, 16));
375 |             System.out.print(",");
376 |         }
377 |         System.out.println();
378 |         byte[] tmp = new byte[]{0x6a, (byte) 0xf7, 0x1d, 0x7b, 0x4d, 0x40, 0x73, 0x6d, (byte) 0xf8};
379 |         System.out.println("ground truth of serialized: " + tmp.length);
380 |         for (byte ch : tmp) {
381 |             System.out.print(Integer.toString(ch & 0xff, 16) + ",");
382 |         }
383 |         System.out.println();
384 | 
385 |         // 2nd: deserialization
386 |         byte[] deserialized = new byte[chars.length];
387 |         ChromiumBase64Encode(deserialized, serialized, 0, serialized.length);
388 |         System.out.println(new String(chars));
389 |         System.out.println(new String(deserialized));
390 |     }
391 | 
392 |     private static void TestByteBuffer(byte[] chars) {
393 |         // 1st: serialization
394 |         ByteBuffer serialized = ByteBuffer.allocateDirect(1024);
395 |         ChromiumBase64DecodeByteBuffer(serialized, chars, chars.length);
396 | 
397 |         System.out.println();
398 |         byte[] tmp = new byte[]{0x6a, (byte) 0xf7, 0x1d, 0x7b, 0x4d, 0x40, 0x73, 0x6d, (byte) 0xf8};
399 |         System.out.println("ground truth of serialized: " + tmp.length);
400 |         for (byte ch : tmp) {
401 |             System.out.print(Integer.toString(ch & 0xff, 16) + ",");
402 |         }
403 |         System.out.println();
404 | 
405 |         // 2nd: deserialization
406 |         byte[] deserialized = new byte[chars.length];
407 |         serialized.flip();    // for later reading
408 |         ChromiumBase64EncodeByteBuffer(deserialized, serialized, serialized.limit());
409 |         System.out.println(new String(chars));
410 |         System.out.println(new String(deserialized));
411 |     }
412 | 
413 |     public static void main(String[] args) {
414 |         byte[] chars = {'a', 'v', 'c', 'd', 'e', '0', '1', 'A', 'c', '2', '3', '4'};
415 |         TestRawArr(chars);
416 |         TestByteBuffer(chars);
417 |     }
418 | }
419 | 


--------------------------------------------------------------------------------