├── .gitignore ├── config.yaml ├── pom.xml └── src ├── main └── java │ └── kr │ └── ac │ └── yonsei │ └── delab │ └── addb_loader │ ├── Config.java │ ├── Global.java │ ├── Loader.java │ ├── Main.java │ ├── TableConfig.java │ ├── jedis_loader │ ├── JedisLoader.java │ ├── JedisManager.java │ ├── PipelineWorker.java │ ├── RNWorker.java │ ├── RedisClient.java │ └── RedisNode.java │ └── lettuce_loader │ ├── LettuceLoader.java │ ├── PipelineManager.java │ ├── ReactiveFileReader.java │ └── TaoPipelineManager.java └── test └── java └── kr └── ac └── yonsei └── delab └── addb_loader └── AppTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea/ 3 | *.tbl 4 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | loader: "lettuce" 2 | tables: 3 | nation: 4 | id: 1 5 | file: "data/nation.tbl" 6 | partitionCols: 7 | - "1" 8 | region: 9 | id: 2 10 | file: "data/region.tbl" 11 | partitionCols: 12 | - "1" 13 | part: 14 | id: 3 15 | file: "data/part.tbl" 16 | partitionCols: 17 | - "4" 18 | supplier: 19 | id: 4 20 | file: "data/supplier.tbl" 21 | partitionCols: 22 | - "4" 23 | partsupp: 24 | id: 5 25 | file: "data/partsupp.tbl" 26 | partitionCols: 27 | - "2" 28 | customer: 29 | id: 6 30 | file: "data/customer.tbl" 31 | partitionCols: 32 | - "4" 33 | orders: 34 | id: 7 35 | file: "data/orders.tbl" 36 | partitionCols: 37 | - "5" 38 | lineitem: 39 | id: 8 40 | file: "data/lineitem.tbl" 41 | partitionCols: 42 | - "3" 43 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | kr.ac.yonsei.delab 6 | addb_loader 7 | 0.0.1-SNAPSHOT 8 | jar 9 | 10 | addb_loader 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | 19 | 20 | commons-cli 21 | commons-cli 22 | 1.3.1 23 | 24 | 25 | 26 | org.yaml 27 | snakeyaml 28 | 1.21 29 | 30 | 31 | 32 | org.projectlombok 33 | lombok 34 | 1.18.8 35 | 36 | 37 | 38 | org.apache.commons 39 | commons-pool2 40 | 2.4.3 41 | 42 | 43 | 44 | org.slf4j 45 | slf4j-simple 46 | 1.7.26 47 | 48 | 49 | com.github.javasync 50 | AsyncFileRw 51 | 1.1.2 52 | 53 | 54 | 55 | junit 56 | junit 57 | 3.8.1 58 | test 59 | 60 | 61 | kr.ac.yonsei.delab 62 | addb-jedis 63 | 0.0.2 64 | 65 | 66 | kr.ac.yonsei.delab 67 | addb-lettuce 68 | 0.0.1 69 | 70 | 71 | io.netty 72 | netty-transport-native-epoll 73 | 4.1.38.Final 74 | linux-x86_64 75 | true 76 | 77 | 78 | io.netty 79 | netty-transport-native-kqueue 80 | 4.1.38.Final 81 | osx-x86_64 82 | false 83 | 84 | 85 | 86 | 87 | 88 | org.codehaus.mojo 89 | exec-maven-plugin 90 | 1.6.0 91 | 92 | kr.ac.yonsei.delab.addb_loader.Main 93 | 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-jar-plugin 98 | 3.0.2 99 | 100 | 101 | 102 | kr.ac.yonsei.delab.addb_loader.Main 103 | true 104 | true 105 | kr.ac.yonsei.delab.addb_loader 106 | 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-assembly-plugin 113 | 3.1.0 114 | 115 | 116 | 117 | kr.ac.yonsei.delab.addb_loader.Main 118 | true 119 | true 120 | kr.ac.yonsei.delab.addb_loader 121 | 122 | 123 | 124 | jar-with-dependencies 125 | 126 | 127 | 128 | 129 | package 130 | 131 | single 132 | 133 | 134 | 135 | 136 | 137 | org.apache.maven.plugins 138 | maven-compiler-plugin 139 | 3.3 140 | 141 | 1.8 142 | 1.8 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/Config.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | import java.util.Map; 7 | 8 | public class Config { 9 | @Getter @Setter String loader; 10 | @Getter @Setter Map tables; 11 | 12 | @Override 13 | public String toString() { 14 | StringBuilder builder = new StringBuilder(); 15 | builder.append("==== Config ====\n") 16 | .append("1. Loader\n") 17 | .append(loader).append("\n") 18 | .append("2. Tables\n") 19 | .append(tables).append("\n") 20 | .append("----------------\n") 21 | .append("================"); 22 | return builder.toString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/Global.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader; 2 | 3 | public class Global { 4 | public static Config config; 5 | public static String host; 6 | public static int port; 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/Loader.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader; 2 | 3 | public interface Loader { 4 | public void start(); 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/Main.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader; 2 | 3 | import kr.ac.yonsei.delab.addb_loader.jedis_loader.JedisLoader; 4 | import kr.ac.yonsei.delab.addb_loader.lettuce_loader.LettuceLoader; 5 | import org.apache.commons.cli.*; 6 | import org.yaml.snakeyaml.Yaml; 7 | import org.yaml.snakeyaml.constructor.Constructor; 8 | 9 | import java.io.File; 10 | import java.io.FileInputStream; 11 | import java.io.FileNotFoundException; 12 | import java.io.InputStream; 13 | 14 | public class Main { 15 | public static void main(String[] args) { 16 | /* Command type : ./Loader.jar -c config.yaml -t lineitem -h 255.255.255.0 -p 8000 */ 17 | /* Parses arguments... */ 18 | Option configOption = new Option( 19 | "c", "config", true, "Config file path"); 20 | configOption.setRequired(true); 21 | Option tableOption = new Option( 22 | "t", "table", true, "Table name in config file"); 23 | tableOption.setRequired(true); 24 | Option hostOption = new Option( 25 | "h", "host", true, "Host of Redis master node"); 26 | hostOption.setRequired(true); 27 | Option portOption = new Option( 28 | "p", "port", true, "Port of Redis master node"); 29 | portOption.setRequired(true); 30 | Options options = new Options() 31 | .addOption(configOption) 32 | .addOption(tableOption) 33 | .addOption(hostOption) 34 | .addOption(portOption); 35 | 36 | CommandLineParser parser = new DefaultParser(); 37 | CommandLine cmd; 38 | try { 39 | cmd = parser.parse(options, args); 40 | } catch (ParseException e) { 41 | HelpFormatter formatter = new HelpFormatter(); 42 | formatter.printHelp("loader", options); 43 | throw new RuntimeException(); 44 | } 45 | 46 | String configFileName = cmd.getOptionValue("config"); 47 | String tableName = cmd.getOptionValue("table"); 48 | String host = cmd.getOptionValue("host"); 49 | String port = cmd.getOptionValue("port"); 50 | 51 | /* Parses config file... */ 52 | Yaml yaml = new Yaml(new Constructor(Config.class)); 53 | InputStream inputStream = null; 54 | try { 55 | inputStream = new FileInputStream(new File(configFileName)); 56 | } catch (FileNotFoundException e) { 57 | throw new RuntimeException("Config file is not founded"); 58 | } 59 | Global.config = yaml.loadAs(inputStream, Config.class); 60 | Global.host = host; 61 | Global.port = Integer.valueOf(port); 62 | 63 | TableConfig tableConfig = Global.config.tables.get(tableName); 64 | if (tableConfig == null) { 65 | throw new RuntimeException("Table name is not founded on config.yaml"); 66 | } 67 | 68 | System.out.println(Global.config); 69 | 70 | /* Start Loader... */ 71 | Loader loader; 72 | if (Global.config.getLoader().equalsIgnoreCase("jedis")) { 73 | loader = JedisLoader.create( 74 | Integer.toString(tableConfig.getId()), 75 | tableConfig.getFile(), 76 | tableConfig.getPartitionCols() 77 | ); 78 | } else if (Global.config.getLoader().equalsIgnoreCase("lettuce")) { 79 | // TODO(totoro): Implements Lettuce Loader 80 | System.out.println("TODO(totoro): Implements Lettuce Loader..."); 81 | loader = LettuceLoader.create( 82 | Integer.toString(tableConfig.getId()), 83 | tableConfig.getFile(), 84 | tableConfig.getPartitionCols() 85 | ); 86 | } else { 87 | throw new RuntimeException("Loader must be either 'jedis' or 'lettuce'."); 88 | } 89 | loader.start(); 90 | } 91 | } -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/TableConfig.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | public class TableConfig { 7 | @Getter @Setter private int id; 8 | @Getter @Setter private String file; 9 | @Getter @Setter private String[] partitionCols; 10 | 11 | @Override 12 | public String toString() { 13 | StringBuilder builder = new StringBuilder(); 14 | builder.append("id: ").append(id).append(" | ") 15 | .append("file: ").append(file).append(" | ") 16 | .append("partitionCols: [").append(String.join(",", partitionCols)).append("]"); 17 | return builder.toString(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/JedisLoader.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader; 2 | 3 | import kr.ac.yonsei.delab.addb_loader.Loader; 4 | 5 | import java.io.BufferedReader; 6 | import java.io.File; 7 | import java.io.FileNotFoundException; 8 | import java.io.FileReader; 9 | import java.io.IOException; 10 | 11 | /** 12 | * Hello world! 13 | * 14 | */ 15 | public class JedisLoader implements Loader 16 | { 17 | private String table_id; 18 | private String file_name; 19 | private String[] partition_columns; 20 | 21 | public static Loader create(String table_id, String file_name, String[] partition_columns) { 22 | return new JedisLoader(table_id, file_name, partition_columns); 23 | } 24 | 25 | public JedisLoader(String table_id, String file_name, String[] partition_columns) { 26 | this.table_id = table_id; 27 | this.file_name = file_name; 28 | this.partition_columns = partition_columns; 29 | } 30 | 31 | public void start() { 32 | // System.out.println( "ADDB Loader" ); 33 | long setup; 34 | setup = System.currentTimeMillis(); 35 | JedisManager jManager = new JedisManager(); 36 | System.out.println("Setup time = " + ((System.currentTimeMillis() - setup) / 1000)); 37 | 38 | File file = new File(file_name); 39 | BufferedReader inFile = null; 40 | double startTime, endTime; 41 | double pt; 42 | double duration = 0; 43 | int colCnt = 0; 44 | startTime = System.currentTimeMillis(); 45 | try { 46 | inFile = new BufferedReader(new FileReader(file)); 47 | String line = null; 48 | while ((line = inFile.readLine()) != null) { 49 | String array[] = line.split("\\|"); 50 | if(colCnt == 0) colCnt = array.length; 51 | pt = System.currentTimeMillis(); 52 | RedisClient client = new RedisClient(jManager, table_id, array, partition_columns, colCnt); 53 | client.execute(); 54 | duration += ((System.currentTimeMillis() - pt) /1000); 55 | } 56 | 57 | } catch (FileNotFoundException e) { 58 | e.printStackTrace(); 59 | } catch (IOException e) { 60 | e.printStackTrace(); 61 | } finally { 62 | try{ 63 | if ( inFile != null) { 64 | inFile.close(); 65 | } 66 | } catch (IOException e) { 67 | e.printStackTrace(); 68 | } 69 | } 70 | endTime = System.currentTimeMillis(); 71 | System.out.println("duration = " + duration); 72 | System.out.println("Read Time = " + ((endTime - startTime) / 1000)); 73 | long close = System.currentTimeMillis(); 74 | jManager.close(); 75 | System.out.println("Close Time = " + ((System.currentTimeMillis() - close) / 1000 )); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/JedisManager.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.List; 6 | import java.util.concurrent.ExecutorService; 7 | import java.util.concurrent.Executors; 8 | 9 | import kr.ac.yonsei.delab.addb_loader.Global; 10 | import redis.clients.addb_jedis.*; 11 | import redis.clients.addb_jedis.util.JedisClusterCRC16; 12 | import redis.clients.addb_jedis.util.SafeEncoder; 13 | 14 | public class JedisManager { 15 | List jedisClusterNodes; 16 | ExecutorService executorService; 17 | public JedisManager() { 18 | jedisClusterNodes = new ArrayList(); 19 | 20 | executorService = Executors.newFixedThreadPool(6); 21 | createJedisCluster(); 22 | Collections.sort(jedisClusterNodes); 23 | } 24 | 25 | public RedisNode retRedisNode(String key) { 26 | int slot = JedisClusterCRC16.getSlot(key); 27 | 28 | int start = 0; 29 | int end = jedisClusterNodes.size() - 1; 30 | int pos = 0; 31 | 32 | while (true) { 33 | int middle = (start + end) / 2; 34 | int startSlot = jedisClusterNodes.get(middle).startSlot_; 35 | int endSlot = jedisClusterNodes.get(middle).endSlot_; 36 | if( startSlot <= slot && slot <= endSlot ) { 37 | pos = middle; 38 | break; 39 | } else if ( slot < startSlot ) { 40 | end = middle - 1; 41 | } else if ( endSlot < slot ) { 42 | start = middle + 1; 43 | } 44 | } 45 | 46 | // System.out.println("target slot = " + slot); 47 | // System.out.println("node target = " + 48 | // jedisClusterNodes.get(pos).startSlot_ + " ~ " 49 | // + jedisClusterNodes.get(pos).endSlot_); 50 | return jedisClusterNodes.get(pos); 51 | } 52 | 53 | public void createJedisCluster() { 54 | JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); 55 | /* Master Instance */ 56 | JedisPool pool = new JedisPool(jedisPoolConfig, Global.host, Global.port); 57 | Jedis jedis = pool.getResource(); 58 | List slotinfos = jedis.clusterSlots(); 59 | for (int i = 0; i < slotinfos.size(); i++) { 60 | List slotinfo = (List) slotinfos.get(i); 61 | int sPos = Integer.parseInt(slotinfo.get(0).toString()); 62 | int ePos = Integer.parseInt(slotinfo.get(1).toString()); 63 | 64 | for(int j = 0; j < slotinfo.size() - 2; j++) { 65 | List clusterinfo = (List) slotinfo.get(j + 2); 66 | String host = SafeEncoder.encode((byte[])clusterinfo.get(0)); 67 | int port = Integer.parseInt(clusterinfo.get(1).toString()); 68 | //System.out.println("host and Port" + host + " and " + port); 69 | jedisClusterNodes.add(new RedisNode(host, port, sPos, ePos)); 70 | } 71 | } 72 | 73 | if(jedis != null) { 74 | jedis.close(); 75 | } 76 | pool.close(); 77 | } 78 | 79 | public void close() { 80 | 81 | executorService.shutdown(); 82 | for (int i = 0; i < jedisClusterNodes.size(); i++) { 83 | // if(jedisClusterNodes.get(i).pip != null) { 84 | // jedisClusterNodes.get(i).pip.sync(); 85 | // } 86 | 87 | jedisClusterNodes.get(i).close(); 88 | } 89 | } 90 | 91 | } 92 | 93 | 94 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/PipelineWorker.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader; 2 | 3 | import redis.clients.addb_jedis.Jedis; 4 | import redis.clients.addb_jedis.Pipeline; 5 | 6 | public class PipelineWorker implements Runnable { 7 | Jedis jedis_; 8 | Pipeline pip_; 9 | 10 | public PipelineWorker(Jedis jedis, Pipeline pip) { 11 | this.jedis_ = jedis; 12 | this.pip_ = pip; 13 | } 14 | 15 | public void run() { 16 | pip_.sync(); 17 | pip_.close(); 18 | jedis_.close(); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/RNWorker.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader; 2 | 3 | import java.util.Queue; 4 | 5 | import redis.clients.addb_jedis.Jedis; 6 | import redis.clients.addb_jedis.JedisPool; 7 | import redis.clients.addb_jedis.Pipeline; 8 | import redis.clients.addb_jedis.util.CommandArgsObject; 9 | 10 | public class RNWorker extends Thread { 11 | Jedis jedis_; 12 | Pipeline pip_; 13 | int refCount_; 14 | Queue queue_; 15 | 16 | public RNWorker(JedisPool pool, int refCount, Queue queue) { 17 | this.jedis_ = pool.getResource(); 18 | this.pip_ = jedis_.pipelined(); 19 | this.refCount_ = refCount; 20 | this.queue_ = queue; 21 | } 22 | 23 | @Override 24 | public void run() { 25 | while(!Thread.currentThread().isInterrupted()) { 26 | CommandArgsObject args = queue_.poll(); 27 | if (args != null) { 28 | pip_.fpwrite(args); 29 | refCount_++; 30 | if(refCount_ == 200) { 31 | refCount_ = 0; 32 | pip_.close(); 33 | pip_ = jedis_.pipelined(); 34 | } 35 | } 36 | } 37 | if(refCount_ != 0) pip_.sync(); 38 | jedis_.close(); 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/RedisClient.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import redis.clients.addb_jedis.util.CommandArgsObject; 7 | 8 | 9 | public class RedisClient { 10 | String table_id_ = null; 11 | String data[]; 12 | String partition_list[]; 13 | int colCnt_ = 0; 14 | String partitionInfo_ = null; 15 | JedisManager jmanager_; 16 | 17 | public RedisClient(JedisManager jmanager, String table_id, String [] array, String [] partition_columns, int colCnt){ 18 | table_id_ = table_id; 19 | data = new String[colCnt]; 20 | partition_list = new String[partition_columns.length]; 21 | System.arraycopy(array, 0, data, 0, colCnt); 22 | System.arraycopy(partition_columns, 0, partition_list, 0, partition_columns.length); 23 | this.colCnt_ = colCnt; 24 | jmanager_ = jmanager; 25 | } 26 | 27 | public void execute() { 28 | CommandArgsObject args = createArgsObject(); 29 | RedisNode rn = jmanager_.retRedisNode(args.getDataKey()); 30 | // System.out.println("selected :" + rn.Host_ + " and " + rn.Port_ 31 | // + " start slot " + rn.startSlot_ + " end slot " + rn.endSlot_); 32 | PipelineWorker worker = rn.execute(args); 33 | if(worker != null) { 34 | jmanager_.executorService.execute(worker); 35 | } 36 | } 37 | 38 | public CommandArgsObject createArgsObject() { 39 | String datakey = createDataKey(); 40 | String columnCount = Integer.toString(colCnt_); 41 | String partitionInfo = partitionInfo_; 42 | List dataList = Arrays.asList(data); 43 | // System.out.println("datakey = " + datakey); 44 | // System.out.println("columnCount = " + columnCount); 45 | // System.out.println("partitionInfo = " + partitionInfo); 46 | // for(int i = 0; i < dataList.size(); i++) 47 | // System.out.println("datalist = " + dataList.get(i)); 48 | 49 | return new CommandArgsObject(datakey, columnCount, partitionInfo, dataList); 50 | } 51 | 52 | public String createPartitionInfo() { 53 | StringBuilder builder = new StringBuilder(); 54 | for(int i = 0; i < partition_list.length; i++) { 55 | builder.append(partition_list[i]); 56 | builder.append(":"); 57 | builder.append(data[Integer.parseInt(partition_list[i]) - 1 ]); 58 | if (i != partition_list.length -1) builder.append(":"); 59 | } 60 | 61 | return builder.toString(); 62 | } 63 | 64 | public String createDataKey() { 65 | StringBuilder builder = new StringBuilder(); 66 | builder.append("D:{"); 67 | builder.append(table_id_); 68 | builder.append(":"); 69 | partitionInfo_ = createPartitionInfo(); 70 | builder.append(partitionInfo_); 71 | builder.append("}"); 72 | return builder.toString(); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/RedisNode.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader; 2 | 3 | import redis.clients.addb_jedis.Jedis; 4 | import redis.clients.addb_jedis.JedisPool; 5 | import redis.clients.addb_jedis.JedisPoolConfig; 6 | import redis.clients.addb_jedis.Pipeline; 7 | import redis.clients.addb_jedis.Protocol; 8 | import redis.clients.addb_jedis.util.CommandArgsObject; 9 | 10 | public class RedisNode implements Comparable { 11 | String Host_; 12 | int Port_; 13 | int startSlot_; 14 | int endSlot_; 15 | JedisPool pool; 16 | Jedis jedis; 17 | Pipeline pip; 18 | int Refcount = 0; 19 | 20 | // Queue dataQueue; 21 | // List worklist; 22 | 23 | public RedisNode(String Host, int Port, int startSlot, int endSlot) { 24 | Host_ = Host; 25 | Port_ = Port; 26 | startSlot_ = startSlot; 27 | endSlot_ = endSlot; 28 | JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); 29 | pool = new JedisPool(jedisPoolConfig, Host_, Port_, 3000000, null, Protocol.DEFAULT_DATABASE); 30 | 31 | jedis = pool.getResource(); 32 | pip = jedis.pipelined(); 33 | // dataQueue = new ConcurrentLinkedQueue(); 34 | // worklist = new ArrayList(2); 35 | // for(int i = 0; i < 2; i++) { 36 | // RNWorker rn = new RNWorker(pool, 0, dataQueue); 37 | // worklist.add(rn); 38 | // rn.start(); 39 | // } 40 | } 41 | 42 | public int compareTo(Object obj) { 43 | RedisNode other = (RedisNode)obj; 44 | if(startSlot_ < other.startSlot_) return -1; 45 | else if (startSlot_ > other.startSlot_) return 1; 46 | else return 0; 47 | } 48 | 49 | public PipelineWorker execute(CommandArgsObject args) { 50 | //System.out.println("queue insert = " + dataQueue.size()); 51 | //dataQueue.offer(args); 52 | // Jedis jedis = pool.getResource(); 53 | // Pipeline pip = jedis.pipelined(); 54 | 55 | pip.fpwrite(args); 56 | Refcount++; 57 | if (Refcount == 200) { 58 | PipelineWorker worker = new PipelineWorker(this.jedis, this.pip); 59 | this.jedis = pool.getResource(); 60 | this.pip = jedis.pipelined(); 61 | Refcount = 0; 62 | return worker; 63 | } 64 | return null; 65 | } 66 | 67 | public void close() { 68 | // for(int i = 0; i < 2; i++) { 69 | // worklist.get(i).interrupt(); 70 | // } 71 | 72 | // Jedis tmp_jedis = pool.getResource(); 73 | // 74 | // while (dataQueue.size() != 0) { 75 | // CommandArgsObject arg = dataQueue.poll(); 76 | // if (arg != null) tmp_jedis.fpwrite(arg); 77 | // } 78 | // 79 | // tmp_jedis.close(); 80 | if(this.pip != null) pip.close(); 81 | if(this.jedis != null) jedis.close(); 82 | 83 | pool.close(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/lettuce_loader/LettuceLoader.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.lettuce_loader; 2 | 3 | import io.lettuce.core.LettuceFutures; 4 | import io.lettuce.core.RedisFuture; 5 | import io.lettuce.core.RedisURI; 6 | import io.lettuce.core.addb.FpWriteArgs; 7 | import io.lettuce.core.cluster.RedisClusterClient; 8 | import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; 9 | import io.lettuce.core.resource.ClientResources; 10 | import io.lettuce.core.resource.DefaultClientResources; 11 | import kr.ac.yonsei.delab.addb_loader.Global; 12 | import kr.ac.yonsei.delab.addb_loader.Loader; 13 | import reactor.core.publisher.Mono; 14 | import reactor.core.scheduler.Schedulers; 15 | 16 | import java.io.BufferedReader; 17 | import java.io.FileReader; 18 | import java.io.IOException; 19 | import java.nio.file.Path; 20 | import java.nio.file.Paths; 21 | import java.util.*; 22 | import java.util.concurrent.*; 23 | 24 | public class LettuceLoader implements Loader { 25 | private String tableId; 26 | private String fileName; 27 | private String[] partitionColumns; 28 | 29 | public static Loader create(String tableId, String fileName, String[] partitionColumns) { 30 | return new LettuceLoader(tableId, fileName, partitionColumns); 31 | } 32 | 33 | public LettuceLoader(String tableId, String fileName, String[] partitionColumns) { 34 | this.tableId = tableId; 35 | this.fileName = fileName; 36 | this.partitionColumns = partitionColumns; 37 | } 38 | 39 | public void start() { 40 | double total_time = 0; 41 | double total_time_start, total_time_end; 42 | double lettuce_time = 0; 43 | 44 | RedisURI uri = RedisURI.Builder.redis(Global.host, Global.port).build(); 45 | ClientResources resources = DefaultClientResources.builder() 46 | // .ioThreadPoolSize(4) 47 | // .computationThreadPoolSize(4) 48 | .build(); 49 | RedisClusterClient clusterClient = RedisClusterClient.create(resources, uri); 50 | StatefulRedisClusterConnection conn = clusterClient.connect(); 51 | 52 | total_time_start = System.currentTimeMillis(); 53 | Path filePath = Paths.get(fileName); 54 | 55 | // Version 1 - Reactor API 56 | // reactorLoader(filePath, conn); 57 | 58 | // Version 2 - Future API 59 | // futureLoader(filePath, conn); 60 | 61 | // Version 3 - Pipeline Loader (Iteration base) 62 | // pipelineIterLoader(filePath, conn); 63 | 64 | // Version 4 - Pipeline Loader (Reactor base) 65 | pipelineReactorLoader(filePath, conn); 66 | 67 | conn.close(); 68 | clusterClient.shutdown(); 69 | 70 | total_time_end = System.currentTimeMillis(); 71 | total_time = total_time_end - total_time_start; 72 | System.out.println("Total Time = " + (total_time / 1000)); 73 | } 74 | 75 | private void reactorLoader(Path filePath, StatefulRedisClusterConnection conn) { 76 | ReactiveFileReader reader = new ReactiveFileReader(filePath); 77 | reader.parse() 78 | .map(line -> line.split("\\|")) 79 | .flatMap(colValues -> { 80 | FpWriteArgs args = FpWriteArgs.Builder 81 | .dataKey(createDataKey(colValues)) 82 | .partitionInfo(createPartitionInfo(colValues)) 83 | .columnCount(String.valueOf(colValues.length)) 84 | .data(colValues); 85 | return conn.reactive().fpwrite(args) 86 | .subscribeOn(Schedulers.parallel()); 87 | }) 88 | .blockLast(); 89 | } 90 | 91 | private void futureLoader(Path filePath, StatefulRedisClusterConnection conn) { 92 | ReactiveFileReader reader = new ReactiveFileReader(filePath); 93 | reader.parse() 94 | .map(line -> line.split("\\|")) 95 | .map(colValues -> { 96 | FpWriteArgs args = FpWriteArgs.Builder 97 | .dataKey(createDataKey(colValues)) 98 | .partitionInfo(createPartitionInfo(colValues)) 99 | .columnCount(String.valueOf(colValues.length)) 100 | .data(colValues); 101 | return conn.async().fpwrite(args); 102 | }) 103 | .publishOn(Schedulers.elastic()) 104 | .map(future -> { 105 | try { 106 | return future.await(1, TimeUnit.HOURS); 107 | } catch (Exception e) { 108 | throw new RuntimeException(e); 109 | } 110 | }) 111 | .blockLast(); 112 | } 113 | 114 | private void pipelineIterLoader(Path filePath, StatefulRedisClusterConnection conn) { 115 | BufferedReader reader; 116 | try { 117 | reader = new BufferedReader(new FileReader(filePath.toFile())); 118 | } catch (IOException e) { 119 | throw new RuntimeException(e); 120 | } 121 | 122 | String line; 123 | List> futures = new LinkedList<>(); 124 | final int PIPELINE_COUNT = 100000; 125 | while ((line = getLineFromFile(reader)) != null) { 126 | String[] colValues = line.split("\\|"); 127 | FpWriteArgs args = FpWriteArgs.Builder 128 | .dataKey(createDataKey(colValues)) 129 | .partitionInfo(createPartitionInfo(colValues)) 130 | .columnCount(String.valueOf(colValues.length)) 131 | .data(colValues); 132 | futures.add(conn.async().fpwrite(args)); 133 | if (futures.size() >= PIPELINE_COUNT) { 134 | try { 135 | LettuceFutures.awaitAll( 136 | 1, 137 | TimeUnit.HOURS, 138 | futures.toArray(new RedisFuture[0]) 139 | ); 140 | } catch (Exception e) { 141 | throw new RuntimeException(e); 142 | } finally { 143 | futures.clear(); 144 | } 145 | } 146 | } 147 | 148 | try { 149 | LettuceFutures.awaitAll( 150 | 1, 151 | TimeUnit.HOURS, 152 | futures.toArray(new RedisFuture[0]) 153 | ); 154 | } catch (Exception e) { 155 | throw new RuntimeException(e); 156 | } 157 | } 158 | 159 | private void pipelineReactorLoader(Path filePath, StatefulRedisClusterConnection conn) { 160 | final int WINDOW_SIZE = 100000; 161 | ReactiveFileReader reader = new ReactiveFileReader(filePath); 162 | reader.parse() 163 | .map(line -> line.split("\\|")) 164 | .map(colValues -> { 165 | FpWriteArgs args = FpWriteArgs.Builder 166 | .dataKey(createDataKey(colValues)) 167 | .partitionInfo(createPartitionInfo(colValues)) 168 | .columnCount(String.valueOf(colValues.length)) 169 | .data(colValues); 170 | return conn.async().fpwrite(args); 171 | }) 172 | .window(WINDOW_SIZE) 173 | .flatMap(futureFlux -> futureFlux 174 | .collectList() 175 | .flatMap(futures -> Mono.fromCallable(() -> LettuceFutures.awaitAll( 176 | 1, 177 | TimeUnit.HOURS, 178 | futures.toArray(new RedisFuture[0]) 179 | ))) 180 | ) 181 | .blockLast(); 182 | } 183 | 184 | private CompletableFuture toCompletableFuture(Future future) { 185 | return CompletableFuture.supplyAsync(() -> { 186 | try { 187 | return future.get(); 188 | } catch (Exception e) { 189 | throw new RuntimeException(e); 190 | } 191 | }); 192 | } 193 | 194 | private CompletableFuture toCompletableFuture(RedisFuture redisFuture) { 195 | return CompletableFuture.supplyAsync(() -> { 196 | try { 197 | return redisFuture.get(); 198 | } catch (Exception e) { 199 | throw new RuntimeException(e); 200 | } 201 | }); 202 | } 203 | 204 | private String getLineFromFile(BufferedReader reader) { 205 | try { 206 | return reader.readLine(); 207 | } catch (IOException e) { 208 | throw new RuntimeException(e); 209 | } 210 | } 211 | 212 | private String createPartitionInfo(String[] colValues) { 213 | StringBuilder builder = new StringBuilder(); 214 | for (int i = 0; i < partitionColumns.length; ++i) { 215 | int partitionCol = Integer.parseInt(partitionColumns[i]); 216 | builder.append(partitionCol); 217 | builder.append(":"); 218 | builder.append(colValues[partitionCol - 1]); 219 | if (i != partitionColumns.length - 1) { 220 | builder.append(":"); 221 | } 222 | } 223 | return builder.toString(); 224 | } 225 | 226 | private String createDataKey(String[] colValues) { 227 | StringBuilder builder = new StringBuilder(); 228 | builder.append("D:{") 229 | .append(tableId) 230 | .append(":") 231 | .append(createPartitionInfo(colValues)) 232 | .append("}"); 233 | return builder.toString(); 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/lettuce_loader/PipelineManager.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.lettuce_loader; 2 | 3 | import io.lettuce.core.LettuceFutures; 4 | import io.lettuce.core.RedisFuture; 5 | import io.lettuce.core.addb.FpWriteArgs; 6 | import io.lettuce.core.api.StatefulRedisConnection; 7 | import io.lettuce.core.api.async.RedisAsyncCommands; 8 | import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; 9 | import io.lettuce.core.cluster.models.partitions.RedisClusterNode; 10 | import io.netty.channel.EventLoopGroup; 11 | import lombok.Getter; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | import java.util.concurrent.TimeUnit; 18 | import java.util.stream.Collectors; 19 | 20 | public class PipelineManager { 21 | final private static int PIPELINE_COUNT = 1000; 22 | 23 | public static class Worker implements Runnable { 24 | List> futures; 25 | 26 | Worker(List> futures) { 27 | this.futures = futures; 28 | } 29 | 30 | public void run() { 31 | 32 | } 33 | } 34 | 35 | // Thread pool 36 | ExecutorService executorService; 37 | 38 | @Getter 39 | public StatefulRedisConnection nodeConn; 40 | @Getter 41 | public RedisAsyncCommands commands; 42 | private List> futures; 43 | 44 | public static PipelineManager create(StatefulRedisClusterConnection conn, 45 | int slot, 46 | ExecutorService executorService) { 47 | RedisClusterNode node = conn.getPartitions().getPartitionBySlot(slot); 48 | return new PipelineManager(conn.getConnection(node.getNodeId()), executorService); 49 | } 50 | 51 | public PipelineManager(StatefulRedisConnection nodeConn, ExecutorService executorService) { 52 | this.nodeConn = nodeConn; 53 | this.commands = nodeConn.async(); 54 | this.commands.setAutoFlushCommands(false); 55 | this.futures = new ArrayList<>(); 56 | this.executorService = executorService; 57 | } 58 | 59 | public void fpwrite(FpWriteArgs args) { 60 | if (futures.size() > PIPELINE_COUNT) { 61 | commands.flushCommands(); 62 | 63 | this.executorService.execute(new Worker(futures)); 64 | futures = new ArrayList<>(); 65 | } 66 | 67 | futures.add(commands.fpwrite(args)); 68 | } 69 | 70 | public void close() { 71 | nodeConn.close(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/lettuce_loader/ReactiveFileReader.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.lettuce_loader; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import reactor.core.publisher.Flux; 6 | 7 | import java.nio.file.Files; 8 | import java.nio.file.Path; 9 | import java.util.stream.BaseStream; 10 | 11 | public class ReactiveFileReader { 12 | @Getter @Setter private Path path; 13 | 14 | public ReactiveFileReader(Path path) { 15 | this.path = path; 16 | } 17 | 18 | public Flux parse() { 19 | return Flux.using( 20 | () -> Files.lines(path), 21 | Flux::fromStream, 22 | BaseStream::close 23 | ); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/kr/ac/yonsei/delab/addb_loader/lettuce_loader/TaoPipelineManager.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader.lettuce_loader; 2 | 3 | import io.lettuce.core.LettuceFutures; 4 | import io.lettuce.core.RedisFuture; 5 | import io.lettuce.core.addb.FpWriteArgs; 6 | import io.lettuce.core.api.StatefulRedisConnection; 7 | import io.lettuce.core.api.async.RedisAsyncCommands; 8 | import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; 9 | import io.lettuce.core.cluster.models.partitions.RedisClusterNode; 10 | import lombok.Getter; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.concurrent.ExecutorService; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.stream.Collectors; 17 | 18 | public class TaoPipelineManager { 19 | final private static int PIPELINE_COUNT = 1000; 20 | 21 | ExecutorService service = null; 22 | 23 | @Getter 24 | public StatefulRedisConnection nodeConn; 25 | @Getter 26 | public RedisAsyncCommands commands; 27 | private List argsList; 28 | 29 | public static TaoPipelineManager create(StatefulRedisClusterConnection conn, 30 | int slot) { 31 | RedisClusterNode node = conn.getPartitions().getPartitionBySlot(slot); 32 | return new TaoPipelineManager(conn.getConnection(node.getNodeId())); 33 | } 34 | 35 | public static TaoPipelineManager create(StatefulRedisClusterConnection conn, 36 | int slot, 37 | ExecutorService service) { 38 | RedisClusterNode node = conn.getPartitions().getPartitionBySlot(slot); 39 | return new TaoPipelineManager(conn.getConnection(node.getNodeId()), service); 40 | } 41 | 42 | public TaoPipelineManager(StatefulRedisConnection nodeConn) { 43 | this.nodeConn = nodeConn; 44 | this.commands = nodeConn.async(); 45 | this.commands.setAutoFlushCommands(false); 46 | this.argsList = new ArrayList<>(); 47 | } 48 | 49 | public TaoPipelineManager(StatefulRedisConnection nodeConn, 50 | ExecutorService service) { 51 | this.nodeConn = nodeConn; 52 | this.commands = nodeConn.async(); 53 | this.commands.setAutoFlushCommands(false); 54 | this.argsList = new ArrayList<>(); 55 | this.service = service; 56 | } 57 | 58 | public void fpwrite(FpWriteArgs args) { 59 | if (argsList.size() >= PIPELINE_COUNT) { 60 | if (service != null) { 61 | service.execute(() -> pipeline(new ArrayList<>(argsList))); 62 | } else { 63 | pipeline(argsList); 64 | } 65 | argsList.clear(); 66 | argsList = new ArrayList<>(); 67 | } 68 | 69 | argsList.add(args); 70 | } 71 | 72 | private void pipeline(List argsList) { 73 | List> futures = argsList.stream() 74 | .map(_args -> commands.fpwrite(_args)) 75 | .collect(Collectors.toList()); 76 | commands.flushCommands(); 77 | LettuceFutures.awaitAll( 78 | 1, 79 | TimeUnit.HOURS, 80 | futures.toArray(new RedisFuture[0]) 81 | ); 82 | futures.clear(); 83 | argsList.clear(); 84 | } 85 | 86 | public void close() { 87 | pipeline(argsList); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/kr/ac/yonsei/delab/addb_loader/AppTest.java: -------------------------------------------------------------------------------- 1 | package kr.ac.yonsei.delab.addb_loader; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | --------------------------------------------------------------------------------