├── .gitignore
├── config.yaml
├── pom.xml
└── src
├── main
└── java
│ └── kr
│ └── ac
│ └── yonsei
│ └── delab
│ └── addb_loader
│ ├── Config.java
│ ├── Global.java
│ ├── Loader.java
│ ├── Main.java
│ ├── TableConfig.java
│ ├── jedis_loader
│ ├── JedisLoader.java
│ ├── JedisManager.java
│ ├── PipelineWorker.java
│ ├── RNWorker.java
│ ├── RedisClient.java
│ └── RedisNode.java
│ └── lettuce_loader
│ ├── LettuceLoader.java
│ ├── PipelineManager.java
│ ├── ReactiveFileReader.java
│ └── TaoPipelineManager.java
└── test
└── java
└── kr
└── ac
└── yonsei
└── delab
└── addb_loader
└── AppTest.java
/.gitignore:
--------------------------------------------------------------------------------
1 | target/
2 | .idea/
3 | *.tbl
4 |
--------------------------------------------------------------------------------
/config.yaml:
--------------------------------------------------------------------------------
1 | loader: "lettuce"
2 | tables:
3 | nation:
4 | id: 1
5 | file: "data/nation.tbl"
6 | partitionCols:
7 | - "1"
8 | region:
9 | id: 2
10 | file: "data/region.tbl"
11 | partitionCols:
12 | - "1"
13 | part:
14 | id: 3
15 | file: "data/part.tbl"
16 | partitionCols:
17 | - "4"
18 | supplier:
19 | id: 4
20 | file: "data/supplier.tbl"
21 | partitionCols:
22 | - "4"
23 | partsupp:
24 | id: 5
25 | file: "data/partsupp.tbl"
26 | partitionCols:
27 | - "2"
28 | customer:
29 | id: 6
30 | file: "data/customer.tbl"
31 | partitionCols:
32 | - "4"
33 | orders:
34 | id: 7
35 | file: "data/orders.tbl"
36 | partitionCols:
37 | - "5"
38 | lineitem:
39 | id: 8
40 | file: "data/lineitem.tbl"
41 | partitionCols:
42 | - "3"
43 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
3 | 4.0.0
4 |
5 | kr.ac.yonsei.delab
6 | addb_loader
7 | 0.0.1-SNAPSHOT
8 | jar
9 |
10 | addb_loader
11 | http://maven.apache.org
12 |
13 |
14 | UTF-8
15 |
16 |
17 |
18 |
19 |
20 | commons-cli
21 | commons-cli
22 | 1.3.1
23 |
24 |
25 |
26 | org.yaml
27 | snakeyaml
28 | 1.21
29 |
30 |
31 |
32 | org.projectlombok
33 | lombok
34 | 1.18.8
35 |
36 |
37 |
38 | org.apache.commons
39 | commons-pool2
40 | 2.4.3
41 |
42 |
43 |
44 | org.slf4j
45 | slf4j-simple
46 | 1.7.26
47 |
48 |
49 | com.github.javasync
50 | AsyncFileRw
51 | 1.1.2
52 |
53 |
54 |
55 | junit
56 | junit
57 | 3.8.1
58 | test
59 |
60 |
61 | kr.ac.yonsei.delab
62 | addb-jedis
63 | 0.0.2
64 |
65 |
66 | kr.ac.yonsei.delab
67 | addb-lettuce
68 | 0.0.1
69 |
70 |
71 | io.netty
72 | netty-transport-native-epoll
73 | 4.1.38.Final
74 | linux-x86_64
75 | true
76 |
77 |
78 | io.netty
79 | netty-transport-native-kqueue
80 | 4.1.38.Final
81 | osx-x86_64
82 | false
83 |
84 |
85 |
86 |
87 |
88 | org.codehaus.mojo
89 | exec-maven-plugin
90 | 1.6.0
91 |
92 | kr.ac.yonsei.delab.addb_loader.Main
93 |
94 |
95 |
96 | org.apache.maven.plugins
97 | maven-jar-plugin
98 | 3.0.2
99 |
100 |
101 |
102 | kr.ac.yonsei.delab.addb_loader.Main
103 | true
104 | true
105 | kr.ac.yonsei.delab.addb_loader
106 |
107 |
108 |
109 |
110 |
111 | org.apache.maven.plugins
112 | maven-assembly-plugin
113 | 3.1.0
114 |
115 |
116 |
117 | kr.ac.yonsei.delab.addb_loader.Main
118 | true
119 | true
120 | kr.ac.yonsei.delab.addb_loader
121 |
122 |
123 |
124 | jar-with-dependencies
125 |
126 |
127 |
128 |
129 | package
130 |
131 | single
132 |
133 |
134 |
135 |
136 |
137 | org.apache.maven.plugins
138 | maven-compiler-plugin
139 | 3.3
140 |
141 | 1.8
142 | 1.8
143 |
144 |
145 |
146 |
147 |
148 |
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/Config.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader;
2 |
3 | import lombok.Getter;
4 | import lombok.Setter;
5 |
6 | import java.util.Map;
7 |
8 | public class Config {
9 | @Getter @Setter String loader;
10 | @Getter @Setter Map tables;
11 |
12 | @Override
13 | public String toString() {
14 | StringBuilder builder = new StringBuilder();
15 | builder.append("==== Config ====\n")
16 | .append("1. Loader\n")
17 | .append(loader).append("\n")
18 | .append("2. Tables\n")
19 | .append(tables).append("\n")
20 | .append("----------------\n")
21 | .append("================");
22 | return builder.toString();
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/Global.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader;
2 |
3 | public class Global {
4 | public static Config config;
5 | public static String host;
6 | public static int port;
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/Loader.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader;
2 |
3 | public interface Loader {
4 | public void start();
5 | }
6 |
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/Main.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader;
2 |
3 | import kr.ac.yonsei.delab.addb_loader.jedis_loader.JedisLoader;
4 | import kr.ac.yonsei.delab.addb_loader.lettuce_loader.LettuceLoader;
5 | import org.apache.commons.cli.*;
6 | import org.yaml.snakeyaml.Yaml;
7 | import org.yaml.snakeyaml.constructor.Constructor;
8 |
9 | import java.io.File;
10 | import java.io.FileInputStream;
11 | import java.io.FileNotFoundException;
12 | import java.io.InputStream;
13 |
14 | public class Main {
15 | public static void main(String[] args) {
16 | /* Command type : ./Loader.jar -c config.yaml -t lineitem -h 255.255.255.0 -p 8000 */
17 | /* Parses arguments... */
18 | Option configOption = new Option(
19 | "c", "config", true, "Config file path");
20 | configOption.setRequired(true);
21 | Option tableOption = new Option(
22 | "t", "table", true, "Table name in config file");
23 | tableOption.setRequired(true);
24 | Option hostOption = new Option(
25 | "h", "host", true, "Host of Redis master node");
26 | hostOption.setRequired(true);
27 | Option portOption = new Option(
28 | "p", "port", true, "Port of Redis master node");
29 | portOption.setRequired(true);
30 | Options options = new Options()
31 | .addOption(configOption)
32 | .addOption(tableOption)
33 | .addOption(hostOption)
34 | .addOption(portOption);
35 |
36 | CommandLineParser parser = new DefaultParser();
37 | CommandLine cmd;
38 | try {
39 | cmd = parser.parse(options, args);
40 | } catch (ParseException e) {
41 | HelpFormatter formatter = new HelpFormatter();
42 | formatter.printHelp("loader", options);
43 | throw new RuntimeException();
44 | }
45 |
46 | String configFileName = cmd.getOptionValue("config");
47 | String tableName = cmd.getOptionValue("table");
48 | String host = cmd.getOptionValue("host");
49 | String port = cmd.getOptionValue("port");
50 |
51 | /* Parses config file... */
52 | Yaml yaml = new Yaml(new Constructor(Config.class));
53 | InputStream inputStream = null;
54 | try {
55 | inputStream = new FileInputStream(new File(configFileName));
56 | } catch (FileNotFoundException e) {
57 | throw new RuntimeException("Config file is not founded");
58 | }
59 | Global.config = yaml.loadAs(inputStream, Config.class);
60 | Global.host = host;
61 | Global.port = Integer.valueOf(port);
62 |
63 | TableConfig tableConfig = Global.config.tables.get(tableName);
64 | if (tableConfig == null) {
65 | throw new RuntimeException("Table name is not founded on config.yaml");
66 | }
67 |
68 | System.out.println(Global.config);
69 |
70 | /* Start Loader... */
71 | Loader loader;
72 | if (Global.config.getLoader().equalsIgnoreCase("jedis")) {
73 | loader = JedisLoader.create(
74 | Integer.toString(tableConfig.getId()),
75 | tableConfig.getFile(),
76 | tableConfig.getPartitionCols()
77 | );
78 | } else if (Global.config.getLoader().equalsIgnoreCase("lettuce")) {
79 | // TODO(totoro): Implements Lettuce Loader
80 | System.out.println("TODO(totoro): Implements Lettuce Loader...");
81 | loader = LettuceLoader.create(
82 | Integer.toString(tableConfig.getId()),
83 | tableConfig.getFile(),
84 | tableConfig.getPartitionCols()
85 | );
86 | } else {
87 | throw new RuntimeException("Loader must be either 'jedis' or 'lettuce'.");
88 | }
89 | loader.start();
90 | }
91 | }
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/TableConfig.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader;
2 |
3 | import lombok.Getter;
4 | import lombok.Setter;
5 |
6 | public class TableConfig {
7 | @Getter @Setter private int id;
8 | @Getter @Setter private String file;
9 | @Getter @Setter private String[] partitionCols;
10 |
11 | @Override
12 | public String toString() {
13 | StringBuilder builder = new StringBuilder();
14 | builder.append("id: ").append(id).append(" | ")
15 | .append("file: ").append(file).append(" | ")
16 | .append("partitionCols: [").append(String.join(",", partitionCols)).append("]");
17 | return builder.toString();
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/JedisLoader.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader;
2 |
3 | import kr.ac.yonsei.delab.addb_loader.Loader;
4 |
5 | import java.io.BufferedReader;
6 | import java.io.File;
7 | import java.io.FileNotFoundException;
8 | import java.io.FileReader;
9 | import java.io.IOException;
10 |
11 | /**
12 | * Hello world!
13 | *
14 | */
15 | public class JedisLoader implements Loader
16 | {
17 | private String table_id;
18 | private String file_name;
19 | private String[] partition_columns;
20 |
21 | public static Loader create(String table_id, String file_name, String[] partition_columns) {
22 | return new JedisLoader(table_id, file_name, partition_columns);
23 | }
24 |
25 | public JedisLoader(String table_id, String file_name, String[] partition_columns) {
26 | this.table_id = table_id;
27 | this.file_name = file_name;
28 | this.partition_columns = partition_columns;
29 | }
30 |
31 | public void start() {
32 | // System.out.println( "ADDB Loader" );
33 | long setup;
34 | setup = System.currentTimeMillis();
35 | JedisManager jManager = new JedisManager();
36 | System.out.println("Setup time = " + ((System.currentTimeMillis() - setup) / 1000));
37 |
38 | File file = new File(file_name);
39 | BufferedReader inFile = null;
40 | double startTime, endTime;
41 | double pt;
42 | double duration = 0;
43 | int colCnt = 0;
44 | startTime = System.currentTimeMillis();
45 | try {
46 | inFile = new BufferedReader(new FileReader(file));
47 | String line = null;
48 | while ((line = inFile.readLine()) != null) {
49 | String array[] = line.split("\\|");
50 | if(colCnt == 0) colCnt = array.length;
51 | pt = System.currentTimeMillis();
52 | RedisClient client = new RedisClient(jManager, table_id, array, partition_columns, colCnt);
53 | client.execute();
54 | duration += ((System.currentTimeMillis() - pt) /1000);
55 | }
56 |
57 | } catch (FileNotFoundException e) {
58 | e.printStackTrace();
59 | } catch (IOException e) {
60 | e.printStackTrace();
61 | } finally {
62 | try{
63 | if ( inFile != null) {
64 | inFile.close();
65 | }
66 | } catch (IOException e) {
67 | e.printStackTrace();
68 | }
69 | }
70 | endTime = System.currentTimeMillis();
71 | System.out.println("duration = " + duration);
72 | System.out.println("Read Time = " + ((endTime - startTime) / 1000));
73 | long close = System.currentTimeMillis();
74 | jManager.close();
75 | System.out.println("Close Time = " + ((System.currentTimeMillis() - close) / 1000 ));
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/src/main/java/kr/ac/yonsei/delab/addb_loader/jedis_loader/JedisManager.java:
--------------------------------------------------------------------------------
1 | package kr.ac.yonsei.delab.addb_loader.jedis_loader;
2 |
3 | import java.util.ArrayList;
4 | import java.util.Collections;
5 | import java.util.List;
6 | import java.util.concurrent.ExecutorService;
7 | import java.util.concurrent.Executors;
8 |
9 | import kr.ac.yonsei.delab.addb_loader.Global;
10 | import redis.clients.addb_jedis.*;
11 | import redis.clients.addb_jedis.util.JedisClusterCRC16;
12 | import redis.clients.addb_jedis.util.SafeEncoder;
13 |
14 | public class JedisManager {
15 | List jedisClusterNodes;
16 | ExecutorService executorService;
17 | public JedisManager() {
18 | jedisClusterNodes = new ArrayList();
19 |
20 | executorService = Executors.newFixedThreadPool(6);
21 | createJedisCluster();
22 | Collections.sort(jedisClusterNodes);
23 | }
24 |
25 | public RedisNode retRedisNode(String key) {
26 | int slot = JedisClusterCRC16.getSlot(key);
27 |
28 | int start = 0;
29 | int end = jedisClusterNodes.size() - 1;
30 | int pos = 0;
31 |
32 | while (true) {
33 | int middle = (start + end) / 2;
34 | int startSlot = jedisClusterNodes.get(middle).startSlot_;
35 | int endSlot = jedisClusterNodes.get(middle).endSlot_;
36 | if( startSlot <= slot && slot <= endSlot ) {
37 | pos = middle;
38 | break;
39 | } else if ( slot < startSlot ) {
40 | end = middle - 1;
41 | } else if ( endSlot < slot ) {
42 | start = middle + 1;
43 | }
44 | }
45 |
46 | // System.out.println("target slot = " + slot);
47 | // System.out.println("node target = " +
48 | // jedisClusterNodes.get(pos).startSlot_ + " ~ "
49 | // + jedisClusterNodes.get(pos).endSlot_);
50 | return jedisClusterNodes.get(pos);
51 | }
52 |
53 | public void createJedisCluster() {
54 | JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
55 | /* Master Instance */
56 | JedisPool pool = new JedisPool(jedisPoolConfig, Global.host, Global.port);
57 | Jedis jedis = pool.getResource();
58 | List