├── .gitignore ├── README.md ├── TODO ├── run.simple.bash └── src └── jiibench.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Package Files # 4 | *.jar 5 | *.war 6 | *.ear 7 | 8 | build.xml 9 | capture-iostat-datasize.bash 10 | doit-hillclimb.bash 11 | doit-multi.bash 12 | doit.bash 13 | prep.bash 14 | run.load.bash 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | iibench-mysql 2 | =============== 3 | 4 | iiBench Benchmark for MySQL / Percona / MariaDB 5 | 6 | 7 | Requirements 8 | ===================== 9 | 10 | * Java 1.6 or 1.7 11 | * The MySQL Java connector must exist and be in the CLASSPATH, as in "export CLASSPATH=/home/tcallaghan/java_goodies/mysql-connector-java-5.1.30-bin.jar:.". 12 | * This example assumes that you already have a MySQL/Percona/MariaDB server running on the same machine as the iiBench client application. 13 | * You can connect a different server or port by editing the run.simple.bash script. 14 | * You must edit the run.simple.bash script to specific a different user/password/database. 15 | 16 | 17 | Running the benchmark 18 | ===================== 19 | 20 | In the default configuration the benchmark will run for 1 hour, or 1 billion inserts, whichever comes first, using 4 insertion threads. 21 | 22 | To run: 23 | 24 | ```bash 25 | git clone https://github.com/tmcallaghan/iibench-mysql.git 26 | cd iibench-mysql 27 | 28 | ``` 29 | 30 | Edit run.simple.bash to match your environment. You will most likely want to change the server/port and credentials for your database. 31 | 32 | ```bash 33 | ./run.simple.bash 34 | 35 | ``` -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - add an optional padding field to get big quickly 2 | - user defined size 3 | - user defined compressible amount 4 | - improve command line argument passing 5 | - use "--argument value" syntax 6 | - add defaults 7 | - additional query workload features 8 | - randomly select query type (based on indexes) 9 | - compare other features in Launchpad version -------------------------------------------------------------------------------- /run.simple.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # simple script to run against running MySQL/MariaDB/Percona server localhost:(default port) 4 | 5 | # database in which to run the benchmark 6 | # valid values : character 7 | export MYSQL_DATABASE=test 8 | 9 | # total number of simultaneous insertion threads 10 | # valid values : integer > 0 11 | export NUM_LOADER_THREADS=4 12 | 13 | # run the benchmark for this many inserts (or the number of minutes defined by RUN_MINUTES) 14 | # valid values : integer > 0 15 | export MAX_ROWS=1000000000 16 | 17 | # total number of rows to insert per "batch" 18 | # valid values : integer > 0 19 | export NUM_ROWS_PER_INSERT=250 20 | 21 | # display performance information every time the client application inserts this many rows 22 | # valid values : integer > 0, set to -1 if using NUM_SECONDS_PER_FEEDBACK 23 | export NUM_INSERTS_PER_FEEDBACK=-1 24 | 25 | # display performance information every time the client application has run for this many seconds 26 | # valid values : integer > 0, set to -1 if using NUM_INSERTS_PER_FEEDBACK 27 | export NUM_SECONDS_PER_FEEDBACK=10 28 | 29 | # run the benchmark for this many minutes (or the number of inserts defined by MAX_ROWS) 30 | # valid values : intever > 0 31 | export RUN_MINUTES=60 32 | export RUN_SECONDS=$[RUN_MINUTES*60] 33 | 34 | # total number of rows to insert per second, allows for the benchmark to be rate limited 35 | # valid values : integer > 0 36 | export MAX_INSERTS_PER_SECOND=999999 37 | 38 | # number of additional character fields (semi-compressible) to add to each inserted row 39 | # valid values : integer >= 0 40 | export NUM_CHAR_FIELDS=0 41 | 42 | # size (in bytes) of each additional semi-compressible character field 43 | # valid values : integer >= 0 44 | export LENGTH_CHAR_FIELDS=1000 45 | 46 | # percentage of highly compressible data (repeated character "a") in character field 47 | # valid values : integer >= 0 and <= 100 48 | export PERCENT_COMPRESSIBLE=90 49 | 50 | # number of secondary indexes to maintain 51 | # valid values : integer >= 0 and <= 3 52 | export NUM_SECONDARY_INDEXES=3 53 | 54 | # the following 4 parameters allow an insert plus query workload benchmark 55 | 56 | # number of queries to perform per QUERY_INTERVAL_SECONDS seconds 57 | # valid values : integer > 0, set to zero for insert only workload 58 | export QUERIES_PER_INTERVAL=0 59 | 60 | # number of seconds during which to perform QUERIES_PER_INTERVAL queries 61 | # valid values : integer > 0 62 | export QUERY_INTERVAL_SECONDS=15 63 | 64 | # number of rows to return per query 65 | # valid values : integer > 0 66 | export QUERY_LIMIT=10 67 | 68 | # wait this many inserts to begin the query workload 69 | # valid values : integer > 0 70 | export QUERY_NUM_ROWS_BEGIN=1000000 71 | 72 | # if InnoDB, compress to 1K, 2K, 4K, 8K, 16K block. Pass 0 for no InnoDB compression. 73 | # valid values : integer in (0,1,2,4,8,16) 74 | export INNODB_KEY_BLOCK_SIZE=0 75 | 76 | # Storage engine for benchmark 77 | # valid values : tokudb, innodb, myisam 78 | export MYSQL_STORAGE_ENGINE=tokudb 79 | 80 | # MySQL server port 81 | # valid values : integer > 0 82 | export MYSQL_PORT=27017 83 | 84 | # name of the server to connect to 85 | export MYSQL_SERVER=localhost 86 | 87 | # mysql username 88 | export MYSQL_USERNAME=tmc 89 | 90 | # mysql password 91 | export MYSQL_PASSWORD=tmc 92 | 93 | # create the table (Y/N) 94 | export CREATE_TABLE=Y 95 | 96 | 97 | 98 | # if TokuDB, need to select compression for primary key and secondary indexes (zlib is default) 99 | # valid values : lzma, quicklz, zlib, none 100 | #export TOKUDB_COMPRESSION=zlib 101 | 102 | # if TokuDB, need to select basement node size (65536 is default) 103 | # valid values : integer > 0 : 65536 for 64K 104 | #export TOKUDB_BASEMENT=65536 105 | 106 | 107 | 108 | javac -cp $CLASSPATH:$PWD/src src/jiibench.java 109 | 110 | 111 | export LOG_NAME=iibench-${MAX_ROWS}-${NUM_ROWS_PER_INSERT}-${MAX_INSERTS_PER_SECOND}-${NUM_LOADER_THREADS}-${QUERIES_PER_INTERVAL}-${QUERY_INTERVAL_SECONDS}.txt 112 | export BENCHMARK_TSV=${LOG_NAME}.tsv 113 | 114 | rm -f $LOG_NAME 115 | rm -f $BENCHMARK_TSV 116 | 117 | T="$(date +%s)" 118 | java -cp $CLASSPATH:$PWD/src jiibench $MYSQL_DATABASE $NUM_LOADER_THREADS $MAX_ROWS $NUM_ROWS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK \ 119 | $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $RUN_SECONDS $QUERIES_PER_INTERVAL $QUERY_INTERVAL_SECONDS \ 120 | $QUERY_LIMIT $QUERY_NUM_ROWS_BEGIN $MAX_INSERTS_PER_SECOND $NUM_CHAR_FIELDS $LENGTH_CHAR_FIELDS \ 121 | $NUM_SECONDARY_INDEXES $PERCENT_COMPRESSIBLE $MYSQL_PORT $MYSQL_STORAGE_ENGINE $INNODB_KEY_BLOCK_SIZE \ 122 | $MYSQL_SERVER $MYSQL_USERNAME $MYSQL_PASSWORD $CREATE_TABLE | tee -a $LOG_NAME 123 | 124 | echo "" | tee -a $LOG_NAME 125 | T="$(($(date +%s)-T))" 126 | printf "`date` | iibench duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME 127 | -------------------------------------------------------------------------------- /src/jiibench.java: -------------------------------------------------------------------------------- 1 | import java.sql.Connection; 2 | import java.sql.DriverManager; 3 | import java.sql.SQLException; 4 | import java.sql.PreparedStatement; 5 | import java.sql.Types; 6 | import java.sql.Timestamp; 7 | import java.sql.Statement; 8 | import java.sql.ResultSet; 9 | 10 | import java.util.Arrays; 11 | import java.util.ArrayList; 12 | import java.util.Date; 13 | import java.util.Properties; 14 | import java.io.BufferedWriter; 15 | import java.io.FileWriter; 16 | import java.io.File; 17 | import java.io.Writer; 18 | import java.io.FileNotFoundException; 19 | import java.io.IOException; 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | import java.util.concurrent.atomic.AtomicLong; 22 | import java.util.concurrent.locks.ReentrantLock; 23 | 24 | public class jiibench { 25 | public static AtomicLong globalInserts = new AtomicLong(0); 26 | public static AtomicLong globalWriterThreads = new AtomicLong(0); 27 | public static AtomicLong globalQueryThreads = new AtomicLong(0); 28 | public static AtomicLong globalQueriesExecuted = new AtomicLong(0); 29 | public static AtomicLong globalQueriesTimeMs = new AtomicLong(0); 30 | public static AtomicLong globalQueriesStarted = new AtomicLong(0); 31 | public static AtomicLong globalInsertExceptions = new AtomicLong(0); 32 | 33 | public static Writer writer = null; 34 | public static boolean outputHeader = true; 35 | 36 | public static int numCashRegisters = 1000; 37 | public static int numProducts = 10000; 38 | public static int numCustomers = 100000; 39 | public static double maxPrice = 500.0; 40 | 41 | public static String dbName; 42 | public static int writerThreads; 43 | public static Integer numMaxInserts; 44 | public static int rowsPerInsert; 45 | public static long insertsPerFeedback; 46 | public static long secondsPerFeedback; 47 | public static String logFileName; 48 | public static String indexTechnology; 49 | public static Long numSeconds; 50 | public static Integer queriesPerInterval; 51 | public static Integer queryIntervalSeconds; 52 | public static double queriesPerMinute; 53 | public static Long msBetweenQueries; 54 | public static Integer queryLimit; 55 | public static Integer queryBeginNumRows; 56 | public static Integer maxInsertsPerSecond; 57 | public static Integer maxThreadInsertsPerSecond; 58 | public static int numCharFields; 59 | public static int lengthCharFields; 60 | public static int numSecondaryIndexes; 61 | public static int percentCompressible; 62 | public static int numCompressibleCharacters; 63 | public static int numUncompressibleCharacters; 64 | 65 | public static int randomStringLength = 4*1024*1024; 66 | public static String randomStringHolder; 67 | public static int compressibleStringLength = 4*1024*1024; 68 | public static String compressibleStringHolder; 69 | 70 | public static String tableName = "purchases_index"; 71 | 72 | public static String storageEngine; 73 | public static int innodbKeyBlockSize; 74 | private static int[] validInnodbKeyBlockSizes = new int[] {1,2,4,8,16}; 75 | 76 | public static String mysqlServer; 77 | public static String mysqlPort; 78 | public static String mysqlUsername; 79 | public static String mysqlPassword; 80 | public static String createTable; 81 | 82 | public static int allDone = 0; 83 | 84 | public jiibench() { 85 | } 86 | 87 | public static void main (String[] args) throws Exception { 88 | if (args.length != 24) { 89 | logMe("*** ERROR : CONFIGURATION ISSUE ***"); 90 | logMe("jiibench [database name] [number of writer threads] [rows per table] [rows per insert] [inserts feedback] "+ 91 | "[seconds feedback] [log file name] [number of seconds to run] [queries per interval] [interval (seconds)] "+ 92 | "[query limit] [inserts for begin query] [max inserts per second] [num char fields] [length char fields] "+ 93 | "[num secondary indexes] [percent compressible] [mysql port] [mysql storage engine] [innodb key block size, 0 if none] "+ 94 | "[mysql server] [mysql username] [mysql password] [create table Y/N]"); 95 | System.exit(1); 96 | } 97 | 98 | dbName = args[0]; 99 | writerThreads = Integer.valueOf(args[1]); 100 | numMaxInserts = Integer.valueOf(args[2]); 101 | rowsPerInsert = Integer.valueOf(args[3]); 102 | insertsPerFeedback = Long.valueOf(args[4]); 103 | secondsPerFeedback = Long.valueOf(args[5]); 104 | logFileName = args[6]; 105 | numSeconds = Long.valueOf(args[7]); 106 | queriesPerInterval = Integer.valueOf(args[8]); 107 | queryIntervalSeconds = Integer.valueOf(args[9]); 108 | queryLimit = Integer.valueOf(args[10]); 109 | queryBeginNumRows = Integer.valueOf(args[11]); 110 | maxInsertsPerSecond = Integer.valueOf(args[12]); 111 | numCharFields = Integer.valueOf(args[13]); 112 | lengthCharFields = Integer.valueOf(args[14]); 113 | numSecondaryIndexes = Integer.valueOf(args[15]); 114 | percentCompressible = Integer.valueOf(args[16]); 115 | mysqlPort = args[17]; 116 | storageEngine = args[18].toLowerCase(); 117 | innodbKeyBlockSize = Integer.valueOf(args[19]); 118 | mysqlServer = args[20]; 119 | mysqlUsername = args[21]; 120 | mysqlPassword = args[22]; 121 | createTable = args[23].toLowerCase(); 122 | 123 | maxThreadInsertsPerSecond = (maxInsertsPerSecond / writerThreads); 124 | 125 | if ((numSecondaryIndexes < 0) || (numSecondaryIndexes > 3)) { 126 | logMe("*** ERROR : INVALID NUMBER OF SECONDARY INDEXES, MUST BE >= 0 and <= 3 ***"); 127 | logMe(" %d secondary indexes is not supported",numSecondaryIndexes); 128 | System.exit(1); 129 | } 130 | 131 | if (storageEngine.equals("innodb") && innodbKeyBlockSize != 0 132 | && !Arrays.asList(validInnodbKeyBlockSizes).contains(innodbKeyBlockSize)) { 133 | logMe("*** ERROR : INVALID INNODB KEY BLOCK SIZE, MUST BE 1, 2, 4, 8 or 16 ***"); 134 | logMe(" innodb key block size of %d is not supported",innodbKeyBlockSize); 135 | System.exit(1); 136 | } 137 | 138 | if ((queriesPerInterval <= 0) || (queryIntervalSeconds <= 0)) 139 | { 140 | queriesPerMinute = 0.0; 141 | msBetweenQueries = 0l; 142 | } 143 | else 144 | { 145 | queriesPerMinute = (double)queriesPerInterval * (60.0 / (double)queryIntervalSeconds); 146 | msBetweenQueries = (long)((1000.0 * (double)queryIntervalSeconds) / (double)queriesPerInterval); 147 | } 148 | 149 | if ((percentCompressible < 0) || (percentCompressible > 100)) { 150 | logMe("*** ERROR : INVALID PERCENT COMPRESSIBLE, MUST BE >=0 and <= 100 ***"); 151 | logMe(" %d secondary indexes is not supported",percentCompressible); 152 | System.exit(1); 153 | } 154 | 155 | numCompressibleCharacters = (int) (((double) percentCompressible / 100.0) * (double) lengthCharFields); 156 | numUncompressibleCharacters = (int) (((100.0 - (double) percentCompressible) / 100.0) * (double) lengthCharFields); 157 | 158 | logMe("Application Parameters"); 159 | logMe("--------------------------------------------------"); 160 | logMe(" database name = %s",dbName); 161 | logMe(" %d writer thread(s)",writerThreads); 162 | logMe(" %,d rows per table",numMaxInserts); 163 | logMe(" %d character fields",numCharFields); 164 | if (numCharFields > 0) 165 | { 166 | logMe(" %d bytes per character field",lengthCharFields); 167 | logMe(" %d percent compressible",percentCompressible); 168 | } 169 | logMe(" %d secondary indexes",numSecondaryIndexes); 170 | logMe(" Rows Per Insert = %d",rowsPerInsert); 171 | logMe(" Maximum of %,d insert(s) per second",maxInsertsPerSecond); 172 | if (writerThreads > 0) 173 | { 174 | logMe(" Maximum of %,d insert(s) per second per writer thread",maxThreadInsertsPerSecond); 175 | } 176 | logMe(" Feedback every %,d seconds(s)",secondsPerFeedback); 177 | logMe(" Feedback every %,d inserts(s)",insertsPerFeedback); 178 | logMe(" logging to file %s",logFileName); 179 | logMe(" Run for %,d second(s)",numSeconds); 180 | if (queriesPerMinute > 0.0) 181 | { 182 | logMe(" Attempting %,.2f queries per minute",queriesPerMinute); 183 | logMe(" Queries limited to %,d row(s)",queryLimit); 184 | logMe(" Starting queries after %,d row(s) inserted",queryBeginNumRows); 185 | } 186 | else 187 | { 188 | logMe(" NO queries, insert only benchmark"); 189 | } 190 | logMe(" MySQL Server= %s",mysqlServer); 191 | logMe(" MySQL Port= %s",mysqlPort); 192 | logMe(" MySQL Username= %s",mysqlUsername); 193 | logMe(" MySQL Storage Engine= %s",storageEngine); 194 | if (storageEngine.equals("innodb") && innodbKeyBlockSize != 0) 195 | { 196 | logMe(" InnoDB Key Block Size = %d",innodbKeyBlockSize); 197 | } 198 | logMe("--------------------------------------------------"); 199 | 200 | try { 201 | // The newInstance() call is a work around for some broken Java implementations 202 | Class.forName("com.mysql.jdbc.Driver").newInstance(); 203 | } catch (Exception ex) { 204 | // handle the error 205 | logMe("*** ERROR : MySQL JDBC driver not found, have you set your CLASSPATH? ***"); 206 | ex.printStackTrace(); 207 | System.exit(1); 208 | } 209 | 210 | if (writerThreads > 1) { 211 | numMaxInserts = numMaxInserts / writerThreads; 212 | } 213 | 214 | try { 215 | writer = new BufferedWriter(new FileWriter(new File(logFileName))); 216 | } catch (IOException e) { 217 | e.printStackTrace(); 218 | } 219 | 220 | if (createTable.equals("n")) 221 | { 222 | logMe("Skipping table creation"); 223 | } 224 | else 225 | { 226 | // create the table 227 | Connection conn = null; 228 | Statement stmt = null; 229 | 230 | try { 231 | conn = DriverManager.getConnection("jdbc:mysql://"+mysqlServer+":"+mysqlPort+"/"+dbName+"?user="+mysqlUsername+"&password="+mysqlPassword+"&rewriteBatchedStatements=true"); 232 | stmt = conn.createStatement(); 233 | 234 | // drop the table if it exists 235 | String sqlDrop = "drop table if exists " + tableName; 236 | stmt.executeUpdate(sqlDrop); 237 | 238 | // create the table 239 | String sqlCharFields = ""; 240 | for (int i = 1; i <= numCharFields; i++) { 241 | sqlCharFields += " charfield" + Integer.toString(i) + " varchar(" + lengthCharFields + ") not null, "; 242 | } 243 | 244 | // innodb compression 245 | String sqlKeyBlockSize = ""; 246 | if (storageEngine.equals("innodb") && innodbKeyBlockSize != 0) { 247 | sqlKeyBlockSize = " ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=" + Integer.toString(innodbKeyBlockSize); 248 | } 249 | 250 | String sqlCreate = "create table " + tableName + " (" + 251 | "transactionid int not null auto_increment, " + 252 | "dateandtime datetime, " + 253 | "cashregisterid int not null, " + 254 | "customerid int not null, " + 255 | "productid int not null, " + 256 | "price float not null, " + 257 | sqlCharFields + 258 | "primary key (transactionid) " + 259 | ") engine=" + storageEngine + sqlKeyBlockSize; 260 | stmt.executeUpdate(sqlCreate); 261 | 262 | if (numSecondaryIndexes >= 1) { 263 | logMe(" *** creating secondary index marketsegment (price, customerid)"); 264 | String sqlIndex1 = "create index marketsegment on " + tableName + " (price, customerid)"; 265 | stmt.executeUpdate(sqlIndex1); 266 | } 267 | if (numSecondaryIndexes >= 2) { 268 | logMe(" *** creating secondary index registersegment (cashregisterid, price, customerid)"); 269 | String sqlIndex2 = "create index registersegment on " + tableName + " (cashregisterid, price, customerid)"; 270 | stmt.executeUpdate(sqlIndex2); 271 | } 272 | if (numSecondaryIndexes >= 3) { 273 | logMe(" *** creating secondary index pdc (price, dateandtime, customerid)"); 274 | String sqlIndex3 = "create index pdc on " + tableName + " (price, dateandtime, customerid)"; 275 | stmt.executeUpdate(sqlIndex3); 276 | } 277 | 278 | } catch (SQLException ex) { 279 | // handle any errors 280 | System.out.println("SQLException: " + ex.getMessage()); 281 | System.out.println("SQLState: " + ex.getSQLState()); 282 | System.out.println("VendorError: " + ex.getErrorCode()); 283 | } 284 | } 285 | 286 | java.util.Random rand = new java.util.Random(); 287 | 288 | // create random string holder 289 | logMe(" creating %,d bytes of random character data...",randomStringLength); 290 | char[] tempString = new char[randomStringLength]; 291 | for (int i = 0 ; i < randomStringLength ; i++) { 292 | tempString[i] = (char) (rand.nextInt(26) + 'a'); 293 | } 294 | randomStringHolder = new String(tempString); 295 | 296 | // create compressible string holder 297 | logMe(" creating %,d bytes of compressible character data...",compressibleStringLength); 298 | char[] tempStringCompressible = new char[compressibleStringLength]; 299 | for (int i = 0 ; i < compressibleStringLength ; i++) { 300 | tempStringCompressible[i] = 'a'; 301 | } 302 | compressibleStringHolder = new String(tempStringCompressible); 303 | 304 | 305 | jiibench t = new jiibench(); 306 | 307 | Thread reporterThread = new Thread(t.new MyReporter()); 308 | reporterThread.start(); 309 | 310 | Thread queryThread = new Thread(t.new MyQuery(1, 1, numMaxInserts)); 311 | if (queriesPerMinute > 0.0) { 312 | queryThread.start(); 313 | } 314 | 315 | Thread[] tWriterThreads = new Thread[writerThreads]; 316 | 317 | // start the loaders 318 | for (int i=0; i= maxInsertsPerSecond) { 403 | // pause until a second has passed 404 | while (System.currentTimeMillis() < nextMs) { 405 | try { 406 | Thread.sleep(20); 407 | } catch (Exception e) { 408 | e.printStackTrace(); 409 | } 410 | } 411 | numLastInserts = numInserts; 412 | nextMs = System.currentTimeMillis() + 1000; 413 | } 414 | 415 | Timestamp tsBatch = new Timestamp(System.currentTimeMillis()); 416 | 417 | for (int i = 0; i < rowsPerInsert; i++) { 418 | //id++; 419 | int thisCustomerId = rand.nextInt(numCustomers); 420 | double thisPrice= ((rand.nextDouble() * maxPrice) + (double) thisCustomerId) / 100.0; 421 | 422 | pstmt.setTimestamp(1, tsBatch); 423 | pstmt.setInt(2, rand.nextInt(numCashRegisters)); 424 | pstmt.setInt(3, thisCustomerId); 425 | pstmt.setInt(4, rand.nextInt(numProducts)); 426 | pstmt.setDouble(5, thisPrice); 427 | 428 | for (int charField = 1; charField <= numCharFields; charField++) { 429 | int startPosition = rand.nextInt(randomStringLength-lengthCharFields); 430 | //doc.put("cf"+Integer.toString(charField), randomStringHolder.substring(startPosition,startPosition+numUncompressibleCharacters) + compressibleStringHolder.substring(startPosition,startPosition+numCompressibleCharacters)); 431 | pstmt.setString(5+charField, randomStringHolder.substring(startPosition,startPosition+numUncompressibleCharacters) + compressibleStringHolder.substring(startPosition,startPosition+numCompressibleCharacters)); 432 | } 433 | 434 | pstmt.addBatch(); 435 | } 436 | 437 | try { 438 | pstmt.executeBatch(); 439 | //conn.commit(); 440 | numInserts += rowsPerInsert; 441 | globalInserts.addAndGet(rowsPerInsert); 442 | 443 | } catch (Exception e) { 444 | logMe("Writer thread %d : EXCEPTION",threadNumber); 445 | e.printStackTrace(); 446 | globalInsertExceptions.incrementAndGet(); 447 | } 448 | 449 | if (allDone == 1) 450 | break; 451 | } 452 | 453 | } catch (Exception e) { 454 | logMe("Writer thread %d : EXCEPTION",threadNumber); 455 | e.printStackTrace(); 456 | } 457 | 458 | long numWriters = globalWriterThreads.decrementAndGet(); 459 | if (numWriters == 0) 460 | allDone = 1; 461 | } 462 | } 463 | 464 | 465 | class MyQuery implements Runnable { 466 | int threadCount; 467 | int threadNumber; 468 | int numMaxInserts; 469 | 470 | java.util.Random rand; 471 | 472 | MyQuery(int threadCount, int threadNumber, int numMaxInserts) { 473 | this.threadCount = threadCount; 474 | this.threadNumber = threadNumber; 475 | this.numMaxInserts = numMaxInserts; 476 | rand = new java.util.Random((long) threadNumber); 477 | } 478 | public void run() { 479 | Connection conn = null; 480 | PreparedStatement pstmt1 = null; 481 | PreparedStatement pstmt2 = null; 482 | PreparedStatement pstmt3 = null; 483 | PreparedStatement pstmt4 = null; 484 | 485 | String sqlCharFields = ""; 486 | String sqlCharFieldPlaceHolders = ""; 487 | for (int i = 1; i <= numCharFields; i++) { 488 | sqlCharFields += ",charfield" + Integer.toString(i); 489 | sqlCharFieldPlaceHolders += ",?"; 490 | } 491 | 492 | try { 493 | conn = DriverManager.getConnection("jdbc:mysql://localhost:"+mysqlPort+"/"+dbName+"?user="+mysqlUsername+"&password="+mysqlPassword+"rewriteBatchedStatements=true"); 494 | 495 | // prepare the 4 possible queries 496 | pstmt1 = conn.prepareStatement("select transactionid from "+tableName+" where (transactionid >= ?) limit ?"); 497 | 498 | pstmt2 = conn.prepareStatement("select price,dateandtime,customerid from "+tableName+" FORCE INDEX (pdc) where " + 499 | "(price=? and dateandtime=? and customerid>=?) OR " + 500 | "(price=? and dateandtime>?) OR " + 501 | "(price>?) LIMIT ?"); 502 | 503 | pstmt3 = conn.prepareStatement("select price,customerid from "+tableName+" FORCE INDEX (marketsegment) where " + 504 | "(price=? and customerid>=?) OR " + 505 | "(price>?) LIMIT ?"); 506 | 507 | pstmt4 = conn.prepareStatement("select cashregisterid,price,customerid from "+tableName+" FORCE INDEX (registersegment) where " + 508 | "(cashregisterid=? and price=? and customerid>=?) OR " + 509 | "(cashregisterid=? and price>?) OR " + 510 | "(cashregisterid>?) LIMIT ?"); 511 | 512 | } catch (SQLException ex) { 513 | // handle any errors 514 | System.out.println("SQLException: " + ex.getMessage()); 515 | System.out.println("SQLState: " + ex.getSQLState()); 516 | System.out.println("VendorError: " + ex.getErrorCode()); 517 | } 518 | 519 | long t0 = System.currentTimeMillis(); 520 | long lastMs = t0; 521 | long nextQueryMillis = t0; 522 | boolean outputWaiting = true; 523 | boolean outputStarted = true; 524 | 525 | long numQueriesExecuted = 0; 526 | long numQueriesTimeMs = 0; 527 | 528 | int whichQuery = 0; 529 | 530 | try { 531 | logMe("Query thread %d : ready to query table %s",threadNumber, tableName); 532 | 533 | while (allDone == 0) { 534 | //try { 535 | // Thread.sleep(10); 536 | //} catch (Exception e) { 537 | // e.printStackTrace(); 538 | // } 539 | 540 | long thisNow = System.currentTimeMillis(); 541 | 542 | // wait until my next runtime 543 | if (thisNow > nextQueryMillis) { 544 | nextQueryMillis = thisNow + msBetweenQueries; 545 | 546 | // check if number of inserts reached 547 | if (globalInserts.get() >= queryBeginNumRows) { 548 | if (outputStarted) 549 | { 550 | logMe("Query thread %d : now running",threadNumber,queryBeginNumRows); 551 | outputStarted = false; 552 | // set query start time 553 | globalQueriesStarted.set(thisNow); 554 | } 555 | 556 | whichQuery++; 557 | if (whichQuery > 4) { 558 | whichQuery = 1; 559 | } 560 | 561 | long thisRandomTransactionId = rand.nextLong() % globalInserts.get(); 562 | int thisCustomerId = rand.nextInt(numCustomers); 563 | double thisPrice = ((rand.nextDouble() * maxPrice) + (double) thisCustomerId) / 100.0; 564 | int thisCashRegisterId = rand.nextInt(numCashRegisters); 565 | int thisProductId = rand.nextInt(numProducts); 566 | long thisRandomTime = t0 + (long) ((double) (thisNow - t0) * rand.nextDouble()); 567 | Timestamp thisRandomTimestamp = new Timestamp(thisRandomTime); 568 | 569 | long now = System.currentTimeMillis(); 570 | if (whichQuery == 1) { 571 | pstmt1.setLong(1, thisRandomTransactionId); 572 | pstmt1.setInt(2, queryLimit); 573 | ResultSet rs = pstmt1.executeQuery(); 574 | /* 575 | int count = 0; 576 | while (rs.next()) { 577 | ++count; 578 | } 579 | logMe("Query %d : %,d row(s)",whichQuery,count); 580 | */ 581 | } else if (whichQuery == 2) { 582 | pstmt2.setDouble(1, thisPrice); 583 | pstmt2.setTimestamp(2, thisRandomTimestamp); 584 | pstmt2.setInt(3, thisCustomerId); 585 | pstmt2.setDouble(4, thisPrice); 586 | pstmt2.setTimestamp(5, thisRandomTimestamp); 587 | pstmt2.setDouble(6, thisPrice); 588 | pstmt2.setInt(7, queryLimit); 589 | ResultSet rs = pstmt2.executeQuery(); 590 | /* 591 | int count = 0; 592 | while (rs.next()) { 593 | ++count; 594 | } 595 | logMe("Query %d : %,d row(s)",whichQuery,count); 596 | */ 597 | } else if (whichQuery == 3) { 598 | pstmt3.setDouble(1, thisPrice); 599 | pstmt3.setInt(2, thisCustomerId); 600 | pstmt3.setDouble(3, thisPrice); 601 | pstmt3.setInt(4, queryLimit); 602 | ResultSet rs = pstmt3.executeQuery(); 603 | /* 604 | int count = 0; 605 | while (rs.next()) { 606 | ++count; 607 | } 608 | logMe("Query %d : %,d row(s)",whichQuery,count); 609 | */ 610 | } else if (whichQuery == 4) { 611 | pstmt4.setInt(1, thisCashRegisterId); 612 | pstmt4.setDouble(2, thisPrice); 613 | pstmt4.setInt(3, thisCustomerId); 614 | pstmt4.setInt(4, thisCashRegisterId); 615 | pstmt4.setDouble(5, thisPrice); 616 | pstmt4.setInt(6, thisCashRegisterId); 617 | pstmt4.setInt(7, queryLimit); 618 | ResultSet rs = pstmt4.executeQuery(); 619 | /* 620 | int count = 0; 621 | while (rs.next()) { 622 | ++count; 623 | } 624 | logMe("Query %d : %,d row(s)",whichQuery,count); 625 | */ 626 | } 627 | long elapsed = System.currentTimeMillis() - now; 628 | 629 | globalQueriesExecuted.incrementAndGet(); 630 | globalQueriesTimeMs.addAndGet(elapsed); 631 | } else { 632 | if (outputWaiting) 633 | { 634 | logMe("Query thread %d : waiting for %,d row insert(s) before starting",threadNumber,queryBeginNumRows); 635 | outputWaiting = false; 636 | } 637 | } 638 | } 639 | } 640 | 641 | } catch (Exception e) { 642 | logMe("Query thread %d : EXCEPTION",threadNumber); 643 | e.printStackTrace(); 644 | } 645 | 646 | long numQueries = globalQueryThreads.decrementAndGet(); 647 | } 648 | } 649 | 650 | 651 | // reporting thread, outputs information to console and file 652 | class MyReporter implements Runnable { 653 | public void run() 654 | { 655 | long t0 = System.currentTimeMillis(); 656 | long lastInserts = 0; 657 | long lastQueriesNum = 0; 658 | long lastQueriesMs = 0; 659 | long lastMs = t0; 660 | long intervalNumber = 0; 661 | long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 662 | long nextFeedbackInserts = lastInserts + insertsPerFeedback; 663 | long thisInserts = 0; 664 | long thisQueriesNum = 0; 665 | long thisQueriesMs = 0; 666 | long thisQueriesStarted = 0; 667 | long endDueToTime = System.currentTimeMillis() + (1000 * numSeconds); 668 | 669 | while (allDone == 0) 670 | { 671 | try { 672 | Thread.sleep(100); 673 | } catch (Exception e) { 674 | e.printStackTrace(); 675 | } 676 | 677 | long now = System.currentTimeMillis(); 678 | 679 | if (now >= endDueToTime) 680 | { 681 | allDone = 1; 682 | } 683 | 684 | thisInserts = globalInserts.get(); 685 | thisQueriesNum = globalQueriesExecuted.get(); 686 | thisQueriesMs = globalQueriesTimeMs.get(); 687 | thisQueriesStarted = globalQueriesStarted.get(); 688 | if (((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) || 689 | ((thisInserts >= nextFeedbackInserts) && (insertsPerFeedback > 0))) 690 | { 691 | intervalNumber++; 692 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 693 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 694 | 695 | long elapsed = now - t0; 696 | long thisIntervalMs = now - lastMs; 697 | 698 | long thisIntervalInserts = thisInserts - lastInserts; 699 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 700 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 701 | 702 | long thisIntervalQueriesNum = thisQueriesNum - lastQueriesNum; 703 | long thisIntervalQueriesMs = thisQueriesMs - lastQueriesMs; 704 | double thisIntervalQueryAvgMs = 0; 705 | double thisQueryAvgMs = 0; 706 | double thisIntervalAvgQPM = 0; 707 | double thisAvgQPM = 0; 708 | 709 | long thisInsertExceptions = globalInsertExceptions.get(); 710 | 711 | if (thisIntervalQueriesNum > 0) { 712 | thisIntervalQueryAvgMs = thisIntervalQueriesMs/(double)thisIntervalQueriesNum; 713 | } 714 | if (thisQueriesNum > 0) { 715 | thisQueryAvgMs = thisQueriesMs/(double)thisQueriesNum; 716 | } 717 | 718 | if (thisQueriesStarted > 0) 719 | { 720 | long adjustedElapsed = now - thisQueriesStarted; 721 | if (adjustedElapsed > 0) 722 | { 723 | thisAvgQPM = (double)thisQueriesNum/((double)adjustedElapsed/1000.0/60.0); 724 | } 725 | if (thisIntervalMs > 0) 726 | { 727 | thisIntervalAvgQPM = (double)thisIntervalQueriesNum/((double)thisIntervalMs/1000.0/60.0); 728 | } 729 | } 730 | 731 | if (secondsPerFeedback > 0) 732 | { 733 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f : exceptions=%,d", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 734 | } else { 735 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f : exceptions=%,d", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 736 | } 737 | 738 | try { 739 | if (outputHeader) 740 | { 741 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\tcum_qry_avg\tint_qry_avg\tcum_qpm\tint_qpm\texceptions\n"); 742 | outputHeader = false; 743 | } 744 | 745 | String statusUpdate = ""; 746 | 747 | if (secondsPerFeedback > 0) 748 | { 749 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%,d\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 750 | } else { 751 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%,d\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 752 | } 753 | writer.write(statusUpdate); 754 | writer.flush(); 755 | } catch (IOException e) { 756 | e.printStackTrace(); 757 | } 758 | 759 | lastInserts = thisInserts; 760 | lastQueriesNum = thisQueriesNum; 761 | lastQueriesMs = thisQueriesMs; 762 | 763 | lastMs = now; 764 | } 765 | } 766 | 767 | // output final numbers... 768 | long now = System.currentTimeMillis(); 769 | thisInserts = globalInserts.get(); 770 | thisQueriesNum = globalQueriesExecuted.get(); 771 | thisQueriesMs = globalQueriesTimeMs.get(); 772 | thisQueriesStarted = globalQueriesStarted.get(); 773 | intervalNumber++; 774 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 775 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 776 | long elapsed = now - t0; 777 | long thisIntervalMs = now - lastMs; 778 | long thisIntervalInserts = thisInserts - lastInserts; 779 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 780 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 781 | long thisIntervalQueriesNum = thisQueriesNum - lastQueriesNum; 782 | long thisIntervalQueriesMs = thisQueriesMs - lastQueriesMs; 783 | double thisIntervalQueryAvgMs = 0; 784 | double thisQueryAvgMs = 0; 785 | double thisIntervalAvgQPM = 0; 786 | double thisAvgQPM = 0; 787 | if (thisIntervalQueriesNum > 0) { 788 | thisIntervalQueryAvgMs = thisIntervalQueriesMs/(double)thisIntervalQueriesNum; 789 | } 790 | if (thisQueriesNum > 0) { 791 | thisQueryAvgMs = thisQueriesMs/(double)thisQueriesNum; 792 | } 793 | if (thisQueriesStarted > 0) 794 | { 795 | long adjustedElapsed = now - thisQueriesStarted; 796 | if (adjustedElapsed > 0) 797 | { 798 | thisAvgQPM = (double)thisQueriesNum/((double)adjustedElapsed/1000.0/60.0); 799 | } 800 | if (thisIntervalMs > 0) 801 | { 802 | thisIntervalAvgQPM = (double)thisIntervalQueriesNum/((double)thisIntervalMs/1000.0/60.0); 803 | } 804 | } 805 | if (secondsPerFeedback > 0) 806 | { 807 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 808 | } else { 809 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 810 | } 811 | try { 812 | if (outputHeader) 813 | { 814 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\tcum_qry_avg\tint_qry_avg\tcum_qpm\tint_qpm\n"); 815 | outputHeader = false; 816 | } 817 | String statusUpdate = ""; 818 | if (secondsPerFeedback > 0) 819 | { 820 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 821 | } else { 822 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 823 | } 824 | writer.write(statusUpdate); 825 | writer.flush(); 826 | } catch (IOException e) { 827 | e.printStackTrace(); 828 | } 829 | 830 | } 831 | } 832 | 833 | 834 | public static void logMe(String format, Object... args) { 835 | System.out.println(Thread.currentThread() + String.format(format, args)); 836 | } 837 | } 838 | --------------------------------------------------------------------------------