├── check-results.bash
├── TODO
├── .gitignore
├── README.md
├── config.bash
├── run.simple.bash
├── run.looped.bash
├── LICENSE
└── src
    ├── jmongosysbenchload.java
    ├── jmongosysbenchshardedload.java
    ├── jmongosysbenchexecute.java
    └── jmongosysbenchshardedexecute.java


/check-results.bash:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 | 
3 | grep '280' log/*.txt.tsv | cut -f2
4 | 


--------------------------------------------------------------------------------
/TODO:
--------------------------------------------------------------------------------
1 | - add insert governer for loader
2 | - improve command line argument passing
3 |   - use "--argument value" syntax
4 |   - add defaults
5 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *.class
 2 | .classpath
 3 | .project
 4 | 
 5 | # Package Files #
 6 | *.jar
 7 | *.war
 8 | *.earA
 9 | 
10 | # Specific Files #
11 | config.tmc
12 | config.tmc.*
13 | config.anshu.*
14 | config.01
15 | config.02
16 | config.03
17 | config.04
18 | config.05
19 | config.06
20 | config.07
21 | config.08
22 | config.10
23 | clean.bash
24 | rds-truststore.jks
25 | log
26 | mongoSysbenchExecute*.txt
27 | mongoSysbenchExecute*.txt.tsv
28 | mongoSysbenchLoad*.txt
29 | mongoSysbenchLoad*.txt.tsv
30 | run.multi.bash
31 | run.tmc.bash
32 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | <h3>*DEPRECATED*</h3>
 2 | 
 3 | This repository is no longer maintained. A simpler Python implementation is available as [py-mongo-sysbench](https://github.com/tmcallaghan/py-mongo-sysbench)
 4 | 
 5 | 
 6 | sysbench-mongodb
 7 | ================
 8 | 
 9 | Sysbench benchmark for MongoDB compatible databases
10 | 
11 | 
12 | Requirements
13 | =====================
14 | 
15 | * Java 17 (tested working with OpenJDK)
16 | * MongoDB Java driver jars - specifically version 3.9.1 (for now)
17 |   * they must be in the working directory (for now)
18 |   * wget https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongo-java-driver/3.9.1/mongo-java-driver-3.9.1.jar
19 |   * wget https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongodb-driver-core/3.9.1/mongodb-driver-core-3.9.1.jar
20 |   * wget https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongodb-driver-legacy/3.9.1/mongodb-driver-legacy-3.9.1.jar
21 | * Modify config.bash file for URI (connection string)
22 | * TLS is optional
23 |   * If not using TLS, simply disable it in config.bash or your custom config file
24 |   * Otherwise you'll need to create your key store file. Instructions are available at https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html
25 | 
26 | 
27 | Running the benchmark
28 | =====================
29 | 
30 | In the default configuration the benchmark creates 1 collection with 1 million documents. You may want to watch the size of the database relative to your memory size to ensure you are testing just a memory based workload vs a workload that is exceeding memory and utilizing disk as well. All options are configurable in config.bash (or custom config file with the same options)
31 | 
32 | To run:
33 | 
34 | ```bash
35 | git clone https://github.com/tmcallaghan/sysbench-mongodb.git
36 | cd sysbench-mongodb
37 | 
38 | ```
39 | 
40 | Edit config.bash to match your environment, you must change the URI for connecting to your server.
41 | 
42 | ```bash
43 | ./run.simple.bash
44 | 
45 | ```
46 | 
47 | If you want to have multiple config files you can simply copy config.bash and specify the config you would like on the command line:
48 | 
49 | ```bash
50 | ./run.simple.bash my_custom_config.bash
51 | 
52 | ```
53 | 


--------------------------------------------------------------------------------
/config.bash:
--------------------------------------------------------------------------------
  1 | # You will most likely want to change these values
  2 | 
  3 | export SHARDED=no
  4 | 
  5 | # database in which to run the benchmark
  6 | export DB_NAME=sbtest10by1mm
  7 | 
  8 | # URI connection string
  9 | export URI=<replace-with-proper-URI>
 10 | 
 11 | # trust store file
 12 | #   leave this alone if not using TLS
 13 | export TRUST_STORE="./rds-truststore.jks"
 14 | 
 15 | # trust store password
 16 | #   leave this alone if not using TLS
 17 | export TRUST_STORE_PASSWORD="<replace-with-trust-store-password>"
 18 | 
 19 | # Use "yes" to load the collections
 20 | export DOLOAD=yes
 21 | 
 22 | # Use "yes" to execute the benchmark
 23 | export DOQUERY=yes
 24 | 
 25 | # number of collections to create for the benchmark
 26 | #   valid values : integer > 0
 27 | export NUM_COLLECTIONS=1
 28 | 
 29 | # number of documents to maintain per collection
 30 | #   valid values : integer > 0
 31 | export NUM_DOCUMENTS_PER_COLLECTION=1000000
 32 | 
 33 | # total number of documents to insert per "batch"
 34 | #   valid values : integer > 0
 35 | export NUM_DOCUMENTS_PER_INSERT=1000
 36 | 
 37 | # total number of simultaneous insertion threads (for loader)
 38 | #   valid values : integer > 0
 39 | export NUM_LOADER_THREADS=10
 40 | 
 41 | # total number of simultaneous benchmark threads
 42 | #   valid values : integer > 0
 43 | export NUM_WRITER_THREADS=10
 44 | 
 45 | # run the benchmark for this many minutes
 46 | #   valid values : intever > 0
 47 | export RUN_TIME_MINUTES=10
 48 | export RUN_TIME_SECONDS=$[RUN_TIME_MINUTES*60]
 49 | 
 50 | # total number of transactions per second, allows for the benchmark to be rate limited
 51 | #   valid values : integer > 0
 52 | export MAX_TPS=999999
 53 | 
 54 | 
 55 | # ------------------------------------------------------------------------------------
 56 | # Sysbench specific parameters
 57 | # ------------------------------------------------------------------------------------
 58 | 
 59 | # number of documents to retrieve in range queries
 60 | #   valid values : integer > 0
 61 | export SYSBENCH_RANGE_SIZE=100
 62 | 
 63 | # number of point queries per sysbench "transaction"
 64 | #   valid values : integer >= 0
 65 | export SYSBENCH_POINT_SELECTS=10
 66 | 
 67 | # number of simple range queries per sysbench "transaction"
 68 | #   valid values : integer >= 0
 69 | export SYSBENCH_SIMPLE_RANGES=1
 70 | 
 71 | # number of aggregation queries per sysbench "transaction"
 72 | #   valid values : integer >= 0
 73 | export SYSBENCH_SUM_RANGES=1
 74 | 
 75 | # number of ordered range queries per sysbench "transaction"
 76 | #   valid values : integer >= 0
 77 | export SYSBENCH_ORDER_RANGES=1
 78 | 
 79 | # number of distinct range queries per sysbench "transaction"
 80 | #   valid values : integer >= 0
 81 | export SYSBENCH_DISTINCT_RANGES=1
 82 | 
 83 | # set all of the following 3 parameters to zero for a read-only benchmark
 84 | 
 85 | # number of indexed updates per sysbench "transaction"
 86 | #   valid values : integer >= 0
 87 | export SYSBENCH_INDEX_UPDATES=1
 88 | 
 89 | # number of non-indexed updates per sysbench "transaction"
 90 | #   valid values : integer >= 0
 91 | export SYSBENCH_NON_INDEX_UPDATES=1
 92 | 
 93 | # number of delete/insert operations per sysbench "transaction"
 94 | #   valid values : integer >= 0
 95 | export SYSBENCH_INSERTS=1
 96 | 
 97 | 
 98 | # ------------------------------------------------------------------------------------
 99 | # Parameters to leave alone
100 | # ------------------------------------------------------------------------------------
101 | 
102 | # Use a seed for the RNG, like $( date +%s )
103 | #   This can be used to repeat, or not, the sequence of keys used per test.
104 | export SEED=$( date +%s )
105 | 
106 | # display performance information every time the client application inserts this many documents
107 | #   valid values : integer > 0, set to -1 if using NUM_SECONDS_PER_FEEDBACK
108 | export NUM_INSERTS_PER_FEEDBACK=-1
109 | 
110 | # display performance information every time the client application has run for this many seconds
111 | #   valid values : integer > 0, set to -1 if using NUM_INSERTS_PER_FEEDBACK
112 | export NUM_SECONDS_PER_FEEDBACK=10
113 | 
114 | 
115 | 


--------------------------------------------------------------------------------
/run.simple.bash:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # enable passing different config files
 4 | 
 5 | if [ ! $1 ];
 6 | then
 7 |     FILE="config.bash"
 8 | else
 9 |     FILE=$1
10 | fi
11 | 
12 | if [ -f $FILE ];
13 | then
14 |    echo "Loading config from $FILE....."
15 |    source $FILE
16 | else
17 |    echo "Unable to read config $FILE"
18 |    exit 1
19 | fi
20 | 
21 | if [ $2 ];
22 | then
23 |    echo "Loading config overrides from $2....."
24 |     source $2
25 | fi
26 | 
27 | CLASSPATH="mongo-java-driver-3.9.1.jar:mongodb-driver-core-3.9.1.jar:mongodb-driver-legacy-3.9.1.jar:$CLASSPATH"
28 | TAIL_LINES=21
29 | 
30 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchload.java
31 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchexecute.java
32 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedload.java
33 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedexecute.java
34 | 
35 | 
36 | # load the data
37 | 
38 | if [[ $DOLOAD = "yes" ]]; then
39 |     echo Do load at $( date )
40 |     export LOG_NAME=mongoSysbenchLoad-${DOCDB_DBNAME}-${NUM_COLLECTIONS}-${NUM_DOCUMENTS_PER_COLLECTION}-${NUM_LOADER_THREADS}.txt
41 |     export BENCHMARK_TSV=${LOG_NAME}.tsv
42 |  
43 |     rm -f $LOG_NAME
44 |     rm -f $BENCHMARK_TSV
45 | 
46 |     T="$(date +%s)"
47 | 
48 |     if [[ $SHARDED = "no" ]]; then
49 |         echo "executing standard sysbench load (not sharded)" | tee -a $LOG_NAME
50 |         java -cp $CLASSPATH:$PWD/src jmongosysbenchload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $URI
51 |     else
52 |         echo "executing sharded sysbench load" | tee -a $LOG_NAME
53 |         java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $MAX_SHARD_KEY $DOCS_PER_SHARD $URI
54 |     fi
55 | 
56 |     echo "" | tee -a $LOG_NAME
57 |     T="$(($(date +%s)-T))"
58 |     printf "`date` | sysbench loader duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME
59 |     echo ""
60 |     echo "************************************************************************"
61 |     echo "final ${numTailLines} interval(s)"
62 |     echo "************************************************************************"
63 |     tail -n $TAIL_LINES $LOG_NAME
64 | fi
65 | 
66 | 
67 | # execute the benchmark
68 | 
69 | if [[ $DOQUERY = "yes" ]]; then
70 |     echo Do query at $( date )
71 |     export LOG_NAME=mongoSysbenchExecute-${DOCDB_DBNAME}-${NUM_COLLECTIONS}-${NUM_DOCUMENTS_PER_COLLECTION}-${NUM_WRITER_THREADS}.txt
72 |     export BENCHMARK_TSV=${LOG_NAME}.tsv
73 |  
74 |     rm -f $LOG_NAME
75 |     rm -f $BENCHMARK_TSV
76 | 
77 |     T="$(date +%s)"
78 | 
79 |     if [[ $SHARDED = "no" ]]; then
80 |         echo "executing standard sysbench" | tee -a $LOG_NAME
81 |         java -cp $CLASSPATH:$PWD/src jmongosysbenchexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $MAX_TPS $SEED "$TRUST_STORE" "$TRUST_STORE_PASSWORD" "$URI" | tee -a $LOG_NAME
82 |      else
83 |         echo "executing standard sysbench" | tee -a $LOG_NAME
84 |         java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $MAX_TPS $SEED "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $MAX_SHARD_KEY $DOCS_PER_SHARD "$URI" | tee -a $LOG_NAME
85 |     fi
86 | 
87 |     echo "" | tee -a $LOG_NAME
88 |     T="$(($(date +%s)-T))"
89 |     printf "`date` | sysbench benchmark duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME
90 |     echo ""
91 |     echo "************************************************************************"
92 |     echo "final ${numTailLines} interval(s)"
93 |     echo "************************************************************************"
94 |     tail -n $TAIL_LINES $LOG_NAME
95 | 
96 |     #cat $LOG_NAME | grep '170 seconds' >> results.txt
97 | fi
98 | 
99 | 


--------------------------------------------------------------------------------
/run.looped.bash:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # enable passing different config files
  4 | 
  5 | if [ ! $1 ];
  6 | then
  7 |     FILE="config.bash"
  8 | else
  9 |     FILE=$1
 10 | fi
 11 | 
 12 | if [ -f $FILE ];
 13 | then
 14 |    echo "Loading config from $FILE....."
 15 |    source $FILE
 16 | else
 17 |    echo "Unable to read config $FILE"
 18 |    exit 1
 19 | fi
 20 | 
 21 | CLASSPATH="mongo-java-driver-3.9.1.jar:mongodb-driver-core-3.9.1.jar:mongodb-driver-legacy-3.9.1.jar:$CLASSPATH"
 22 | TAIL_LINES=21
 23 | 
 24 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchload.java
 25 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchexecute.java
 26 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedload.java
 27 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedexecute.java
 28 | 
 29 | 
 30 | # load the data
 31 | 
 32 | if [[ $DOLOAD = "yes" ]]; then
 33 |     echo Do load at $( date )
 34 |     export LOG_NAME=mongoSysbenchLoad-${NUM_COLLECTIONS}-${NUM_DOCUMENTS_PER_COLLECTION}-${NUM_LOADER_THREADS}.txt
 35 |     export BENCHMARK_TSV=${LOG_NAME}.tsv
 36 |  
 37 |     rm -f $LOG_NAME
 38 |     rm -f $BENCHMARK_TSV
 39 | 
 40 |     T="$(date +%s)"
 41 | 
 42 |     if [[ $SHARDED = "no" ]]; then
 43 | 	echo "executing standard sysbench load (not sharded)" | tee -a $LOG_NAME
 44 |         java -cp $CLASSPATH:$PWD/src jmongosysbenchload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS 0 0
 45 |     else
 46 | 	echo "executing sharded sysbench load" | tee -a $LOG_NAME
 47 |         java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD
 48 |     fi
 49 | 
 50 |     echo "" | tee -a $LOG_NAME
 51 |     T="$(($(date +%s)-T))"
 52 |     printf "`date` | sysbench loader duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME
 53 |     echo ""
 54 |     echo "************************************************************************"
 55 |     echo "final ${numTailLines} interval(s)"
 56 |     echo "************************************************************************"
 57 |     tail -n $TAIL_LINES $LOG_NAME
 58 | fi
 59 | 
 60 | 
 61 | # execute the benchmark
 62 | 
 63 | if [[ $DOQUERY = "yes" ]]; then
 64 |     #threadList="1 1 1 1 2 2 2 2 4 4 4 4 8 8 8 8 16 16 16 16 32 32 32 32 64 64 64 64 96 96 96 96 128 128 128 128 160 160 160 160 192 192 192 192 224 224 224 224 256 256 256 256 512 512 512 512"
 65 |     #threadList="1 1 2 2 4 4 8 8 16 16 32 32 64 64"
 66 |     threadList="1 1 2 2 4 4 8 8 16 16 32 32 64 64 96 96 128 128 160 160 192 192 224 224 256 256 512 512 1024 2048"
 67 |     #threadList="1 2 4"
 68 | 
 69 |     logDir="./log"
 70 |     mkdir $logDir
 71 | 
 72 |     for numThreads in $threadList; do
 73 | 	NUM_WRITER_THREADS=$numThreads
 74 | 
 75 |         thisTimestamp=`date +%Y%m%d_%H%M%S`
 76 |         LOG_NAME="${logDir}/${thisTimestamp}_${NUM_WRITER_THREADS}_${MONGO_READ_PREFERENCE}_${RUN_TIME_SECONDS}.txt"
 77 |         export BENCHMARK_TSV=${LOG_NAME}.tsv
 78 | 
 79 |         echo "running with NUM_WRITER_THREADS=$NUM_WRITER_THREADS | MONGO_READ_PREFERENCE=$MONGO_READ_PREFERENCE | MONGO_SERVER=$MONGO_SERVER" | tee -a $thisLogName
 80 |  
 81 |         T="$(date +%s)"
 82 | 
 83 |         if [[ $SHARDED = "no" ]]; then
 84 | 	    echo "executing standard sysbench (not sharded)" | tee -a $LOG_NAME
 85 |             java -cp $CLASSPATH:$PWD/src jmongosysbenchexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS 0 0 | tee -a $LOG_NAME
 86 |         else
 87 | 	    echo "executing sharded sysbench" | tee -a $LOG_NAME
 88 |             java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD | tee -a $LOG_NAME
 89 |         fi
 90 | 
 91 |         echo "" | tee -a $LOG_NAME
 92 |         T="$(($(date +%s)-T))"
 93 |         printf "`date` | sysbench benchmark duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME
 94 |         echo ""
 95 |         echo "************************************************************************"
 96 |         echo "final ${numTailLines} interval(s)"
 97 |         echo "************************************************************************"
 98 |         tail -n $TAIL_LINES $LOG_NAME
 99 |     done
100 | fi
101 | 
102 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/src/jmongosysbenchload.java:
--------------------------------------------------------------------------------
  1 | //import com.mongodb.Mongo;
  2 | import java.io.BufferedWriter;
  3 | import java.io.File;
  4 | import java.io.FileWriter;
  5 | import java.io.IOException;
  6 | import java.io.Writer;
  7 | import java.util.Arrays;
  8 | import java.util.concurrent.atomic.AtomicLong;
  9 | 
 10 | import java.util.logging.Level;
 11 | import java.util.logging.Logger;
 12 | 
 13 | import com.mongodb.BasicDBObject;
 14 | import com.mongodb.CommandResult;
 15 | import com.mongodb.DB;
 16 | import com.mongodb.DBCollection;
 17 | import com.mongodb.DBObject;
 18 | import com.mongodb.MongoClient;
 19 | import com.mongodb.MongoClientOptions;
 20 | import com.mongodb.MongoClientURI;
 21 | import com.mongodb.MongoCredential;
 22 | import com.mongodb.ServerAddress;
 23 | import com.mongodb.WriteConcern;
 24 | 
 25 | public class jmongosysbenchload {
 26 |     public static AtomicLong globalInserts = new AtomicLong(0);
 27 |     public static AtomicLong globalNewBatchQty = new AtomicLong(0);
 28 |     public static AtomicLong globalNewBatchLatency = new AtomicLong(0);
 29 |     public static AtomicLong globalWriterThreads = new AtomicLong(0);
 30 | 
 31 |     public static Writer writer = null;
 32 |     public static boolean outputHeader = true;
 33 | 
 34 |     public static int numCollections;
 35 |     public static String dbName;
 36 |     public static int writerThreads;
 37 |     public static Integer numMaxInserts;
 38 |     public static int documentsPerInsert;
 39 |     public static long insertsPerFeedback;
 40 |     public static long secondsPerFeedback;
 41 |     public static String logFileName;
 42 |     public static String trustStore;
 43 |     public static String trustStorePassword;
 44 |     public static String connectionString;
 45 | 
 46 |     public static int allDone = 0;
 47 | 
 48 |     public jmongosysbenchload() {
 49 |     }
 50 | 
 51 |     public static void main (String[] args) throws Exception {
 52 |         if (args.length != 11) {
 53 |             logMe("*** ERROR : CONFIGURATION ISSUE ***");
 54 |             logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [trust store file] [trust store password] [connection string]");
 55 |             System.exit(1);
 56 |         }
 57 | 
 58 |         numCollections = Integer.valueOf(args[0]);
 59 |         dbName = args[1];
 60 |         writerThreads = Integer.valueOf(args[2]);
 61 |         numMaxInserts = Integer.valueOf(args[3]);
 62 |         documentsPerInsert = Integer.valueOf(args[4]);
 63 |         insertsPerFeedback = Long.valueOf(args[5]);
 64 |         secondsPerFeedback = Long.valueOf(args[6]);
 65 |         logFileName = args[7];
 66 |         trustStore = args[8];
 67 |         trustStorePassword = args[9];
 68 |         connectionString = args[10];
 69 | 
 70 |         logMe("Application Parameters");
 71 |         logMe("--------------------------------------------------");
 72 |         logMe("  %d collections",numCollections);
 73 |         logMe("  database name = %s",dbName);
 74 |         logMe("  %d writer thread(s)",writerThreads);
 75 |         logMe("  %,d documents per collection",numMaxInserts);
 76 |         logMe("  Documents Per Insert = %d",documentsPerInsert);
 77 |         logMe("  Feedback every %,d seconds(s)",secondsPerFeedback);
 78 |         logMe("  Feedback every %,d inserts(s)",insertsPerFeedback);
 79 |         logMe("  logging to file %s",logFileName);
 80 |         logMe("  connection string = %s",connectionString);
 81 | 
 82 |         //if (useSSL.equals("true")) {
 83 |         //    System.setProperty("javax.net.ssl.trustStore", trustStore);
 84 |         //    System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
 85 | 	//	}
 86 | 
 87 |         logMe("setting mongodb driver log level to WARNING");
 88 |         java.util.logging.Logger.getLogger("org.mongodb.driver").setLevel(Level.WARNING);
 89 | 
 90 |         MongoClient m = new MongoClient(new MongoClientURI(connectionString));
 91 | 
 92 |         DB db = m.getDB(dbName);
 93 | 
 94 |         // determine server type : mongo or tokumx
 95 |         //DBObject checkServerCmd = new BasicDBObject();
 96 |         //CommandResult commandResult = db.command("buildInfo");
 97 | 
 98 |         // check if tokumxVersion exists, otherwise assume mongo
 99 |         //if (commandResult.toString().contains("tokumxVersion")) {
100 |         //    indexTechnology = "tokumx";
101 |         //}
102 |         //else
103 |         //{
104 |         //    indexTechnology = "mongo";
105 |         //}
106 | 
107 |         //logMe("  index technology = %s",indexTechnology);
108 | 
109 |         logMe("--------------------------------------------------");
110 | 
111 |         try {
112 |             writer = new BufferedWriter(new FileWriter(new File(logFileName)));
113 |         } catch (IOException e) {
114 |             e.printStackTrace();
115 |         }
116 | 
117 |         jmongosysbenchload t = new jmongosysbenchload();
118 | 
119 |         Thread reporterThread = new Thread(t.new MyReporter());
120 |         reporterThread.start();
121 | 
122 |         Thread[] tWriterThreads = new Thread[writerThreads];
123 | 
124 |         for (int collectionNumber = 0; collectionNumber < numCollections; collectionNumber++) {
125 |             // if necessary, wait for an available slot for this loader
126 |             boolean waitingForSlot = true;
127 |             while (globalWriterThreads.get() >= writerThreads) {
128 |                 if (waitingForSlot) {
129 |                     logMe("  collection %d is waiting for an available loader slot",collectionNumber+1);
130 |                     waitingForSlot = false;
131 |                 }
132 |                 try {
133 |                     Thread.sleep(5000);
134 |                 } catch (Exception e) {
135 |                     e.printStackTrace();
136 |                 }
137 |             }
138 | 
139 |             // start the loader
140 |             for (int i=0; i<writerThreads; i++) {
141 |                 if ((tWriterThreads[i] == null) || (!tWriterThreads[i].isAlive())) {
142 |                     globalWriterThreads.incrementAndGet();
143 |                     tWriterThreads[i] = new Thread(t.new MyWriter(collectionNumber+1, writerThreads, i, numMaxInserts, db));
144 |                     tWriterThreads[i].start();
145 |                     break;
146 |                 }
147 |             }
148 | 
149 |         }
150 | 
151 |         // sleep a bit, then wait for all the writers to complete
152 |         try {
153 |             Thread.sleep(10000);
154 |         } catch (Exception e) {
155 |             e.printStackTrace();
156 |         }
157 |         while (globalWriterThreads.get() > 0) {
158 |             try {
159 |                 Thread.sleep(5000);
160 |             } catch (Exception e) {
161 |                 e.printStackTrace();
162 |             }
163 |         }
164 | 
165 |         // all the writers are finished
166 |         allDone = 1;
167 | 
168 |         if (reporterThread.isAlive())
169 |             reporterThread.join();
170 | 
171 |         try {
172 |             if (writer != null) {
173 |                 writer.close();
174 |             }
175 |         } catch (IOException e) {
176 |             e.printStackTrace();
177 |         }
178 | 
179 |         m.close();
180 | 
181 |         logMe("Done!");
182 |     }
183 | 
184 |     class MyWriter implements Runnable {
185 |         int collectionNumber;
186 |         int threadCount;
187 |         int threadNumber;
188 |         int numTables;
189 |         int numMaxInserts;
190 |         DB db;
191 | 
192 |         java.util.Random rand;
193 | 
194 |         MyWriter(int collectionNumber, int threadCount, int threadNumber, int numMaxInserts, DB db) {
195 |             this.collectionNumber = collectionNumber;
196 |             this.threadCount = threadCount;
197 |             this.threadNumber = threadNumber;
198 |             this.numMaxInserts = numMaxInserts;
199 |             this.db = db;
200 |             rand = new java.util.Random((long) collectionNumber);
201 |         }
202 |         public void run() {
203 |             String collectionName = "sbtest" + Integer.toString(collectionNumber);
204 | 
205 | 	    /*
206 |             if (indexTechnology.toLowerCase().equals("tokumx")) {
207 |                 DBObject cmd = new BasicDBObject();
208 |                 cmd.put("create", collectionName);
209 |                 cmd.put("compression", compressionType);
210 |                 cmd.put("readPageSize", basementSize);
211 |                 //cmd.put("basementSize", basementSize);
212 |                 CommandResult result = db.command(cmd);
213 |                 //logMe(result.toString());
214 |             } else if (indexTechnology.toLowerCase().equals("mongo")) {
215 |                 // nothing special to do for a regular mongo collection
216 | 
217 |             } else {
218 |                 // unknown index technology, abort
219 |                 logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology);
220 |                 System.exit(1);
221 |             }
222 | 	    */
223 | 
224 |             logMe("Writer thread %d : creating collection %s",threadNumber, collectionName);
225 | 
226 |             DBCollection coll = db.getCollection(collectionName);
227 | 
228 |             BasicDBObject idxOptions = new BasicDBObject();
229 |             idxOptions.put("background",false);
230 | 
231 |             //if (indexTechnology.toLowerCase().equals("tokumx")) {
232 |             //    idxOptions.put("compression",compressionType);
233 |             //    idxOptions.put("readPageSize",basementSize);
234 |             //}
235 | 
236 |             logMe("Writer thread %d : creating collection %s secondary index",threadNumber, collectionName);
237 | 
238 |             coll.createIndex(new BasicDBObject("k", 1), idxOptions);
239 | 
240 |             long numInserts = 0;
241 |             long id = 0;
242 | 
243 |             try {
244 |                 logMe("Writer thread %d : started to load collection %s",threadNumber, collectionName);
245 | 
246 |                 BasicDBObject[] aDocs = new BasicDBObject[documentsPerInsert];
247 | 
248 |                 int numRounds = numMaxInserts / documentsPerInsert;
249 | 
250 |                 for (int roundNum = 0; roundNum < numRounds; roundNum++) {
251 |                     for (int i = 0; i < documentsPerInsert; i++) {
252 |                         id++;
253 |                         BasicDBObject doc = new BasicDBObject();
254 |                         doc.put("_id",id);
255 |                         doc.put("k",rand.nextInt(numMaxInserts)+1);
256 |                         String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########");
257 |                         doc.put("c",cVal);
258 |                         String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########");
259 |                         doc.put("pad",padVal);
260 |                         aDocs[i]=doc;
261 |                     }
262 | 
263 | 		    long timeStart = System.currentTimeMillis();
264 |                     coll.insert(aDocs);
265 | 		    long timeEnd = System.currentTimeMillis();
266 |                     numInserts += documentsPerInsert;
267 |                     globalInserts.addAndGet(documentsPerInsert);
268 | 		    globalNewBatchQty.addAndGet(1);
269 | 		    globalNewBatchLatency.addAndGet(timeEnd-timeStart);
270 |                 }
271 | 
272 |             } catch (Exception e) {
273 |                 logMe("Writer thread %d : EXCEPTION",threadNumber);
274 |                 e.printStackTrace();
275 |             }
276 | 
277 |             globalWriterThreads.decrementAndGet();
278 |         }
279 |     }
280 | 
281 | 
282 |     public static String sysbenchString(java.util.Random rand, String thisMask) {
283 |         StringBuilder sb = new StringBuilder();
284 |         for (int i = 0, n = thisMask.length() ; i < n ; i++) { 
285 |             char c = thisMask.charAt(i); 
286 |             if (c == '#') {
287 |                 sb.append(String.valueOf(rand.nextInt(10)));
288 |             } else if (c == '@') {
289 |                 sb.append((char) (rand.nextInt(26) + 'a'));
290 |             } else {
291 |                 sb.append(c);
292 |             }
293 |         }
294 |         return sb.toString();
295 |     }
296 | 
297 | 
298 |     // reporting thread, outputs information to console and file
299 |     class MyReporter implements Runnable {
300 |         public void run()
301 |         {
302 |             long t0 = System.currentTimeMillis();
303 |             long lastInserts = 0;
304 |             long lastMs = t0;
305 |             long intervalNumber = 0;
306 |             long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
307 |             long nextFeedbackInserts = lastInserts + insertsPerFeedback;
308 |             long thisInserts = 0;
309 | 
310 |             while (allDone == 0)
311 |             {
312 |                 try {
313 |                     Thread.sleep(100);
314 |                 } catch (Exception e) {
315 |                     e.printStackTrace();
316 |                 }
317 | 
318 |                 long now = System.currentTimeMillis();
319 |                 thisInserts = globalInserts.get();
320 |                 if (((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) ||
321 |                     ((thisInserts >= nextFeedbackInserts) && (insertsPerFeedback > 0)))
322 |                 {
323 |                     intervalNumber++;
324 |                     nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
325 |                     nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback;
326 | 
327 | 		    long thisBatchQty = globalNewBatchQty.getAndSet(0);
328 | 		    long thisBatchLatency = globalNewBatchLatency.getAndSet(0);
329 | 		    double thisIntervalLatency = 0.0;
330 | 
331 | 		    if (thisBatchQty > 0)
332 | 		    {
333 |                         thisIntervalLatency = (double)thisBatchLatency / (double)thisBatchQty;
334 | 		    }	
335 | 
336 |                     long elapsed = now - t0;
337 |                     long thisIntervalMs = now - lastMs;
338 | 
339 |                     long thisIntervalInserts = thisInserts - lastInserts;
340 |                     double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0;
341 |                     double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0;
342 | 
343 |                     if (secondsPerFeedback > 0)
344 |                     {
345 |                         logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : int lat=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency);
346 |                     } else {
347 |                         logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : int lat=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency);
348 |                     }
349 | 
350 |                     try {
351 |                         if (outputHeader)
352 |                         {
353 |                             writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\tint_lat\n");
354 |                             outputHeader = false;
355 |                         }
356 | 
357 |                         String statusUpdate = "";
358 | 
359 |                         if (secondsPerFeedback > 0)
360 |                         {
361 |                             statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency);
362 |                         } else {
363 |                             statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency);
364 |                         }
365 |                         writer.write(statusUpdate);
366 |                         writer.flush();
367 |                     } catch (IOException e) {
368 |                         e.printStackTrace();
369 |                     }
370 | 
371 |                     lastInserts = thisInserts;
372 | 
373 |                     lastMs = now;
374 |                 }
375 |             }
376 | 
377 |             // output final numbers...
378 |             long now = System.currentTimeMillis();
379 |             thisInserts = globalInserts.get();
380 |             intervalNumber++;
381 |             nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
382 |             nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback;
383 |             long elapsed = now - t0;
384 |             long thisIntervalMs = now - lastMs;
385 |             long thisIntervalInserts = thisInserts - lastInserts;
386 |             double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0;
387 |             double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0;
388 |             if (secondsPerFeedback > 0)
389 |             {
390 |                 logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
391 |             } else {
392 |                 logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
393 |             }
394 |             try {
395 |                 if (outputHeader)
396 |                 {
397 |                     writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\n");
398 |                     outputHeader = false;
399 |                 }
400 |                 String statusUpdate = "";
401 |                 if (secondsPerFeedback > 0)
402 |                 {
403 |                     statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
404 |                 } else {
405 |                     statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
406 |                 }
407 |                 writer.write(statusUpdate);
408 |                 writer.flush();
409 |             } catch (IOException e) {
410 |                 e.printStackTrace();
411 |             }
412 | 
413 |         }
414 |     }
415 | 
416 | 
417 |     public static void logMe(String format, Object... args) {
418 |         System.out.println(Thread.currentThread() + String.format(format, args));
419 |     }
420 | }
421 | 


--------------------------------------------------------------------------------
/src/jmongosysbenchshardedload.java:
--------------------------------------------------------------------------------
  1 | //import com.mongodb.Mongo;
  2 | import java.io.BufferedWriter;
  3 | import java.io.File;
  4 | import java.io.FileWriter;
  5 | import java.io.IOException;
  6 | import java.io.Writer;
  7 | import java.util.Arrays;
  8 | import java.util.concurrent.atomic.AtomicLong;
  9 | 
 10 | import com.mongodb.BasicDBObject;
 11 | import com.mongodb.CommandResult;
 12 | import com.mongodb.DB;
 13 | import com.mongodb.DBCollection;
 14 | import com.mongodb.DBObject;
 15 | import com.mongodb.MongoClient;
 16 | import com.mongodb.MongoClientOptions;
 17 | import com.mongodb.MongoClientURI;
 18 | import com.mongodb.MongoCredential;
 19 | import com.mongodb.ServerAddress;
 20 | import com.mongodb.WriteConcern;
 21 | 
 22 | public class jmongosysbenchshardedload {
 23 |     public static AtomicLong globalInserts = new AtomicLong(0);
 24 |     public static AtomicLong globalWriterThreads = new AtomicLong(0);
 25 | 
 26 |     public static Writer writer = null;
 27 |     public static boolean outputHeader = true;
 28 | 
 29 |     public static int numCollections;
 30 |     public static String dbName;
 31 |     public static int writerThreads;
 32 |     public static Integer numMaxInserts;
 33 |     public static int documentsPerInsert;
 34 |     public static long insertsPerFeedback;
 35 |     public static long secondsPerFeedback;
 36 |     public static String compressionType;
 37 |     public static int basementSize;
 38 |     public static String logFileName;
 39 |     public static String indexTechnology;
 40 |     public static String myWriteConcern;
 41 |     public static String serverName;
 42 |     public static int serverPort;
 43 |     public static String userName;
 44 |     public static String passWord;
 45 |     public static String trustStore;
 46 |     public static String trustStorePassword;
 47 | 	public static String useSSL;
 48 |     public static int maxShardKey;
 49 |     public static int docsPerShard;
 50 | 
 51 |     public static int allDone = 0;
 52 | 
 53 |     public jmongosysbenchshardedload() {
 54 |     }
 55 | 
 56 |     public static void main (String[] args) throws Exception {
 57 |         if (args.length != 20) {
 58 |             logMe("*** ERROR : CONFIGURATION ISSUE ***");
 59 |             logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [compression type] [basement node size (bytes)]  [writeconcern] [server] [port] [username] [password] [trust store file] [trust store password] [use ssl] [max shard key] [docs per shard]");
 60 |             System.exit(1);
 61 |         }
 62 | 
 63 |         numCollections = Integer.valueOf(args[0]);
 64 |         dbName = args[1];
 65 |         writerThreads = Integer.valueOf(args[2]);
 66 |         numMaxInserts = Integer.valueOf(args[3]);
 67 |         documentsPerInsert = Integer.valueOf(args[4]);
 68 |         insertsPerFeedback = Long.valueOf(args[5]);
 69 |         secondsPerFeedback = Long.valueOf(args[6]);
 70 |         logFileName = args[7];
 71 |         compressionType = args[8];
 72 |         basementSize = Integer.valueOf(args[9]);
 73 |         myWriteConcern = args[10];
 74 |         serverName = args[11];
 75 |         serverPort = Integer.valueOf(args[12]);
 76 |         userName = args[13];
 77 |         passWord = args[14];
 78 |         trustStore = args[15];
 79 |         trustStorePassword = args[16];
 80 | 		useSSL = args[17].toLowerCase();
 81 |         maxShardKey = Integer.valueOf(args[18]);
 82 |         docsPerShard = Integer.valueOf(args[19]);
 83 | 
 84 | 	// override numMaxInserts
 85 | 	numMaxInserts = maxShardKey * docsPerShard;
 86 | 
 87 |         WriteConcern myWC = new WriteConcern();
 88 |         if (myWriteConcern.toLowerCase().equals("acknowledged")) {
 89 |             myWC = WriteConcern.ACKNOWLEDGED;
 90 |         }
 91 |         else if ((myWriteConcern.toLowerCase().equals("unacknowledged"))) {
 92 |             myWC = WriteConcern.UNACKNOWLEDGED;
 93 |         }
 94 |         else if ((myWriteConcern.toLowerCase().equals("w1"))) {
 95 |             myWC = WriteConcern.W1;
 96 |         }
 97 |         else if ((myWriteConcern.toLowerCase().equals("w2"))) {
 98 |             myWC = WriteConcern.W2;
 99 |         }
100 |         else if ((myWriteConcern.toLowerCase().equals("w3"))) {
101 |             myWC = WriteConcern.W3;
102 |         }
103 |         else {
104 |             logMe("*** ERROR : WRITE CONCERN ISSUE ***");
105 |             logMe("  write concern %s is not supported",myWriteConcern);
106 |             System.exit(1);
107 |         }
108 | 
109 |         logMe("Application Parameters");
110 |         logMe("--------------------------------------------------");
111 |         logMe("  %d collections",numCollections);
112 |         logMe("  database name = %s",dbName);
113 |         logMe("  %d writer thread(s)",writerThreads);
114 |         logMe("  %,d documents per collection",numMaxInserts);
115 |         logMe("  Documents Per Insert = %d",documentsPerInsert);
116 |         logMe("  Feedback every %,d seconds(s)",secondsPerFeedback);
117 |         logMe("  Feedback every %,d inserts(s)",insertsPerFeedback);
118 |         logMe("  logging to file %s",logFileName);
119 |         logMe("  write concern = %s",myWriteConcern);
120 |         logMe("  Server:Port = %s:%d",serverName,serverPort);
121 |         logMe("  Username = %s",userName);
122 | 		logMe("  Use SSL = %s",useSSL);
123 |         logMe("  Maximum Shard Key = %s",maxShardKey);
124 |         logMe("  Documents Per Shard = %s",docsPerShard);
125 | 
126 | 		/*
127 |         MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build();
128 |         ServerAddress srvrAdd = new ServerAddress(serverName,serverPort);
129 | 
130 |         // Credential login is optional.
131 |         MongoClient m;
132 |         if (userName.isEmpty() || userName.equalsIgnoreCase("none")) {
133 |             m = new MongoClient(srvrAdd);
134 |         } else {
135 |             MongoCredential credential = MongoCredential.createCredential(userName, dbName, passWord.toCharArray());
136 |             m = new MongoClient(srvrAdd, Arrays.asList(credential));
137 |         }
138 | 		*/
139 | 
140 |         //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&replicaSet=rs0&readpreference=%s";
141 | 	String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=2000";
142 |         //String readPreference = "secondaryPreferred";
143 |         String readPreference = "primary";
144 |         String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference);
145 |         logMe("  connection string = %s",connectionString);
146 | 
147 |         //if (useSSL.equals("true")) {
148 |         //    System.setProperty("javax.net.ssl.trustStore", trustStore);
149 |         //    System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
150 | 	//	}
151 | 
152 |         MongoClient m = new MongoClient(new MongoClientURI(connectionString));
153 | 
154 |         //logMe("mongoOptions | " + m.getMongoOptions().toString());
155 |         //logMe("mongoWriteConcern | " + m.getWriteConcern().toString());
156 | 
157 |         DB db = m.getDB(dbName);
158 | 
159 |         // determine server type : mongo or tokumx
160 |         DBObject checkServerCmd = new BasicDBObject();
161 |         CommandResult commandResult = db.command("buildInfo");
162 | 
163 |         // check if tokumxVersion exists, otherwise assume mongo
164 |         if (commandResult.toString().contains("tokumxVersion")) {
165 |             indexTechnology = "tokumx";
166 |         }
167 |         else
168 |         {
169 |             indexTechnology = "mongo";
170 |         }
171 | 
172 |         logMe("  index technology = %s",indexTechnology);
173 | 
174 |         if (indexTechnology.toLowerCase().equals("tokumx")) {
175 |             logMe("  + compression type = %s",compressionType);
176 |             logMe("  + basement node size (bytes) = %d",basementSize);
177 |         }
178 | 
179 |         logMe("--------------------------------------------------");
180 | 
181 |         try {
182 |             writer = new BufferedWriter(new FileWriter(new File(logFileName)));
183 |         } catch (IOException e) {
184 |             e.printStackTrace();
185 |         }
186 | 
187 |         if ((!indexTechnology.toLowerCase().equals("tokumx")) && (!indexTechnology.toLowerCase().equals("mongo"))) {
188 |             // unknown index technology, abort
189 |             logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology);
190 |             System.exit(1);
191 |         }
192 | 
193 |         jmongosysbenchshardedload t = new jmongosysbenchshardedload();
194 | 
195 |         Thread reporterThread = new Thread(t.new MyReporter());
196 |         reporterThread.start();
197 | 
198 |         Thread[] tWriterThreads = new Thread[writerThreads];
199 | 
200 |         for (int collectionNumber = 0; collectionNumber < numCollections; collectionNumber++) {
201 |             // if necessary, wait for an available slot for this loader
202 |             boolean waitingForSlot = true;
203 |             while (globalWriterThreads.get() >= writerThreads) {
204 |                 if (waitingForSlot) {
205 |                     logMe("  collection %d is waiting for an available loader slot",collectionNumber+1);
206 |                     waitingForSlot = false;
207 |                 }
208 |                 try {
209 |                     Thread.sleep(5000);
210 |                 } catch (Exception e) {
211 |                     e.printStackTrace();
212 |                 }
213 |             }
214 | 
215 |             // start the loader
216 |             for (int i=0; i<writerThreads; i++) {
217 |                 if ((tWriterThreads[i] == null) || (!tWriterThreads[i].isAlive())) {
218 |                     globalWriterThreads.incrementAndGet();
219 |                     tWriterThreads[i] = new Thread(t.new MyWriter(collectionNumber+1, writerThreads, i, numMaxInserts, db));
220 |                     tWriterThreads[i].start();
221 |                     break;
222 |                 }
223 |             }
224 | 
225 |         }
226 | 
227 |         // sleep a bit, then wait for all the writers to complete
228 |         try {
229 |             Thread.sleep(10000);
230 |         } catch (Exception e) {
231 |             e.printStackTrace();
232 |         }
233 |         while (globalWriterThreads.get() > 0) {
234 |             try {
235 |                 Thread.sleep(5000);
236 |             } catch (Exception e) {
237 |                 e.printStackTrace();
238 |             }
239 |         }
240 | 
241 |         // wait for writer threads to terminate
242 |         //for (int i=0; i<writerThreads; i++) {
243 |         //    if (tWriterThreads[i].isAlive())
244 |         //        tWriterThreads[i].join();
245 |         //}
246 | 
247 |         // all the writers are finished
248 |         allDone = 1;
249 | 
250 |         if (reporterThread.isAlive())
251 |             reporterThread.join();
252 | 
253 |         try {
254 |             if (writer != null) {
255 |                 writer.close();
256 |             }
257 |         } catch (IOException e) {
258 |             e.printStackTrace();
259 |         }
260 | 
261 |         // m.dropDatabase("mydb");
262 | 
263 |         m.close();
264 | 
265 |         logMe("Done!");
266 |     }
267 | 
268 |     class MyWriter implements Runnable {
269 |         int collectionNumber;
270 |         int threadCount;
271 |         int threadNumber;
272 |         int numTables;
273 |         int numMaxInserts;
274 |         DB db;
275 | 
276 |         java.util.Random rand;
277 | 
278 |         MyWriter(int collectionNumber, int threadCount, int threadNumber, int numMaxInserts, DB db) {
279 |             this.collectionNumber = collectionNumber;
280 |             this.threadCount = threadCount;
281 |             this.threadNumber = threadNumber;
282 |             this.numMaxInserts = numMaxInserts;
283 |             this.db = db;
284 |             rand = new java.util.Random((long) collectionNumber);
285 |         }
286 |         public void run() {
287 |             String collectionName = "sbtest" + Integer.toString(collectionNumber);
288 | 
289 |             if (indexTechnology.toLowerCase().equals("tokumx")) {
290 |                 DBObject cmd = new BasicDBObject();
291 |                 cmd.put("create", collectionName);
292 |                 cmd.put("compression", compressionType);
293 |                 cmd.put("readPageSize", basementSize);
294 |                 //cmd.put("basementSize", basementSize);
295 |                 CommandResult result = db.command(cmd);
296 |                 //logMe(result.toString());
297 |             } else if (indexTechnology.toLowerCase().equals("mongo")) {
298 |                 // nothing special to do for a regular mongo collection
299 | 
300 |             } else {
301 |                 // unknown index technology, abort
302 |                 logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology);
303 |                 System.exit(1);
304 |             }
305 | 
306 |             logMe("Writer thread %d : creating collection %s",threadNumber, collectionName);
307 | 
308 |             DBCollection coll = db.getCollection(collectionName);
309 | 
310 |             BasicDBObject idxOptions = new BasicDBObject();
311 |             idxOptions.put("background",false);
312 | 
313 |             if (indexTechnology.toLowerCase().equals("tokumx")) {
314 |                 idxOptions.put("compression",compressionType);
315 |                 idxOptions.put("readPageSize",basementSize);
316 |             }
317 | 
318 |             logMe("Writer thread %d : creating collection %s secondary index",threadNumber, collectionName);
319 | 
320 |             coll.createIndex(new BasicDBObject("k", 1), idxOptions);
321 | 
322 |             long numInserts = 0;
323 |             long id = 0;
324 | 
325 |             try {
326 |                 logMe("Writer thread %d : started to load collection %s",threadNumber, collectionName);
327 | 
328 |                 BasicDBObject[] aDocs = new BasicDBObject[docsPerShard];
329 | 
330 |                 int numRounds = maxShardKey;
331 | 
332 |                 for (int roundNum = 0; roundNum < numRounds; roundNum++) {
333 |                     for (int i = 1; i <= docsPerShard; i++) {
334 |                         id++;
335 |                         BasicDBObject doc = new BasicDBObject();
336 |                         doc.put("_id",id);
337 |                         doc.put("shardKey",roundNum);
338 |                         doc.put("orderId",i);
339 |                         doc.put("k",rand.nextInt(numMaxInserts)+1);
340 |                         String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########");
341 |                         doc.put("c",cVal);
342 |                         String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########");
343 |                         doc.put("pad",padVal);
344 |                         aDocs[i-1]=doc;
345 |                     }
346 | 
347 |                     coll.insert(aDocs);
348 |                     //numInserts += documentsPerInsert;
349 |                     //globalInserts.addAndGet(documentsPerInsert);
350 |                     numInserts += docsPerShard;
351 |                     globalInserts.addAndGet(docsPerShard);
352 |                 }
353 | 
354 |             } catch (Exception e) {
355 |                 logMe("Writer thread %d : EXCEPTION",threadNumber);
356 |                 e.printStackTrace();
357 |             }
358 | 
359 |             globalWriterThreads.decrementAndGet();
360 |         }
361 |     }
362 | 
363 | 
364 |     public static String sysbenchString(java.util.Random rand, String thisMask) {
365 |         StringBuilder sb = new StringBuilder();
366 |         for (int i = 0, n = thisMask.length() ; i < n ; i++) { 
367 |             char c = thisMask.charAt(i); 
368 |             if (c == '#') {
369 |                 sb.append(String.valueOf(rand.nextInt(10)));
370 |             } else if (c == '@') {
371 |                 sb.append((char) (rand.nextInt(26) + 'a'));
372 |             } else {
373 |                 sb.append(c);
374 |             }
375 |         }
376 |         return sb.toString();
377 |     }
378 | 
379 | 
380 |     // reporting thread, outputs information to console and file
381 |     class MyReporter implements Runnable {
382 |         public void run()
383 |         {
384 |             long t0 = System.currentTimeMillis();
385 |             long lastInserts = 0;
386 |             long lastMs = t0;
387 |             long intervalNumber = 0;
388 |             long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
389 |             long nextFeedbackInserts = lastInserts + insertsPerFeedback;
390 |             long thisInserts = 0;
391 | 
392 |             while (allDone == 0)
393 |             {
394 |                 try {
395 |                     Thread.sleep(100);
396 |                 } catch (Exception e) {
397 |                     e.printStackTrace();
398 |                 }
399 | 
400 |                 long now = System.currentTimeMillis();
401 |                 thisInserts = globalInserts.get();
402 |                 if (((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) ||
403 |                     ((thisInserts >= nextFeedbackInserts) && (insertsPerFeedback > 0)))
404 |                 {
405 |                     intervalNumber++;
406 |                     nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
407 |                     nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback;
408 | 
409 |                     long elapsed = now - t0;
410 |                     long thisIntervalMs = now - lastMs;
411 | 
412 |                     long thisIntervalInserts = thisInserts - lastInserts;
413 |                     double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0;
414 |                     double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0;
415 | 
416 |                     if (secondsPerFeedback > 0)
417 |                     {
418 |                         logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
419 |                     } else {
420 |                         logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
421 |                     }
422 | 
423 |                     try {
424 |                         if (outputHeader)
425 |                         {
426 |                             writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\n");
427 |                             outputHeader = false;
428 |                         }
429 | 
430 |                         String statusUpdate = "";
431 | 
432 |                         if (secondsPerFeedback > 0)
433 |                         {
434 |                             statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
435 |                         } else {
436 |                             statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
437 |                         }
438 |                         writer.write(statusUpdate);
439 |                         writer.flush();
440 |                     } catch (IOException e) {
441 |                         e.printStackTrace();
442 |                     }
443 | 
444 |                     lastInserts = thisInserts;
445 | 
446 |                     lastMs = now;
447 |                 }
448 |             }
449 | 
450 |             // output final numbers...
451 |             long now = System.currentTimeMillis();
452 |             thisInserts = globalInserts.get();
453 |             intervalNumber++;
454 |             nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
455 |             nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback;
456 |             long elapsed = now - t0;
457 |             long thisIntervalMs = now - lastMs;
458 |             long thisIntervalInserts = thisInserts - lastInserts;
459 |             double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0;
460 |             double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0;
461 |             if (secondsPerFeedback > 0)
462 |             {
463 |                 logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
464 |             } else {
465 |                 logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
466 |             }
467 |             try {
468 |                 if (outputHeader)
469 |                 {
470 |                     writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\n");
471 |                     outputHeader = false;
472 |                 }
473 |                 String statusUpdate = "";
474 |                 if (secondsPerFeedback > 0)
475 |                 {
476 |                     statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
477 |                 } else {
478 |                     statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond);
479 |                 }
480 |                 writer.write(statusUpdate);
481 |                 writer.flush();
482 |             } catch (IOException e) {
483 |                 e.printStackTrace();
484 |             }
485 | 
486 |         }
487 |     }
488 | 
489 | 
490 |     public static void logMe(String format, Object... args) {
491 |         System.out.println(Thread.currentThread() + String.format(format, args));
492 |     }
493 | }
494 | 


--------------------------------------------------------------------------------
/src/jmongosysbenchexecute.java:
--------------------------------------------------------------------------------
  1 | //import com.mongodb.Mongo;
  2 | import java.io.BufferedWriter;
  3 | import java.io.File;
  4 | import java.io.FileWriter;
  5 | import java.io.IOException;
  6 | import java.io.Writer;
  7 | import java.util.Arrays;
  8 | import java.util.List;
  9 | import java.util.concurrent.atomic.AtomicLong;
 10 | 
 11 | import java.util.logging.Level;
 12 | import java.util.logging.Logger;
 13 | 
 14 | import com.mongodb.AggregationOutput;
 15 | import com.mongodb.BasicDBObject;
 16 | import com.mongodb.CommandResult;
 17 | import com.mongodb.DB;
 18 | import com.mongodb.DBCollection;
 19 | import com.mongodb.DBCursor;
 20 | import com.mongodb.DBObject;
 21 | import com.mongodb.MongoClient;
 22 | import com.mongodb.MongoClientOptions;
 23 | import com.mongodb.MongoClientURI;
 24 | import com.mongodb.MongoCredential;
 25 | import com.mongodb.ServerAddress;
 26 | import com.mongodb.WriteConcern;
 27 | import com.mongodb.WriteResult;
 28 | 
 29 | public class jmongosysbenchexecute {
 30 |     public static AtomicLong globalInserts = new AtomicLong(0);
 31 |     public static AtomicLong globalDeletes = new AtomicLong(0);
 32 |     public static AtomicLong globalUpdates = new AtomicLong(0);
 33 |     public static AtomicLong globalPointQueries = new AtomicLong(0);
 34 |     public static AtomicLong globalRangeQueries = new AtomicLong(0);
 35 |     public static AtomicLong globalSysbenchTransactions = new AtomicLong(0);
 36 |     public static AtomicLong globalWriterThreads = new AtomicLong(0);
 37 |     public static AtomicLong globalExceptions = new AtomicLong(0);
 38 |     public static AtomicLong globalNewBatchQty = new AtomicLong(0);
 39 |     public static AtomicLong globalNewBatchLatency = new AtomicLong(0);
 40 | 
 41 |     public static Writer writer = null;
 42 |     public static boolean outputHeader = true;
 43 | 
 44 |     public static int numCollections;
 45 |     public static String dbName;
 46 |     public static int writerThreads;
 47 |     public static Integer numMaxInserts;
 48 |     public static long secondsPerFeedback;
 49 |     public static String logFileName;
 50 |     //public static String autoCommit;
 51 |     public static int runSeconds;
 52 |     public static Integer maxTPS;
 53 |     public static Integer maxThreadTPS;
 54 |     public static String trustStore;
 55 |     public static String trustStorePassword;
 56 |     public static String connectionString;
 57 | 
 58 |     public static int oltpRangeSize;
 59 |     public static int oltpPointSelects;
 60 |     public static int oltpSimpleRanges;
 61 |     public static int oltpSumRanges;
 62 |     public static int oltpOrderRanges;
 63 |     public static int oltpDistinctRanges;
 64 |     public static int oltpIndexUpdates;
 65 |     public static int oltpNonIndexUpdates;
 66 |     public static int oltpInserts;
 67 | 
 68 |     public static int allDone = 0;
 69 |     public static long rngSeed = 0;
 70 |     
 71 |     public jmongosysbenchexecute() {
 72 |     }
 73 | 
 74 |     public static void main (String[] args) throws Exception {
 75 |         if (args.length != 21) {
 76 |             logMe("*** ERROR : CONFIGURATION ISSUE ***");
 77 |             logMe("jsysbenchexecute [number of collections] [database name] [number of writer threads] [documents per collection] [seconds feedback] "+
 78 |                                    "[log file name] [runtime (seconds)] [range size] [point selects] "+
 79 |                                    "[simple ranges] [sum ranges] [order ranges] [distinct ranges] [index updates] [non index updates] [inserts] "+
 80 |                                    "[max tps] [seed] [trust store] [trust store password] [connection string]");
 81 |             System.exit(1);
 82 |         }
 83 |         
 84 |         numCollections = Integer.valueOf(args[0]);
 85 |         dbName = args[1];
 86 |         writerThreads = Integer.valueOf(args[2]);
 87 |         numMaxInserts = Integer.valueOf(args[3]);
 88 |         secondsPerFeedback = Long.valueOf(args[4]);
 89 |         logFileName = args[5];
 90 |         runSeconds = Integer.valueOf(args[6]);
 91 |         oltpRangeSize = Integer.valueOf(args[7]);
 92 |         oltpPointSelects = Integer.valueOf(args[8]);
 93 |         oltpSimpleRanges = Integer.valueOf(args[9]);
 94 |         oltpSumRanges = Integer.valueOf(args[10]);
 95 |         oltpOrderRanges = Integer.valueOf(args[11]);
 96 |         oltpDistinctRanges = Integer.valueOf(args[12]);
 97 |         oltpIndexUpdates = Integer.valueOf(args[13]);
 98 |         oltpNonIndexUpdates = Integer.valueOf(args[14]);
 99 |         oltpInserts = Integer.valueOf(args[15]);
100 |         maxTPS = Integer.valueOf(args[16]);
101 |         rngSeed = Long.valueOf(args[17]);
102 | 	trustStore = args[18];
103 | 	trustStorePassword = args[19];
104 | 	connectionString = args[20];
105 | 
106 |         maxThreadTPS = (maxTPS / writerThreads);
107 | 
108 |         logMe("Application Parameters");
109 |         logMe("-------------------------------------------------------------------------------------------------");
110 |         logMe("  collections              = %d",numCollections);
111 |         logMe("  database name            = %s",dbName);
112 |         logMe("  writer threads           = %d",writerThreads);
113 |         logMe("  documents per collection = %,d",numMaxInserts);
114 |         logMe("  feedback seconds         = %,d",secondsPerFeedback);
115 |         logMe("  log file                 = %s",logFileName);
116 |         logMe("  run seconds              = %d",runSeconds);
117 |         logMe("  oltp range size          = %d",oltpRangeSize);
118 |         logMe("  oltp point selects       = %d",oltpPointSelects);
119 |         logMe("  oltp simple ranges       = %d",oltpSimpleRanges);
120 |         logMe("  oltp sum ranges          = %d",oltpSumRanges);
121 |         logMe("  oltp order ranges        = %d",oltpOrderRanges);
122 |         logMe("  oltp distinct ranges     = %d",oltpDistinctRanges);
123 |         logMe("  oltp index updates       = %d",oltpIndexUpdates);
124 |         logMe("  oltp non index updates   = %d",oltpNonIndexUpdates);
125 |         logMe("  oltp inserts             = %d",oltpInserts);
126 |         logMe("  maximum tps (global)     = %d",maxTPS);
127 |         logMe("  maximum tps (per thread) = %d",maxThreadTPS);
128 |         logMe("  seed                     = %d",rngSeed);
129 |         logMe("  uri                      = %s",connectionString);
130 |         logMe("-------------------------------------------------------------------------------------------------");
131 | 
132 |         logMe("setting mongodb driver log level to WARNING");
133 | 	java.util.logging.Logger.getLogger("org.mongodb.driver").setLevel(Level.WARNING);
134 | 
135 |         MongoClient m = new MongoClient(new MongoClientURI(connectionString));
136 | 
137 |         DB db = m.getDB(dbName);
138 | 
139 | 	/*
140 |         // determine server type : mongo or tokumx
141 |         DBObject checkServerCmd = new BasicDBObject();
142 |         CommandResult commandResult = db.command("buildInfo");
143 | 
144 |         // check if tokumxVersion exists, otherwise assume mongo
145 |         if (commandResult.toString().contains("tokumxVersion")) {
146 |             indexTechnology = "tokumx";
147 |         }
148 |         else
149 |         {
150 |             indexTechnology = "mongo";
151 |         }
152 | 	*/
153 | 
154 |         try {
155 |             writer = new BufferedWriter(new FileWriter(new File(logFileName)));
156 |         } catch (IOException e) {
157 |             e.printStackTrace();
158 |         }
159 | 
160 |         jmongosysbenchexecute t = new jmongosysbenchexecute();
161 | 
162 |         Thread[] tWriterThreads = new Thread[writerThreads];
163 | 
164 |         for (int i=0; i<writerThreads; i++) {
165 |             tWriterThreads[i] = new Thread(t.new MyWriter(writerThreads, i, numMaxInserts, db, numCollections, rngSeed));
166 |             tWriterThreads[i].start();
167 |         }
168 | 
169 |         Thread reporterThread = new Thread(t.new MyReporter());
170 |         reporterThread.start();
171 |         reporterThread.join();
172 | 
173 |         // wait for writer threads to terminate
174 |         for (int i=0; i<writerThreads; i++) {
175 |             if (tWriterThreads[i].isAlive())
176 |                 tWriterThreads[i].join();
177 |         }
178 | 
179 |         try {
180 |             if (writer != null) {
181 |                 writer.close();
182 |             }
183 |         } catch (IOException e) {
184 |             e.printStackTrace();
185 |         }
186 | 
187 |         m.close();
188 | 
189 |         logMe("Done!");
190 |     }
191 | 
192 |     class MyWriter implements Runnable {
193 |         int threadCount;
194 |         int threadNumber;
195 |         int numTables;
196 |         int numMaxInserts;
197 |         int numCollections;
198 |         DB db;
199 | 
200 |         long numInserts = 0;
201 |         long numDeletes = 0;
202 |         long numUpdates = 0;
203 |         long numPointQueries = 0;
204 |         long numRangeQueries = 0;
205 | 
206 |         java.util.Random rand;
207 |         
208 |         MyWriter(int threadCount, int threadNumber, int numMaxInserts, DB db, int numCollections, long rngSeed) {
209 |             this.threadCount = threadCount;
210 |             this.threadNumber = threadNumber;
211 |             this.numMaxInserts = numMaxInserts;
212 |             this.db = db;
213 |             this.numCollections = numCollections;
214 |             rand = new java.util.Random((long) threadNumber + rngSeed);
215 |         }
216 |         public void run() {
217 |             logMe("Writer thread %d : started",threadNumber);
218 |             globalWriterThreads.incrementAndGet();
219 | 
220 |             long numTransactions = 0;
221 |             long numLastTransactions = 0;
222 |             long nextMs = System.currentTimeMillis() + 1000;
223 | 
224 |             while (allDone == 0) {
225 |                 if ((numTransactions - numLastTransactions) >= maxThreadTPS) {
226 |                     // pause until a second has passed
227 |                     while (System.currentTimeMillis() < nextMs) {
228 |                         try {
229 |                             Thread.sleep(20);
230 |                         } catch (Exception e) {
231 |                             e.printStackTrace();
232 |                         }
233 |                     }
234 |                     numLastTransactions = numTransactions;
235 |                     nextMs = System.currentTimeMillis() + 1000;
236 |                 }
237 | 
238 |                 String collectionName = "sbtest" + Integer.toString(rand.nextInt(numCollections)+1);
239 |                 DBCollection coll = db.getCollection(collectionName);
240 | 
241 | 		long timeStart = System.currentTimeMillis();
242 | 
243 |                 try {
244 |                     for (int i=1; i <= oltpPointSelects; i++) {
245 |                         //for i=1, oltp_point_selects do
246 |                         //   rs = db_query("SELECT c FROM ".. table_name .." WHERE id=" .. sb_rand(1, oltp_table_size))
247 |                         //end
248 | 
249 |                         // db.sbtest8.find({_id: 554312}, {c: 1, _id: 0})
250 | 
251 |                         int startId = rand.nextInt(numMaxInserts)+1;
252 | 
253 |                         BasicDBObject query = new BasicDBObject("_id", startId);
254 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
255 | 
256 |                         DBObject myDoc = coll.findOne(query, columns);
257 |                         //System.out.println(myDoc);
258 | 
259 |                         globalPointQueries.incrementAndGet();
260 |                     }
261 | 
262 |                     for (int i=1; i <= oltpSimpleRanges; i++) {
263 |                         //for i=1, oltp_simple_ranges do
264 |                         //   range_start = sb_rand(1, oltp_table_size)
265 |                         //   rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1)
266 |                         //end
267 | 
268 |                         //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0})
269 | 
270 |                         int startId = rand.nextInt(numMaxInserts)+1;
271 |                         int endId = startId + oltpRangeSize - 1;
272 | 
273 |                         BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId));
274 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
275 |                         DBCursor cursor = coll.find(query, columns);
276 |                         try {
277 |                             while(cursor.hasNext()) {
278 |                                 cursor.next();
279 |                                 //System.out.println(cursor.next());
280 |                             }
281 |                         } finally {
282 |                             cursor.close();
283 |                         }
284 | 
285 |                         globalRangeQueries.incrementAndGet();
286 |                     }
287 | 
288 |                     for (int i=1; i <= oltpSumRanges; i++) {
289 |                         //for i=1, oltp_sum_ranges do
290 |                         //   range_start = sb_rand(1, oltp_table_size)
291 |                         //   rs = db_query("SELECT SUM(K) FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1)
292 |                         //end
293 | 
294 |                         //db.sbtest8.aggregate([ {$match: {_id: {$gt: 5523412, $lt: 5523512}}}, { $group: { _id: null, total: { $sum: "$k"}} } ])
295 | 
296 |                         int startId = rand.nextInt(numMaxInserts)+1;
297 |                         int endId = startId + oltpRangeSize - 1;
298 | 
299 |                         // create our pipeline operations, first with the $match
300 |                         DBObject match = new BasicDBObject("$match", new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId)));
301 | 
302 |                         // build the $projection operation
303 |                         DBObject fields = new BasicDBObject("k", 1);
304 |                         fields.put("_id", 0);
305 |                         DBObject project = new BasicDBObject("$project", fields );
306 | 
307 |                         // Now the $group operation
308 |                         DBObject groupFields = new BasicDBObject( "_id", null);
309 |                         groupFields.put("average", new BasicDBObject( "$sum", "$k"));
310 |                         DBObject group = new BasicDBObject("$group", groupFields);
311 | 
312 |                         // run aggregation
313 |                         AggregationOutput output = coll.aggregate( match, project, group );
314 | 
315 |                         //System.out.println(output.getCommandResult());
316 | 
317 |                         globalRangeQueries.incrementAndGet();
318 |                     }
319 | 
320 |                     for (int i=1; i <= oltpOrderRanges; i++) {
321 |                         //for i=1, oltp_order_ranges do
322 |                         //   range_start = sb_rand(1, oltp_table_size)
323 |                         //   rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c")
324 |                         //end
325 | 
326 |                         //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0}).sort({c: 1})
327 | 
328 |                         int startId = rand.nextInt(numMaxInserts)+1;
329 |                         int endId = startId + oltpRangeSize - 1;
330 | 
331 |                         BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId));
332 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
333 |                         DBCursor cursor = coll.find(query, columns).sort(new BasicDBObject("c",1));
334 |                         try {
335 |                             while(cursor.hasNext()) {
336 |                                 cursor.next();
337 |                                 //System.out.println(cursor.next());
338 |                             }
339 |                         } finally {
340 |                             cursor.close();
341 |                         }
342 | 
343 |                         globalRangeQueries.incrementAndGet();
344 |                     }
345 | 
346 |                     for (int i=1; i <= oltpDistinctRanges; i++) {
347 |                         //for i=1, oltp_distinct_ranges do
348 |                         //   range_start = sb_rand(1, oltp_table_size)
349 |                         //   rs = db_query("SELECT DISTINCT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c")
350 |                         //end
351 | 
352 |                         //db.sbtest8.distinct("c",{_id: {$gt: 5523412, $lt: 5523512}}).sort()
353 | 
354 |                         int startId = rand.nextInt(numMaxInserts)+1;
355 |                         int endId = startId + oltpRangeSize - 1;
356 | 
357 |                         BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId));
358 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
359 |                         List lstDistinct = coll.distinct("c", query);
360 |                         //System.out.println(lstDistinct.toString());
361 | 
362 |                         globalRangeQueries.incrementAndGet();
363 |                     }
364 | 
365 |                     for (int i=1; i <= oltpIndexUpdates; i++) {
366 |                         //for i=1, oltp_index_updates do
367 |                         //   rs = db_query("UPDATE " .. table_name .. " SET k=k+1 WHERE id=" .. sb_rand(1, oltp_table_size))
368 |                         //end
369 | 
370 |                         //db.sbtest8.update({_id: 5523412}, {$inc: {k: 1}}, false, false)
371 | 
372 |                         int startId = rand.nextInt(numMaxInserts)+1;
373 | 
374 |                         WriteResult wrUpdate = coll.update(new BasicDBObject("_id", startId), new BasicDBObject("$inc", new BasicDBObject("k",1)), false, false);
375 |     
376 |                         //System.out.println(wrUpdate.toString());
377 |                     }
378 |     
379 |                     for (int i=1; i <= oltpNonIndexUpdates; i++) {
380 |                         //for i=1, oltp_non_index_updates do
381 |                         //   c_val = sb_rand_str("###########-###########-###########-###########-###########-###########-###########-###########-###########-###########")
382 |                         //   query = "UPDATE " .. table_name .. " SET c='" .. c_val .. "' WHERE id=" .. sb_rand(1, oltp_table_size)
383 |                         //   rs = db_query(query)
384 |                         //   if rs then
385 |                         //     print(query)
386 |                         //   end
387 |                         //end
388 | 
389 |                         //db.sbtest8.update({_id: 5523412}, {$set: {c: "hello there"}}, false, false)
390 | 
391 |                         int startId = rand.nextInt(numMaxInserts)+1;
392 | 
393 |                         String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########");
394 | 
395 |                         WriteResult wrUpdate = coll.update(new BasicDBObject("_id", startId), new BasicDBObject("$set", new BasicDBObject("c",cVal)), false, false);
396 | 
397 |                         //System.out.println(wrUpdate.toString());
398 |                     }
399 | 
400 |                     for (int i=1; i <= oltpInserts; i++) {
401 |                         //i = sb_rand(1, oltp_table_size)
402 |                         //rs = db_query("DELETE FROM " .. table_name .. " WHERE id=" .. i)
403 |                       
404 |                         //db.sbtest8.remove({_id: 5523412})
405 | 
406 |                         int startId = rand.nextInt(numMaxInserts)+1;
407 | 
408 |                         WriteResult wrRemove = coll.remove(new BasicDBObject("_id", startId));
409 | 
410 |                         //c_val = sb_rand_str([[###########-###########-###########-###########-###########-###########-###########-###########-###########-###########]])
411 |                         //pad_val = sb_rand_str([[###########-###########-###########-###########-###########]])
412 |                         //rs = db_query("INSERT INTO " .. table_name ..  " (id, k, c, pad) VALUES " .. string.format("(%d, %d, '%s', '%s')",i, sb_rand(1, oltp_table_size) , c_val, pad_val))
413 | 
414 |                         BasicDBObject doc = new BasicDBObject();
415 |                         doc.put("_id",startId);
416 |                         doc.put("k",rand.nextInt(numMaxInserts)+1);
417 |                         String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########");
418 |                         doc.put("c",cVal);
419 |                         String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########");
420 |                         doc.put("pad",padVal);
421 |                         WriteResult wrInsert = coll.insert(doc);
422 |                     }
423 | 
424 |                     globalSysbenchTransactions.incrementAndGet();
425 |                     numTransactions += 1;
426 | 
427 | 		    long timeEnd = System.currentTimeMillis();
428 |                     globalNewBatchQty.addAndGet(1);
429 |                     globalNewBatchLatency.addAndGet(timeEnd-timeStart);
430 |                 }
431 | 
432 |                 catch (Exception e) {
433 |                     globalExceptions.incrementAndGet();
434 |                 //    //logMe("Writer thread %d : EXCEPTION",threadNumber);
435 |                 //    //e.printStackTrace();
436 |                 }
437 |                 
438 |                 finally {
439 |                     
440 |                 }
441 |                 
442 |             }
443 | 
444 |             globalWriterThreads.decrementAndGet();
445 |         }
446 |     }
447 | 
448 | 
449 |     public static String sysbenchString(java.util.Random rand, String thisMask) {
450 |         StringBuilder sb = new StringBuilder();
451 |         for (int i = 0, n = thisMask.length() ; i < n ; i++) { 
452 |             char c = thisMask.charAt(i); 
453 |             if (c == '#') {
454 |                 sb.append(String.valueOf(rand.nextInt(10)));
455 |             } else if (c == '@') {
456 |                 sb.append((char) (rand.nextInt(26) + 'a'));
457 |             } else {
458 |                 sb.append(c);
459 |             }
460 |         }
461 |         return sb.toString();
462 |     }
463 | 
464 | 
465 |     // reporting thread, outputs information to console and file
466 |     class MyReporter implements Runnable {
467 |         public void run()
468 |         {
469 |             long t0 = System.currentTimeMillis();
470 |             long lastInserts = 0;
471 |             long thisInserts = 0;
472 |             long lastDeletes = 0;
473 |             long thisDeletes = 0;
474 |             long lastUpdates = 0;
475 |             long thisUpdates = 0;
476 |             long thisExceptions = 0;
477 |             long lastExceptions = 0;
478 |             long lastPointQueries = 0;
479 |             long thisPointQueries = 0;
480 |             long lastRangeQueries = 0;
481 |             long thisRangeQueries = 0;
482 |             long lastSysbenchTransactions = 0;
483 |             long thisSysbenchTransactions = 0;
484 |             long lastMs = t0;
485 |             long intervalNumber = 0;
486 |             long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
487 |             long runEndMillis = Long.MAX_VALUE;
488 |             if (runSeconds > 0)
489 |                 runEndMillis = t0 + (1000 * runSeconds);
490 | 
491 |             while ((System.currentTimeMillis() < runEndMillis) && (thisInserts < numMaxInserts))
492 |             {
493 |                 try {
494 |                     Thread.sleep(100);
495 |                 } catch (Exception e) {
496 |                     e.printStackTrace();
497 |                 }
498 | 
499 |                 long now = System.currentTimeMillis();
500 | 
501 |                 if ((now > nextFeedbackMillis) && (secondsPerFeedback > 0))
502 |                 {
503 |                     thisInserts = globalInserts.get();
504 |                     thisSysbenchTransactions = globalSysbenchTransactions.get();
505 |                     thisExceptions = globalExceptions.get();
506 | 
507 |                     intervalNumber++;
508 |                     nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
509 | 
510 |                     long elapsed = now - t0;
511 |                     long thisIntervalMs = now - lastMs;
512 |                     long thisWriterThreads = globalWriterThreads.get();
513 | 
514 |                     long thisIntervalSysbenchTransactions = thisSysbenchTransactions - lastSysbenchTransactions;
515 |                     double thisIntervalSysbenchTransactionsPerSecond = thisIntervalSysbenchTransactions/(double)thisIntervalMs*1000.0;
516 |                     double thisSysbenchTransactionsPerSecond = thisSysbenchTransactions/(double)elapsed*1000.0;
517 | 
518 |                     long thisIntervalInserts = thisInserts - lastInserts;
519 |                     double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0;
520 |                     double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0;
521 |                     
522 |                     long thisIntervalExceptions = thisExceptions - lastExceptions;
523 | 
524 |                     long thisBatchQty = globalNewBatchQty.getAndSet(0);
525 |                     long thisBatchLatency = globalNewBatchLatency.getAndSet(0);
526 |                     double thisIntervalLatency = 0.0;
527 | 
528 |                     if (thisBatchQty > 0)
529 |                     {
530 |                         thisIntervalLatency = (double)thisBatchLatency / (double)thisBatchQty;
531 |                     }
532 | 
533 |                     logMe("%,d seconds : cum tps=%,.2f : int tps=%,.2f : cum ips=%,.2f : int ips=%,.2f : cum excp=%,d : int excpt=%,d : writers=%,d : int lat=%,.2f", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisExceptions, thisIntervalExceptions, thisWriterThreads, thisIntervalLatency);
534 |                     
535 |                     try {
536 |                         if (outputHeader)
537 |                         {
538 |                             writer.write("elap_secs\tcum_tps\tint_tps\tcum_ips\tint_ips\tcum_excp\tint_excp\tint_lat\n");
539 |                             outputHeader = false;
540 |                         }
541 | 
542 |                         String statusUpdate = "";
543 | 
544 |                         statusUpdate = String.format("%d\t%.2f\t%.2f\t%.2f\t%.2f\t%d\t%d\t%.2f\n", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisExceptions, thisIntervalExceptions, thisIntervalLatency);
545 | 
546 |                         writer.write(statusUpdate);
547 |                         writer.flush();
548 |                     } catch (IOException e) {
549 |                         e.printStackTrace();
550 |                     }
551 | 
552 |                     lastInserts = thisInserts;
553 |                     lastSysbenchTransactions = thisSysbenchTransactions;
554 |                     lastExceptions = thisExceptions;
555 | 
556 |                     lastMs = now;
557 |                 }
558 |             }
559 | 
560 |             // shutdown all the writers
561 |             allDone = 1;
562 |         }
563 |     }
564 | 
565 | 
566 |     public static void logMe(String format, Object... args) {
567 |         System.out.println(Thread.currentThread() + String.format(format, args));
568 |     }
569 | }
570 | 


--------------------------------------------------------------------------------
/src/jmongosysbenchshardedexecute.java:
--------------------------------------------------------------------------------
  1 | //import com.mongodb.Mongo;
  2 | import java.io.BufferedWriter;
  3 | import java.io.File;
  4 | import java.io.FileWriter;
  5 | import java.io.IOException;
  6 | import java.io.Writer;
  7 | import java.util.Arrays;
  8 | import java.util.ArrayList;
  9 | import java.util.List;
 10 | import java.util.concurrent.atomic.AtomicLong;
 11 | 
 12 | import com.mongodb.AggregationOutput;
 13 | import com.mongodb.BasicDBObject;
 14 | import com.mongodb.CommandResult;
 15 | import com.mongodb.DB;
 16 | import com.mongodb.DBCollection;
 17 | import com.mongodb.DBCursor;
 18 | import com.mongodb.DBObject;
 19 | import com.mongodb.MongoClient;
 20 | import com.mongodb.MongoClientOptions;
 21 | import com.mongodb.MongoClientURI;
 22 | import com.mongodb.MongoCredential;
 23 | import com.mongodb.ServerAddress;
 24 | import com.mongodb.WriteConcern;
 25 | import com.mongodb.WriteResult;
 26 | 
 27 | public class jmongosysbenchshardedexecute {
 28 |     public static AtomicLong globalInserts = new AtomicLong(0);
 29 |     public static AtomicLong globalDeletes = new AtomicLong(0);
 30 |     public static AtomicLong globalUpdates = new AtomicLong(0);
 31 |     public static AtomicLong globalPointQueries = new AtomicLong(0);
 32 |     public static AtomicLong globalRangeQueries = new AtomicLong(0);
 33 |     public static AtomicLong globalSysbenchTransactions = new AtomicLong(0);
 34 |     public static AtomicLong globalWriterThreads = new AtomicLong(0);
 35 |     public static AtomicLong globalExceptions = new AtomicLong(0);
 36 | 
 37 |     public static Writer writer = null;
 38 |     public static boolean outputHeader = true;
 39 | 
 40 |     public static int numCollections;
 41 |     public static String dbName;
 42 |     public static int writerThreads;
 43 |     public static Integer numMaxInserts;
 44 |     public static long secondsPerFeedback;
 45 |     public static String logFileName;
 46 |     public static String indexTechnology;
 47 |     public static String autoCommit;
 48 |     public static int runSeconds;
 49 |     public static String myWriteConcern;
 50 |     public static Integer maxTPS;
 51 |     public static Integer maxThreadTPS;
 52 |     public static String serverName;
 53 |     public static int serverPort;
 54 |     public static String userName;
 55 |     public static String passWord;
 56 |     public static String readPreference;
 57 |     public static String trustStore;
 58 |     public static String trustStorePassword;
 59 |     public static String useSSL;
 60 |     public static int maxShardKey;
 61 |     public static int docsPerShard;
 62 | 
 63 |     public static int oltpRangeSize;
 64 |     public static int oltpPointSelects;
 65 |     public static int oltpSimpleRanges;
 66 |     public static int oltpSumRanges;
 67 |     public static int oltpOrderRanges;
 68 |     public static int oltpDistinctRanges;
 69 |     public static int oltpIndexUpdates;
 70 |     public static int oltpNonIndexUpdates;
 71 |     public static int oltpInserts;
 72 | 
 73 |     public static boolean bIsTokuMX = false;
 74 | 
 75 |     public static int allDone = 0;
 76 | 
 77 |     public static long rngSeed = 0;
 78 |     
 79 |     public jmongosysbenchshardedexecute() {
 80 |     }
 81 | 
 82 |     public static void main (String[] args) throws Exception {
 83 |         if (args.length != 30) {
 84 |             logMe("*** ERROR : CONFIGURATION ISSUE ***");
 85 |             logMe("jsysbenchshardedexecute [number of collections] [database name] [number of writer threads] [documents per collection] [seconds feedback] "+
 86 |                                    "[log file name] [auto commit Y/N] [runtime (seconds)] [range size] [point selects] "+
 87 |                                    "[simple ranges] [sum ranges] [order ranges] [distinct ranges] [index updates] [non index updates] [inserts] [writeconcern] "+
 88 |                                    "[max tps] [server] [port] [seed] [username] [password] [read preference] [trust store] [trust store password] [use ssl] [max shard key] [num docs per shard]");
 89 |             System.exit(1);
 90 |         }
 91 |         
 92 |         numCollections = Integer.valueOf(args[0]);
 93 |         dbName = args[1];
 94 |         writerThreads = Integer.valueOf(args[2]);
 95 |         numMaxInserts = Integer.valueOf(args[3]);
 96 |         secondsPerFeedback = Long.valueOf(args[4]);
 97 |         logFileName = args[5];
 98 |         autoCommit = args[6];
 99 |         runSeconds = Integer.valueOf(args[7]);
100 |         oltpRangeSize = Integer.valueOf(args[8]);
101 |         oltpPointSelects = Integer.valueOf(args[9]);
102 |         oltpSimpleRanges = Integer.valueOf(args[10]);
103 |         oltpSumRanges = Integer.valueOf(args[11]);
104 |         oltpOrderRanges = Integer.valueOf(args[12]);
105 |         oltpDistinctRanges = Integer.valueOf(args[13]);
106 |         oltpIndexUpdates = Integer.valueOf(args[14]);
107 |         oltpNonIndexUpdates = Integer.valueOf(args[15]);
108 |         oltpInserts = Integer.valueOf(args[16]);
109 |         myWriteConcern = args[17];
110 |         maxTPS = Integer.valueOf(args[18]);
111 |         serverName = args[19];
112 |         serverPort = Integer.valueOf(args[20]);
113 |         rngSeed = Long.valueOf(args[21]);
114 |         userName = args[22];
115 |         passWord = args[23];
116 | 		readPreference = args[24];
117 | 		trustStore = args[25];
118 | 		trustStorePassword = args[26];
119 | 		useSSL = args[27].toLowerCase();
120 |         maxShardKey = Integer.valueOf(args[28]);
121 |         docsPerShard = Integer.valueOf(args[29]);
122 | 
123 | 	    // override
124 | 	    numMaxInserts = maxShardKey * docsPerShard;
125 | 
126 |         maxThreadTPS = (maxTPS / writerThreads) + 1;
127 | 
128 |         WriteConcern myWC = new WriteConcern();
129 |         if (myWriteConcern.toLowerCase().equals("acknowledged")) {
130 |             myWC = WriteConcern.ACKNOWLEDGED;
131 |         }
132 |         else if ((myWriteConcern.toLowerCase().equals("unacknowledged"))) {
133 |             myWC = WriteConcern.UNACKNOWLEDGED;
134 |         }
135 |         else if ((myWriteConcern.toLowerCase().equals("w1"))) {
136 |             myWC = WriteConcern.W1;
137 |         }
138 |         else if ((myWriteConcern.toLowerCase().equals("w2"))) {
139 |             myWC = WriteConcern.W2;
140 |         }
141 |         else if ((myWriteConcern.toLowerCase().equals("w3"))) {
142 |             myWC = WriteConcern.W3;
143 |         }
144 |         else {
145 |             logMe("*** ERROR : WRITE CONCERN ISSUE ***");
146 |             logMe("  write concern %s is not supported",myWriteConcern);
147 |             System.exit(1);
148 |         }
149 | 
150 |         logMe("Application Parameters");
151 |         logMe("-------------------------------------------------------------------------------------------------");
152 |         logMe("  collections              = %d",numCollections);
153 |         logMe("  database name            = %s",dbName);
154 |         logMe("  writer threads           = %d",writerThreads);
155 |         logMe("  documents per collection = %,d",numMaxInserts);
156 |         logMe("  feedback seconds         = %,d",secondsPerFeedback);
157 |         logMe("  log file                 = %s",logFileName);
158 |         logMe("  auto commit              = %s",autoCommit);
159 |         logMe("  run seconds              = %d",runSeconds);
160 |         logMe("  oltp range size          = %d",oltpRangeSize);
161 |         logMe("  oltp point selects       = %d",oltpPointSelects);
162 |         logMe("  oltp simple ranges       = %d",oltpSimpleRanges);
163 |         logMe("  oltp sum ranges          = %d",oltpSumRanges);
164 |         logMe("  oltp order ranges        = %d",oltpOrderRanges);
165 |         logMe("  oltp distinct ranges     = %d",oltpDistinctRanges);
166 |         logMe("  oltp index updates       = %d",oltpIndexUpdates);
167 |         logMe("  oltp non index updates   = %d",oltpNonIndexUpdates);
168 |         logMe("  oltp inserts             = %d",oltpInserts);
169 |         logMe("  write concern            = %s",myWriteConcern);
170 |         logMe("  maximum tps (global)     = %d",maxTPS);
171 |         logMe("  maximum tps (per thread) = %d",maxThreadTPS);
172 |         logMe("  Server:Port = %s:%d",serverName,serverPort);
173 |         logMe("  seed                     = %d",rngSeed);
174 |         logMe("  userName                 = %s",userName);
175 |         logMe("  read preference          = %s",readPreference);
176 | 		logMe("  use SSL                  = %s",useSSL);
177 |         logMe("  max shard key            = %s",maxShardKey);
178 |         logMe("  num docs per shard       = %s",docsPerShard);
179 | 
180 | 		/*
181 |         MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build();
182 |         ServerAddress srvrAdd = new ServerAddress(serverName,serverPort);
183 | 
184 |         // Credential login is optional.
185 |         MongoClient m;
186 |         if (userName.isEmpty() || userName.equalsIgnoreCase("none")) {
187 |             m = new MongoClient(srvrAdd);
188 |         } else {
189 |             MongoCredential credential = MongoCredential.createCredential(userName, dbName, passWord.toCharArray());
190 |             m = new MongoClient(srvrAdd, Arrays.asList(credential));
191 |         }
192 | 		*/
193 | 
194 |         //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&replicaSet=rs0&readpreference=%s&maxPoolSize=4096";
195 | 	    //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=4096&serverSelectionTimeoutMS=30000&replicaSet=rs0";
196 | 	    String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=4096";
197 |         String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference);
198 |         //logMe("  connection string = %s",connectionString);
199 | 
200 |         //if (useSSL.equals("true")) {
201 |         //    System.setProperty("javax.net.ssl.trustStore", trustStore);
202 |         //    System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
203 | 	    //	}
204 | 
205 |         //MongoClient m = new MongoClient(new MongoClientURI(connectionString),clientOptions);
206 |         MongoClient m = new MongoClient(new MongoClientURI(connectionString));
207 | 
208 |         //logMe("mongoOptions | " + m.getMongoOptions().toString());
209 |         //logMe("mongoWriteConcern | " + m.getWriteConcern().toString());
210 | 
211 |         DB db = m.getDB(dbName);
212 | 
213 |         // determine server type : mongo or tokumx
214 |         DBObject checkServerCmd = new BasicDBObject();
215 |         CommandResult commandResult = db.command("buildInfo");
216 | 
217 |         // check if tokumxVersion exists, otherwise assume mongo
218 |         if (commandResult.toString().contains("tokumxVersion")) {
219 |             indexTechnology = "tokumx";
220 |         }
221 |         else
222 |         {
223 |             indexTechnology = "mongo";
224 |         }
225 | 
226 |         logMe("  index technology         = %s",indexTechnology);
227 |         logMe("-------------------------------------------------------------------------------------------------");
228 | 
229 |         try {
230 |             writer = new BufferedWriter(new FileWriter(new File(logFileName)));
231 |         } catch (IOException e) {
232 |             e.printStackTrace();
233 |         }
234 | 
235 |         if ((!indexTechnology.toLowerCase().equals("tokumx")) && (!indexTechnology.toLowerCase().equals("mongo"))) {
236 |             // unknown index technology, abort
237 |             logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology);
238 |             System.exit(1);
239 |         }
240 | 
241 |         if (indexTechnology.toLowerCase().equals("tokumx")) {
242 |             bIsTokuMX = true;
243 |         }
244 | 
245 |         jmongosysbenchshardedexecute t = new jmongosysbenchshardedexecute();
246 | 
247 |         Thread[] tWriterThreads = new Thread[writerThreads];
248 | 
249 |         for (int i=0; i<writerThreads; i++) {
250 |             tWriterThreads[i] = new Thread(t.new MyWriter(writerThreads, i, numMaxInserts, db, numCollections, rngSeed));
251 |             tWriterThreads[i].start();
252 |         }
253 | 
254 |         Thread reporterThread = new Thread(t.new MyReporter());
255 |         reporterThread.start();
256 |         reporterThread.join();
257 | 
258 |         // wait for writer threads to terminate
259 |         for (int i=0; i<writerThreads; i++) {
260 |             if (tWriterThreads[i].isAlive())
261 |                 tWriterThreads[i].join();
262 |         }
263 | 
264 |         try {
265 |             if (writer != null) {
266 |                 writer.close();
267 |             }
268 |         } catch (IOException e) {
269 |             e.printStackTrace();
270 |         }
271 | 
272 |         m.close();
273 | 
274 |         logMe("Done!");
275 |     }
276 | 
277 |     class MyWriter implements Runnable {
278 |         int threadCount;
279 |         int threadNumber;
280 |         int numTables;
281 |         int numMaxInserts;
282 |         int numCollections;
283 |         DB db;
284 | 
285 |         long numInserts = 0;
286 |         long numDeletes = 0;
287 |         long numUpdates = 0;
288 |         long numPointQueries = 0;
289 |         long numRangeQueries = 0;
290 | 
291 |         java.util.Random rand;
292 |         
293 |         MyWriter(int threadCount, int threadNumber, int numMaxInserts, DB db, int numCollections, long rngSeed) {
294 |             this.threadCount = threadCount;
295 |             this.threadNumber = threadNumber;
296 |             this.numMaxInserts = numMaxInserts;
297 |             this.db = db;
298 |             this.numCollections = numCollections;
299 |             rand = new java.util.Random((long) threadNumber + rngSeed);
300 |         }
301 |         public void run() {
302 |             logMe("Writer thread %d : started",threadNumber);
303 |             globalWriterThreads.incrementAndGet();
304 | 
305 |             long numTransactions = 0;
306 |             long numLastTransactions = 0;
307 |             long nextMs = System.currentTimeMillis() + 1000;
308 | 
309 |             boolean auto_commit = !autoCommit.toLowerCase().equals("n");
310 | 
311 |             while (allDone == 0) {
312 |                 if ((numTransactions - numLastTransactions) >= maxThreadTPS) {
313 |                     // pause until a second has passed
314 |                     while (System.currentTimeMillis() < nextMs) {
315 |                         try {
316 |                             Thread.sleep(20);
317 |                         } catch (Exception e) {
318 |                             e.printStackTrace();
319 |                         }
320 |                     }
321 |                     numLastTransactions = numTransactions;
322 |                     nextMs = System.currentTimeMillis() + 1000;
323 |                 }
324 | 
325 |                 String collectionName = "sbtest" + Integer.toString(rand.nextInt(numCollections)+1);
326 |                 DBCollection coll = db.getCollection(collectionName);
327 | 
328 |                 try {
329 | 		            // pick shard key
330 | 		            int thisShardKey = rand.nextInt(maxShardKey);
331 | 
332 |                     for (int i=1; i <= oltpPointSelects; i++) {
333 |                         //for i=1, oltp_point_selects do
334 |                         //   rs = db_query("SELECT c FROM ".. table_name .." WHERE id=" .. sb_rand(1, oltp_table_size))
335 |                         //end
336 | 
337 |                         // db.sbtest8.find({_id: 554312}, {c: 1, _id: 0})
338 | 
339 |                         int startId = rand.nextInt(docsPerShard)+1;
340 | 
341 |                         //BasicDBObject query = new BasicDBObject("_id", startId);
342 |                         BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId);
343 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
344 | 
345 |                         DBObject myDoc = coll.findOne(query, columns);
346 |                         //System.out.println(myDoc);
347 | 
348 |                         globalPointQueries.incrementAndGet();
349 |                     }
350 | 
351 |                     for (int i=1; i <= oltpSimpleRanges; i++) {
352 |                         //for i=1, oltp_simple_ranges do
353 |                         //   range_start = sb_rand(1, oltp_table_size)
354 |                         //   rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1)
355 |                         //end
356 | 
357 |                         //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0})
358 | 
359 |                         int startId = rand.nextInt(docsPerShard)+1;
360 |                         int endId = startId + oltpRangeSize - 1;
361 | 
362 |                         //BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId));
363 | 			            List<DBObject> queryCriteria = new ArrayList<DBObject>();
364 |                         queryCriteria.add(new BasicDBObject("shardKey", thisShardKey));
365 |                         queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId)));
366 | 
367 |                         BasicDBObject query = new BasicDBObject("$and", queryCriteria);
368 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
369 |                         DBCursor cursor = coll.find(query, columns);
370 |                         try {
371 |                             while(cursor.hasNext()) {
372 |                                 cursor.next();
373 |                                 //System.out.println(cursor.next());
374 |                             }
375 |                         } finally {
376 |                             cursor.close();
377 |                         }
378 | 
379 |                         globalRangeQueries.incrementAndGet();
380 |                     }
381 | 
382 |                     for (int i=1; i <= oltpSumRanges; i++) {
383 |                         //for i=1, oltp_sum_ranges do
384 |                         //   range_start = sb_rand(1, oltp_table_size)
385 |                         //   rs = db_query("SELECT SUM(K) FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1)
386 |                         //end
387 | 
388 |                         //db.sbtest8.aggregate([ {$match: {_id: {$gt: 5523412, $lt: 5523512}}}, { $group: { _id: null, total: { $sum: "$k"}} } ])
389 | 
390 |                         int startId = rand.nextInt(docsPerShard)+1;
391 |                         int endId = startId + oltpRangeSize - 1;
392 | 
393 |                         // create our pipeline operations, first with the $match
394 | 			            List<DBObject> queryCriteria = new ArrayList<DBObject>();
395 |                         queryCriteria.add(new BasicDBObject("shardKey", thisShardKey));
396 |                         queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId)));
397 |                         DBObject match = new BasicDBObject("$match", new BasicDBObject("$and", queryCriteria));
398 | 
399 |                         // build the $projection operation
400 |                         DBObject fields = new BasicDBObject("k", 1);
401 |                         fields.put("_id", 0);
402 |                         DBObject project = new BasicDBObject("$project", fields );
403 | 
404 |                         // Now the $group operation
405 |                         DBObject groupFields = new BasicDBObject( "_id", null);
406 |                         groupFields.put("average", new BasicDBObject( "$sum", "$k"));
407 |                         DBObject group = new BasicDBObject("$group", groupFields);
408 | 
409 |                         // run aggregation
410 |                         AggregationOutput output = coll.aggregate( match, project, group );
411 | 
412 |                         //System.out.println(output.getCommandResult());
413 | 
414 |                         globalRangeQueries.incrementAndGet();
415 |                     }
416 | 
417 |                     for (int i=1; i <= oltpOrderRanges; i++) {
418 |                         //for i=1, oltp_order_ranges do
419 |                         //   range_start = sb_rand(1, oltp_table_size)
420 |                         //   rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c")
421 |                         //end
422 | 
423 |                         //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0}).sort({c: 1})
424 | 
425 |                         int startId = rand.nextInt(docsPerShard)+1;
426 |                         int endId = startId + oltpRangeSize - 1;
427 | 
428 | 			            List<DBObject> queryCriteria = new ArrayList<DBObject>();
429 |                         queryCriteria.add(new BasicDBObject("shardKey", thisShardKey));
430 |                         queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId)));
431 | 
432 |                         BasicDBObject query = new BasicDBObject("$and", queryCriteria);
433 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
434 |                         DBCursor cursor = coll.find(query, columns).sort(new BasicDBObject("c",1));
435 |                         try {
436 |                             while(cursor.hasNext()) {
437 |                                 cursor.next();
438 |                                 //System.out.println(cursor.next());
439 |                             }
440 |                         } finally {
441 |                             cursor.close();
442 |                         }
443 | 
444 |                         globalRangeQueries.incrementAndGet();
445 |                     }
446 | 
447 |                     for (int i=1; i <= oltpDistinctRanges; i++) {
448 |                         //for i=1, oltp_distinct_ranges do
449 |                         //   range_start = sb_rand(1, oltp_table_size)
450 |                         //   rs = db_query("SELECT DISTINCT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c")
451 |                         //end
452 | 
453 |                         //db.sbtest8.distinct("c",{_id: {$gt: 5523412, $lt: 5523512}}).sort()
454 | 
455 |                         int startId = rand.nextInt(docsPerShard)+1;
456 |                         int endId = startId + oltpRangeSize - 1;
457 | 
458 | 			            List<DBObject> queryCriteria = new ArrayList<DBObject>();
459 |                         queryCriteria.add(new BasicDBObject("shardKey", thisShardKey));
460 |                         queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId)));
461 | 
462 |                         BasicDBObject query = new BasicDBObject("$and", queryCriteria);
463 |                         BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0);
464 |                         List lstDistinct = coll.distinct("c", query);
465 |                         //System.out.println(lstDistinct.toString());
466 | 
467 |                         globalRangeQueries.incrementAndGet();
468 |                     }
469 | 
470 | 		            //'A {multi:false} update on a sharded collection must either contain an exact match on _id or must target a single shard but this update targeted _id (and have the collection default collation) or must target a single shard (and have the simple collation), but this update targeted multiple shards.'
471 |                     for (int i=1; i <= oltpIndexUpdates; i++) {
472 |                         //for i=1, oltp_index_updates do
473 |                         //   rs = db_query("UPDATE " .. table_name .. " SET k=k+1 WHERE id=" .. sb_rand(1, oltp_table_size))
474 |                         //end
475 | 
476 |                         //db.sbtest8.update({_id: 5523412}, {$inc: {k: 1}}, false, false)
477 | 
478 |                         int startId = rand.nextInt(docsPerShard)+1;
479 | 
480 |                         BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId);
481 |                         //System.out.println(query.toString());
482 | 
483 |                         WriteResult wrUpdate = coll.update(query, new BasicDBObject("$inc", new BasicDBObject("k",1)), false, false);
484 |     
485 |                         //System.out.println(wrUpdate.toString());
486 |                     }
487 |    
488 |                     for (int i=1; i <= oltpNonIndexUpdates; i++) {
489 |                         //for i=1, oltp_non_index_updates do
490 |                         //   c_val = sb_rand_str("###########-###########-###########-###########-###########-###########-###########-###########-###########-###########")
491 |                         //   query = "UPDATE " .. table_name .. " SET c='" .. c_val .. "' WHERE id=" .. sb_rand(1, oltp_table_size)
492 |                         //   rs = db_query(query)
493 |                         //   if rs then
494 |                         //     print(query)
495 |                         //   end
496 |                         //end
497 | 
498 |                         //db.sbtest8.update({_id: 5523412}, {$set: {c: "hello there"}}, false, false)
499 | 
500 |                         int startId = rand.nextInt(docsPerShard)+1;
501 | 
502 |                         BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId);
503 | 
504 |                         String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########");
505 | 
506 |                         WriteResult wrUpdate = coll.update(query, new BasicDBObject("$set", new BasicDBObject("c",cVal)), false, false);
507 | 
508 |                         //System.out.println(wrUpdate.toString());
509 |                     }
510 | 
511 |                     for (int i=1; i <= oltpInserts; i++) {
512 |                         //i = sb_rand(1, oltp_table_size)
513 |                         //rs = db_query("DELETE FROM " .. table_name .. " WHERE id=" .. i)
514 |                       
515 |                         //db.sbtest8.remove({_id: 5523412})
516 | 
517 |                         int startId = rand.nextInt(docsPerShard)+1;
518 | 
519 |                         BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId);
520 | 
521 |                         DBObject priorDoc = coll.findAndRemove(query);
522 | 
523 | 		                while (priorDoc == null) {
524 |                             //WriteResult wrRemove = coll.remove(new BasicDBObject("_id", startId));
525 |                             priorDoc = coll.findAndRemove(query);
526 |                             //System.out.println(startId);
527 | 			            }
528 | 
529 |                         //c_val = sb_rand_str([[###########-###########-###########-###########-###########-###########-###########-###########-###########-###########]])
530 |                         //pad_val = sb_rand_str([[###########-###########-###########-###########-###########]])
531 |                         //rs = db_query("INSERT INTO " .. table_name ..  " (id, k, c, pad) VALUES " .. string.format("(%d, %d, '%s', '%s')",i, sb_rand(1, oltp_table_size) , c_val, pad_val))
532 | 
533 |                         BasicDBObject doc = new BasicDBObject();
534 |                         doc.put("_id",priorDoc.get("_id"));
535 |                         doc.put("shardKey",thisShardKey);
536 |                         doc.put("orderId",startId);
537 |                         doc.put("k",rand.nextInt(docsPerShard)+1);
538 |                         String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########");
539 |                         doc.put("c",cVal);
540 |                         String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########");
541 |                         doc.put("pad",padVal);
542 |                         WriteResult wrInsert = coll.insert(doc);
543 |                     }
544 | 
545 |                     globalSysbenchTransactions.incrementAndGet();
546 |                     numTransactions += 1;
547 |                 } 
548 |                 
549 |                 catch (Exception e) {
550 |                     globalExceptions.incrementAndGet();
551 |                     //logMe("Writer thread %d : EXCEPTION",threadNumber);
552 |                     //e.printStackTrace();
553 |                     try {
554 |                         Thread.sleep(1000);
555 |                     } catch (Exception eSleep) {
556 |                         eSleep.printStackTrace();
557 |                     }
558 |                 }
559 |                 
560 |                 finally {
561 |                     
562 |                 }
563 |                 
564 |             }
565 | 
566 |             globalWriterThreads.decrementAndGet();
567 |         }
568 |     }
569 | 
570 | 
571 |     public static String sysbenchString(java.util.Random rand, String thisMask) {
572 |         StringBuilder sb = new StringBuilder();
573 |         for (int i = 0, n = thisMask.length() ; i < n ; i++) { 
574 |             char c = thisMask.charAt(i); 
575 |             if (c == '#') {
576 |                 sb.append(String.valueOf(rand.nextInt(10)));
577 |             } else if (c == '@') {
578 |                 sb.append((char) (rand.nextInt(26) + 'a'));
579 |             } else {
580 |                 sb.append(c);
581 |             }
582 |         }
583 |         return sb.toString();
584 |     }
585 | 
586 | 
587 |     // reporting thread, outputs information to console and file
588 |     class MyReporter implements Runnable {
589 |         public void run()
590 |         {
591 |             long t0 = System.currentTimeMillis();
592 |             long lastInserts = 0;
593 |             long thisInserts = 0;
594 |             long lastDeletes = 0;
595 |             long thisDeletes = 0;
596 |             long lastUpdates = 0;
597 |             long thisUpdates = 0;
598 |             long thisExceptions = 0;
599 |             long lastExceptions = 0;
600 |             long lastPointQueries = 0;
601 |             long thisPointQueries = 0;
602 |             long lastRangeQueries = 0;
603 |             long thisRangeQueries = 0;
604 |             long lastSysbenchTransactions = 0;
605 |             long thisSysbenchTransactions = 0;
606 |             long lastMs = t0;
607 |             long intervalNumber = 0;
608 |             long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
609 |             long runEndMillis = Long.MAX_VALUE;
610 |             if (runSeconds > 0)
611 |                 runEndMillis = t0 + (1000 * runSeconds);
612 | 
613 | 	        // modified the following, only running time based for now
614 |             //while ((System.currentTimeMillis() < runEndMillis) && (thisInserts < numMaxInserts))
615 |             while (System.currentTimeMillis() < runEndMillis)
616 |             {
617 |                 try {
618 |                     Thread.sleep(100);
619 |                 } catch (Exception e) {
620 |                     e.printStackTrace();
621 |                 }
622 | 
623 |                 long now = System.currentTimeMillis();
624 | 
625 |                 if ((now > nextFeedbackMillis) && (secondsPerFeedback > 0))
626 |                 {
627 |                     thisInserts = globalInserts.get();
628 |                     thisSysbenchTransactions = globalSysbenchTransactions.get();
629 |                     thisExceptions = globalExceptions.get();
630 | 
631 |                     intervalNumber++;
632 |                     nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1));
633 | 
634 |                     long elapsed = now - t0;
635 |                     long thisIntervalMs = now - lastMs;
636 |                     long thisWriterThreads = globalWriterThreads.get();
637 | 
638 |                     long thisIntervalSysbenchTransactions = thisSysbenchTransactions - lastSysbenchTransactions;
639 |                     double thisIntervalSysbenchTransactionsPerSecond = thisIntervalSysbenchTransactions/(double)thisIntervalMs*1000.0;
640 |                     double thisSysbenchTransactionsPerSecond = thisSysbenchTransactions/(double)elapsed*1000.0;
641 | 
642 |                     long thisIntervalInserts = thisInserts - lastInserts;
643 |                     double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0;
644 |                     double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0;
645 | 
646 |                     long thisIntervalExceptions = thisExceptions - lastExceptions;
647 |                     
648 |                     logMe("%,d seconds : cum tps=%,.2f : int tps=%,.2f : cum excpt=%,d : int excpt=%,d : writers=%,d", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisExceptions, thisIntervalExceptions, thisWriterThreads);
649 |                     
650 |                     try {
651 |                         if (outputHeader)
652 |                         {
653 |                             writer.write("elap_secs\tcum_tps\tint_tps\tcum_excp\tint_excp\n");
654 |                             outputHeader = false;
655 |                         }
656 | 
657 |                         String statusUpdate = "";
658 | 
659 |                         statusUpdate = String.format("%d\t%.2f\t%.2f\t%d\t%d\n", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisExceptions, thisIntervalExceptions);
660 | 
661 |                         writer.write(statusUpdate);
662 |                         writer.flush();
663 |                     } catch (IOException e) {
664 |                         e.printStackTrace();
665 |                     }
666 | 
667 |                     lastInserts = thisInserts;
668 |                     lastSysbenchTransactions = thisSysbenchTransactions;
669 |                     lastExceptions = thisExceptions;
670 | 
671 |                     lastMs = now;
672 |                 }
673 |             }
674 | 
675 |             // shutdown all the writers
676 |             allDone = 1;
677 |         }
678 |     }
679 | 
680 | 
681 |     public static void logMe(String format, Object... args) {
682 |         System.out.println(Thread.currentThread() + String.format(format, args));
683 |     }
684 | }
685 | 


--------------------------------------------------------------------------------