├── .gitignore ├── LICENSE ├── README.md ├── TODO ├── check-results.bash ├── config.bash ├── run.looped.bash ├── run.simple.bash └── src ├── jmongosysbenchexecute.java ├── jmongosysbenchload.java ├── jmongosysbenchshardedexecute.java └── jmongosysbenchshardedload.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | .classpath 3 | .project 4 | 5 | # Package Files # 6 | *.jar 7 | *.war 8 | *.earA 9 | 10 | # Specific Files # 11 | config.tmc 12 | config.tmc.* 13 | config.anshu.* 14 | config.01 15 | config.02 16 | config.03 17 | config.04 18 | config.05 19 | config.06 20 | config.07 21 | config.08 22 | config.10 23 | clean.bash 24 | rds-truststore.jks 25 | log 26 | mongoSysbenchExecute*.txt 27 | mongoSysbenchExecute*.txt.tsv 28 | mongoSysbenchLoad*.txt 29 | mongoSysbenchLoad*.txt.tsv 30 | run.multi.bash 31 | run.tmc.bash 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | sysbench-mongodb 2 | ================ 3 | 4 | Sysbench benchmark for MongoDB compatible databases 5 | 6 | 7 | Requirements 8 | ===================== 9 | 10 | * Java 17 (tested working with OpenJDK) 11 | * MongoDB Java driver jars - specifically version 3.9.1 (for now) 12 | * they must be in the working directory (for now) 13 | * wget https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongo-java-driver/3.9.1/mongo-java-driver-3.9.1.jar 14 | * wget https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongodb-driver-core/3.9.1/mongodb-driver-core-3.9.1.jar 15 | * wget https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongodb-driver-legacy/3.9.1/mongodb-driver-legacy-3.9.1.jar 16 | * Modify config.bash file for URI (connection string) 17 | * TLS is optional 18 | * If not using TLS, simply disable it in config.bash or your custom config file 19 | * Otherwise you'll need to create your key store file. Instructions are available at https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html 20 | 21 | 22 | Running the benchmark 23 | ===================== 24 | 25 | In the default configuration the benchmark creates 1 collection with 1 million documents. You may want to watch the size of the database relative to your memory size to ensure you are testing just a memory based workload vs a workload that is exceeding memory and utilizing disk as well. All options are configurable in config.bash (or custom config file with the same options) 26 | 27 | To run: 28 | 29 | ```bash 30 | git clone https://github.com/tmcallaghan/sysbench-mongodb.git 31 | cd sysbench-mongodb 32 | 33 | ``` 34 | 35 | Edit config.bash to match your environment, you must change the URI for connecting to your server. 36 | 37 | ```bash 38 | ./run.simple.bash 39 | 40 | ``` 41 | 42 | If you want to have multiple config files you can simply copy config.bash and specify the config you would like on the command line: 43 | 44 | ```bash 45 | ./run.simple.bash my_custom_config.bash 46 | 47 | ``` 48 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - add insert governer for loader 2 | - improve command line argument passing 3 | - use "--argument value" syntax 4 | - add defaults 5 | -------------------------------------------------------------------------------- /check-results.bash: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | grep '280' log/*.txt.tsv | cut -f2 4 | -------------------------------------------------------------------------------- /config.bash: -------------------------------------------------------------------------------- 1 | # You will most likely want to change these values 2 | 3 | export SHARDED=no 4 | 5 | # database in which to run the benchmark 6 | export DB_NAME=sbtest10by1mm 7 | 8 | # URI connection string 9 | export URI= 10 | 11 | # trust store file 12 | # leave this alone if not using TLS 13 | export TRUST_STORE="./rds-truststore.jks" 14 | 15 | # trust store password 16 | # leave this alone if not using TLS 17 | export TRUST_STORE_PASSWORD="" 18 | 19 | # Use "yes" to load the collections 20 | export DOLOAD=yes 21 | 22 | # Use "yes" to execute the benchmark 23 | export DOQUERY=yes 24 | 25 | # number of collections to create for the benchmark 26 | # valid values : integer > 0 27 | export NUM_COLLECTIONS=1 28 | 29 | # number of documents to maintain per collection 30 | # valid values : integer > 0 31 | export NUM_DOCUMENTS_PER_COLLECTION=1000000 32 | 33 | # total number of documents to insert per "batch" 34 | # valid values : integer > 0 35 | export NUM_DOCUMENTS_PER_INSERT=1000 36 | 37 | # total number of simultaneous insertion threads (for loader) 38 | # valid values : integer > 0 39 | export NUM_LOADER_THREADS=10 40 | 41 | # total number of simultaneous benchmark threads 42 | # valid values : integer > 0 43 | export NUM_WRITER_THREADS=10 44 | 45 | # run the benchmark for this many minutes 46 | # valid values : intever > 0 47 | export RUN_TIME_MINUTES=10 48 | export RUN_TIME_SECONDS=$[RUN_TIME_MINUTES*60] 49 | 50 | # total number of transactions per second, allows for the benchmark to be rate limited 51 | # valid values : integer > 0 52 | export MAX_TPS=999999 53 | 54 | 55 | # ------------------------------------------------------------------------------------ 56 | # Sysbench specific parameters 57 | # ------------------------------------------------------------------------------------ 58 | 59 | # number of documents to retrieve in range queries 60 | # valid values : integer > 0 61 | export SYSBENCH_RANGE_SIZE=100 62 | 63 | # number of point queries per sysbench "transaction" 64 | # valid values : integer >= 0 65 | export SYSBENCH_POINT_SELECTS=10 66 | 67 | # number of simple range queries per sysbench "transaction" 68 | # valid values : integer >= 0 69 | export SYSBENCH_SIMPLE_RANGES=1 70 | 71 | # number of aggregation queries per sysbench "transaction" 72 | # valid values : integer >= 0 73 | export SYSBENCH_SUM_RANGES=1 74 | 75 | # number of ordered range queries per sysbench "transaction" 76 | # valid values : integer >= 0 77 | export SYSBENCH_ORDER_RANGES=1 78 | 79 | # number of distinct range queries per sysbench "transaction" 80 | # valid values : integer >= 0 81 | export SYSBENCH_DISTINCT_RANGES=1 82 | 83 | # set all of the following 3 parameters to zero for a read-only benchmark 84 | 85 | # number of indexed updates per sysbench "transaction" 86 | # valid values : integer >= 0 87 | export SYSBENCH_INDEX_UPDATES=1 88 | 89 | # number of non-indexed updates per sysbench "transaction" 90 | # valid values : integer >= 0 91 | export SYSBENCH_NON_INDEX_UPDATES=1 92 | 93 | # number of delete/insert operations per sysbench "transaction" 94 | # valid values : integer >= 0 95 | export SYSBENCH_INSERTS=1 96 | 97 | 98 | # ------------------------------------------------------------------------------------ 99 | # Parameters to leave alone 100 | # ------------------------------------------------------------------------------------ 101 | 102 | # Use a seed for the RNG, like $( date +%s ) 103 | # This can be used to repeat, or not, the sequence of keys used per test. 104 | export SEED=$( date +%s ) 105 | 106 | # display performance information every time the client application inserts this many documents 107 | # valid values : integer > 0, set to -1 if using NUM_SECONDS_PER_FEEDBACK 108 | export NUM_INSERTS_PER_FEEDBACK=-1 109 | 110 | # display performance information every time the client application has run for this many seconds 111 | # valid values : integer > 0, set to -1 if using NUM_INSERTS_PER_FEEDBACK 112 | export NUM_SECONDS_PER_FEEDBACK=10 113 | 114 | 115 | -------------------------------------------------------------------------------- /run.looped.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # enable passing different config files 4 | 5 | if [ ! $1 ]; 6 | then 7 | FILE="config.bash" 8 | else 9 | FILE=$1 10 | fi 11 | 12 | if [ -f $FILE ]; 13 | then 14 | echo "Loading config from $FILE....." 15 | source $FILE 16 | else 17 | echo "Unable to read config $FILE" 18 | exit 1 19 | fi 20 | 21 | CLASSPATH="mongo-java-driver-3.9.1.jar:mongodb-driver-core-3.9.1.jar:mongodb-driver-legacy-3.9.1.jar:$CLASSPATH" 22 | TAIL_LINES=21 23 | 24 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchload.java 25 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchexecute.java 26 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedload.java 27 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedexecute.java 28 | 29 | 30 | # load the data 31 | 32 | if [[ $DOLOAD = "yes" ]]; then 33 | echo Do load at $( date ) 34 | export LOG_NAME=mongoSysbenchLoad-${NUM_COLLECTIONS}-${NUM_DOCUMENTS_PER_COLLECTION}-${NUM_LOADER_THREADS}.txt 35 | export BENCHMARK_TSV=${LOG_NAME}.tsv 36 | 37 | rm -f $LOG_NAME 38 | rm -f $BENCHMARK_TSV 39 | 40 | T="$(date +%s)" 41 | 42 | if [[ $SHARDED = "no" ]]; then 43 | echo "executing standard sysbench load (not sharded)" | tee -a $LOG_NAME 44 | java -cp $CLASSPATH:$PWD/src jmongosysbenchload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS 0 0 45 | else 46 | echo "executing sharded sysbench load" | tee -a $LOG_NAME 47 | java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $MONGO_COMPRESSION $MONGO_BASEMENT $WRITE_CONCERN $MONGO_SERVER $MONGO_PORT "$USERNAME" "$PASSWORD" "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD 48 | fi 49 | 50 | echo "" | tee -a $LOG_NAME 51 | T="$(($(date +%s)-T))" 52 | printf "`date` | sysbench loader duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME 53 | echo "" 54 | echo "************************************************************************" 55 | echo "final ${numTailLines} interval(s)" 56 | echo "************************************************************************" 57 | tail -n $TAIL_LINES $LOG_NAME 58 | fi 59 | 60 | 61 | # execute the benchmark 62 | 63 | if [[ $DOQUERY = "yes" ]]; then 64 | #threadList="1 1 1 1 2 2 2 2 4 4 4 4 8 8 8 8 16 16 16 16 32 32 32 32 64 64 64 64 96 96 96 96 128 128 128 128 160 160 160 160 192 192 192 192 224 224 224 224 256 256 256 256 512 512 512 512" 65 | #threadList="1 1 2 2 4 4 8 8 16 16 32 32 64 64" 66 | threadList="1 1 2 2 4 4 8 8 16 16 32 32 64 64 96 96 128 128 160 160 192 192 224 224 256 256 512 512 1024 2048" 67 | #threadList="1 2 4" 68 | 69 | logDir="./log" 70 | mkdir $logDir 71 | 72 | for numThreads in $threadList; do 73 | NUM_WRITER_THREADS=$numThreads 74 | 75 | thisTimestamp=`date +%Y%m%d_%H%M%S` 76 | LOG_NAME="${logDir}/${thisTimestamp}_${NUM_WRITER_THREADS}_${MONGO_READ_PREFERENCE}_${RUN_TIME_SECONDS}.txt" 77 | export BENCHMARK_TSV=${LOG_NAME}.tsv 78 | 79 | echo "running with NUM_WRITER_THREADS=$NUM_WRITER_THREADS | MONGO_READ_PREFERENCE=$MONGO_READ_PREFERENCE | MONGO_SERVER=$MONGO_SERVER" | tee -a $thisLogName 80 | 81 | T="$(date +%s)" 82 | 83 | if [[ $SHARDED = "no" ]]; then 84 | echo "executing standard sysbench (not sharded)" | tee -a $LOG_NAME 85 | java -cp $CLASSPATH:$PWD/src jmongosysbenchexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS 0 0 | tee -a $LOG_NAME 86 | else 87 | echo "executing sharded sysbench" | tee -a $LOG_NAME 88 | java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $SYSBENCH_AUTO_COMMIT $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $WRITE_CONCERN $MAX_TPS $MONGO_SERVER $MONGO_PORT $SEED "$USERNAME" "$PASSWORD" $MONGO_READ_PREFERENCE "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $USE_TLS $MAX_SHARD_KEY $DOCS_PER_SHARD | tee -a $LOG_NAME 89 | fi 90 | 91 | echo "" | tee -a $LOG_NAME 92 | T="$(($(date +%s)-T))" 93 | printf "`date` | sysbench benchmark duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME 94 | echo "" 95 | echo "************************************************************************" 96 | echo "final ${numTailLines} interval(s)" 97 | echo "************************************************************************" 98 | tail -n $TAIL_LINES $LOG_NAME 99 | done 100 | fi 101 | 102 | -------------------------------------------------------------------------------- /run.simple.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # enable passing different config files 4 | 5 | if [ ! $1 ]; 6 | then 7 | FILE="config.bash" 8 | else 9 | FILE=$1 10 | fi 11 | 12 | if [ -f $FILE ]; 13 | then 14 | echo "Loading config from $FILE....." 15 | source $FILE 16 | else 17 | echo "Unable to read config $FILE" 18 | exit 1 19 | fi 20 | 21 | if [ $2 ]; 22 | then 23 | echo "Loading config overrides from $2....." 24 | source $2 25 | fi 26 | 27 | CLASSPATH="mongo-java-driver-3.9.1.jar:mongodb-driver-core-3.9.1.jar:mongodb-driver-legacy-3.9.1.jar:$CLASSPATH" 28 | TAIL_LINES=21 29 | 30 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchload.java 31 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchexecute.java 32 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedload.java 33 | javac -cp $CLASSPATH:$PWD/src src/jmongosysbenchshardedexecute.java 34 | 35 | 36 | # load the data 37 | 38 | if [[ $DOLOAD = "yes" ]]; then 39 | echo Do load at $( date ) 40 | export LOG_NAME=mongoSysbenchLoad-${DOCDB_DBNAME}-${NUM_COLLECTIONS}-${NUM_DOCUMENTS_PER_COLLECTION}-${NUM_LOADER_THREADS}.txt 41 | export BENCHMARK_TSV=${LOG_NAME}.tsv 42 | 43 | rm -f $LOG_NAME 44 | rm -f $BENCHMARK_TSV 45 | 46 | T="$(date +%s)" 47 | 48 | if [[ $SHARDED = "no" ]]; then 49 | echo "executing standard sysbench load (not sharded)" | tee -a $LOG_NAME 50 | java -cp $CLASSPATH:$PWD/src jmongosysbenchload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $URI 51 | else 52 | echo "executing sharded sysbench load" | tee -a $LOG_NAME 53 | java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedload $NUM_COLLECTIONS $DB_NAME $NUM_LOADER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $MAX_SHARD_KEY $DOCS_PER_SHARD $URI 54 | fi 55 | 56 | echo "" | tee -a $LOG_NAME 57 | T="$(($(date +%s)-T))" 58 | printf "`date` | sysbench loader duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME 59 | echo "" 60 | echo "************************************************************************" 61 | echo "final ${numTailLines} interval(s)" 62 | echo "************************************************************************" 63 | tail -n $TAIL_LINES $LOG_NAME 64 | fi 65 | 66 | 67 | # execute the benchmark 68 | 69 | if [[ $DOQUERY = "yes" ]]; then 70 | echo Do query at $( date ) 71 | export LOG_NAME=mongoSysbenchExecute-${DOCDB_DBNAME}-${NUM_COLLECTIONS}-${NUM_DOCUMENTS_PER_COLLECTION}-${NUM_WRITER_THREADS}.txt 72 | export BENCHMARK_TSV=${LOG_NAME}.tsv 73 | 74 | rm -f $LOG_NAME 75 | rm -f $BENCHMARK_TSV 76 | 77 | T="$(date +%s)" 78 | 79 | if [[ $SHARDED = "no" ]]; then 80 | echo "executing standard sysbench" | tee -a $LOG_NAME 81 | java -cp $CLASSPATH:$PWD/src jmongosysbenchexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $MAX_TPS $SEED "$TRUST_STORE" "$TRUST_STORE_PASSWORD" "$URI" | tee -a $LOG_NAME 82 | else 83 | echo "executing standard sysbench" | tee -a $LOG_NAME 84 | java -cp $CLASSPATH:$PWD/src jmongosysbenchshardedexecute $NUM_COLLECTIONS $DB_NAME $NUM_WRITER_THREADS $NUM_DOCUMENTS_PER_COLLECTION $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $RUN_TIME_SECONDS $SYSBENCH_RANGE_SIZE $SYSBENCH_POINT_SELECTS $SYSBENCH_SIMPLE_RANGES $SYSBENCH_SUM_RANGES $SYSBENCH_ORDER_RANGES $SYSBENCH_DISTINCT_RANGES $SYSBENCH_INDEX_UPDATES $SYSBENCH_NON_INDEX_UPDATES $SYSBENCH_INSERTS $MAX_TPS $SEED "$TRUST_STORE" "$TRUST_STORE_PASSWORD" $MAX_SHARD_KEY $DOCS_PER_SHARD "$URI" | tee -a $LOG_NAME 85 | fi 86 | 87 | echo "" | tee -a $LOG_NAME 88 | T="$(($(date +%s)-T))" 89 | printf "`date` | sysbench benchmark duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME 90 | echo "" 91 | echo "************************************************************************" 92 | echo "final ${numTailLines} interval(s)" 93 | echo "************************************************************************" 94 | tail -n $TAIL_LINES $LOG_NAME 95 | 96 | #cat $LOG_NAME | grep '170 seconds' >> results.txt 97 | fi 98 | 99 | -------------------------------------------------------------------------------- /src/jmongosysbenchexecute.java: -------------------------------------------------------------------------------- 1 | //import com.mongodb.Mongo; 2 | import java.io.BufferedWriter; 3 | import java.io.File; 4 | import java.io.FileWriter; 5 | import java.io.IOException; 6 | import java.io.Writer; 7 | import java.util.Arrays; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicLong; 10 | 11 | import java.util.logging.Level; 12 | import java.util.logging.Logger; 13 | 14 | import com.mongodb.AggregationOutput; 15 | import com.mongodb.BasicDBObject; 16 | import com.mongodb.CommandResult; 17 | import com.mongodb.DB; 18 | import com.mongodb.DBCollection; 19 | import com.mongodb.DBCursor; 20 | import com.mongodb.DBObject; 21 | import com.mongodb.MongoClient; 22 | import com.mongodb.MongoClientOptions; 23 | import com.mongodb.MongoClientURI; 24 | import com.mongodb.MongoCredential; 25 | import com.mongodb.ServerAddress; 26 | import com.mongodb.WriteConcern; 27 | import com.mongodb.WriteResult; 28 | 29 | public class jmongosysbenchexecute { 30 | public static AtomicLong globalInserts = new AtomicLong(0); 31 | public static AtomicLong globalDeletes = new AtomicLong(0); 32 | public static AtomicLong globalUpdates = new AtomicLong(0); 33 | public static AtomicLong globalPointQueries = new AtomicLong(0); 34 | public static AtomicLong globalRangeQueries = new AtomicLong(0); 35 | public static AtomicLong globalSysbenchTransactions = new AtomicLong(0); 36 | public static AtomicLong globalWriterThreads = new AtomicLong(0); 37 | public static AtomicLong globalExceptions = new AtomicLong(0); 38 | public static AtomicLong globalNewBatchQty = new AtomicLong(0); 39 | public static AtomicLong globalNewBatchLatency = new AtomicLong(0); 40 | 41 | public static Writer writer = null; 42 | public static boolean outputHeader = true; 43 | 44 | public static int numCollections; 45 | public static String dbName; 46 | public static int writerThreads; 47 | public static Integer numMaxInserts; 48 | public static long secondsPerFeedback; 49 | public static String logFileName; 50 | //public static String autoCommit; 51 | public static int runSeconds; 52 | public static Integer maxTPS; 53 | public static Integer maxThreadTPS; 54 | public static String trustStore; 55 | public static String trustStorePassword; 56 | public static String connectionString; 57 | 58 | public static int oltpRangeSize; 59 | public static int oltpPointSelects; 60 | public static int oltpSimpleRanges; 61 | public static int oltpSumRanges; 62 | public static int oltpOrderRanges; 63 | public static int oltpDistinctRanges; 64 | public static int oltpIndexUpdates; 65 | public static int oltpNonIndexUpdates; 66 | public static int oltpInserts; 67 | 68 | public static int allDone = 0; 69 | public static long rngSeed = 0; 70 | 71 | public jmongosysbenchexecute() { 72 | } 73 | 74 | public static void main (String[] args) throws Exception { 75 | if (args.length != 21) { 76 | logMe("*** ERROR : CONFIGURATION ISSUE ***"); 77 | logMe("jsysbenchexecute [number of collections] [database name] [number of writer threads] [documents per collection] [seconds feedback] "+ 78 | "[log file name] [runtime (seconds)] [range size] [point selects] "+ 79 | "[simple ranges] [sum ranges] [order ranges] [distinct ranges] [index updates] [non index updates] [inserts] "+ 80 | "[max tps] [seed] [trust store] [trust store password] [connection string]"); 81 | System.exit(1); 82 | } 83 | 84 | numCollections = Integer.valueOf(args[0]); 85 | dbName = args[1]; 86 | writerThreads = Integer.valueOf(args[2]); 87 | numMaxInserts = Integer.valueOf(args[3]); 88 | secondsPerFeedback = Long.valueOf(args[4]); 89 | logFileName = args[5]; 90 | runSeconds = Integer.valueOf(args[6]); 91 | oltpRangeSize = Integer.valueOf(args[7]); 92 | oltpPointSelects = Integer.valueOf(args[8]); 93 | oltpSimpleRanges = Integer.valueOf(args[9]); 94 | oltpSumRanges = Integer.valueOf(args[10]); 95 | oltpOrderRanges = Integer.valueOf(args[11]); 96 | oltpDistinctRanges = Integer.valueOf(args[12]); 97 | oltpIndexUpdates = Integer.valueOf(args[13]); 98 | oltpNonIndexUpdates = Integer.valueOf(args[14]); 99 | oltpInserts = Integer.valueOf(args[15]); 100 | maxTPS = Integer.valueOf(args[16]); 101 | rngSeed = Long.valueOf(args[17]); 102 | trustStore = args[18]; 103 | trustStorePassword = args[19]; 104 | connectionString = args[20]; 105 | 106 | maxThreadTPS = (maxTPS / writerThreads); 107 | 108 | logMe("Application Parameters"); 109 | logMe("-------------------------------------------------------------------------------------------------"); 110 | logMe(" collections = %d",numCollections); 111 | logMe(" database name = %s",dbName); 112 | logMe(" writer threads = %d",writerThreads); 113 | logMe(" documents per collection = %,d",numMaxInserts); 114 | logMe(" feedback seconds = %,d",secondsPerFeedback); 115 | logMe(" log file = %s",logFileName); 116 | logMe(" run seconds = %d",runSeconds); 117 | logMe(" oltp range size = %d",oltpRangeSize); 118 | logMe(" oltp point selects = %d",oltpPointSelects); 119 | logMe(" oltp simple ranges = %d",oltpSimpleRanges); 120 | logMe(" oltp sum ranges = %d",oltpSumRanges); 121 | logMe(" oltp order ranges = %d",oltpOrderRanges); 122 | logMe(" oltp distinct ranges = %d",oltpDistinctRanges); 123 | logMe(" oltp index updates = %d",oltpIndexUpdates); 124 | logMe(" oltp non index updates = %d",oltpNonIndexUpdates); 125 | logMe(" oltp inserts = %d",oltpInserts); 126 | logMe(" maximum tps (global) = %d",maxTPS); 127 | logMe(" maximum tps (per thread) = %d",maxThreadTPS); 128 | logMe(" seed = %d",rngSeed); 129 | logMe(" uri = %s",connectionString); 130 | logMe("-------------------------------------------------------------------------------------------------"); 131 | 132 | logMe("setting mongodb driver log level to WARNING"); 133 | java.util.logging.Logger.getLogger("org.mongodb.driver").setLevel(Level.WARNING); 134 | 135 | MongoClient m = new MongoClient(new MongoClientURI(connectionString)); 136 | 137 | DB db = m.getDB(dbName); 138 | 139 | /* 140 | // determine server type : mongo or tokumx 141 | DBObject checkServerCmd = new BasicDBObject(); 142 | CommandResult commandResult = db.command("buildInfo"); 143 | 144 | // check if tokumxVersion exists, otherwise assume mongo 145 | if (commandResult.toString().contains("tokumxVersion")) { 146 | indexTechnology = "tokumx"; 147 | } 148 | else 149 | { 150 | indexTechnology = "mongo"; 151 | } 152 | */ 153 | 154 | try { 155 | writer = new BufferedWriter(new FileWriter(new File(logFileName))); 156 | } catch (IOException e) { 157 | e.printStackTrace(); 158 | } 159 | 160 | jmongosysbenchexecute t = new jmongosysbenchexecute(); 161 | 162 | Thread[] tWriterThreads = new Thread[writerThreads]; 163 | 164 | for (int i=0; i= maxThreadTPS) { 226 | // pause until a second has passed 227 | while (System.currentTimeMillis() < nextMs) { 228 | try { 229 | Thread.sleep(20); 230 | } catch (Exception e) { 231 | e.printStackTrace(); 232 | } 233 | } 234 | numLastTransactions = numTransactions; 235 | nextMs = System.currentTimeMillis() + 1000; 236 | } 237 | 238 | String collectionName = "sbtest" + Integer.toString(rand.nextInt(numCollections)+1); 239 | DBCollection coll = db.getCollection(collectionName); 240 | 241 | long timeStart = System.currentTimeMillis(); 242 | 243 | try { 244 | for (int i=1; i <= oltpPointSelects; i++) { 245 | //for i=1, oltp_point_selects do 246 | // rs = db_query("SELECT c FROM ".. table_name .." WHERE id=" .. sb_rand(1, oltp_table_size)) 247 | //end 248 | 249 | // db.sbtest8.find({_id: 554312}, {c: 1, _id: 0}) 250 | 251 | int startId = rand.nextInt(numMaxInserts)+1; 252 | 253 | BasicDBObject query = new BasicDBObject("_id", startId); 254 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 255 | 256 | DBObject myDoc = coll.findOne(query, columns); 257 | //System.out.println(myDoc); 258 | 259 | globalPointQueries.incrementAndGet(); 260 | } 261 | 262 | for (int i=1; i <= oltpSimpleRanges; i++) { 263 | //for i=1, oltp_simple_ranges do 264 | // range_start = sb_rand(1, oltp_table_size) 265 | // rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1) 266 | //end 267 | 268 | //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0}) 269 | 270 | int startId = rand.nextInt(numMaxInserts)+1; 271 | int endId = startId + oltpRangeSize - 1; 272 | 273 | BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId)); 274 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 275 | DBCursor cursor = coll.find(query, columns); 276 | try { 277 | while(cursor.hasNext()) { 278 | cursor.next(); 279 | //System.out.println(cursor.next()); 280 | } 281 | } finally { 282 | cursor.close(); 283 | } 284 | 285 | globalRangeQueries.incrementAndGet(); 286 | } 287 | 288 | for (int i=1; i <= oltpSumRanges; i++) { 289 | //for i=1, oltp_sum_ranges do 290 | // range_start = sb_rand(1, oltp_table_size) 291 | // rs = db_query("SELECT SUM(K) FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1) 292 | //end 293 | 294 | //db.sbtest8.aggregate([ {$match: {_id: {$gt: 5523412, $lt: 5523512}}}, { $group: { _id: null, total: { $sum: "$k"}} } ]) 295 | 296 | int startId = rand.nextInt(numMaxInserts)+1; 297 | int endId = startId + oltpRangeSize - 1; 298 | 299 | // create our pipeline operations, first with the $match 300 | DBObject match = new BasicDBObject("$match", new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId))); 301 | 302 | // build the $projection operation 303 | DBObject fields = new BasicDBObject("k", 1); 304 | fields.put("_id", 0); 305 | DBObject project = new BasicDBObject("$project", fields ); 306 | 307 | // Now the $group operation 308 | DBObject groupFields = new BasicDBObject( "_id", null); 309 | groupFields.put("average", new BasicDBObject( "$sum", "$k")); 310 | DBObject group = new BasicDBObject("$group", groupFields); 311 | 312 | // run aggregation 313 | AggregationOutput output = coll.aggregate( match, project, group ); 314 | 315 | //System.out.println(output.getCommandResult()); 316 | 317 | globalRangeQueries.incrementAndGet(); 318 | } 319 | 320 | for (int i=1; i <= oltpOrderRanges; i++) { 321 | //for i=1, oltp_order_ranges do 322 | // range_start = sb_rand(1, oltp_table_size) 323 | // rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c") 324 | //end 325 | 326 | //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0}).sort({c: 1}) 327 | 328 | int startId = rand.nextInt(numMaxInserts)+1; 329 | int endId = startId + oltpRangeSize - 1; 330 | 331 | BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId)); 332 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 333 | DBCursor cursor = coll.find(query, columns).sort(new BasicDBObject("c",1)); 334 | try { 335 | while(cursor.hasNext()) { 336 | cursor.next(); 337 | //System.out.println(cursor.next()); 338 | } 339 | } finally { 340 | cursor.close(); 341 | } 342 | 343 | globalRangeQueries.incrementAndGet(); 344 | } 345 | 346 | for (int i=1; i <= oltpDistinctRanges; i++) { 347 | //for i=1, oltp_distinct_ranges do 348 | // range_start = sb_rand(1, oltp_table_size) 349 | // rs = db_query("SELECT DISTINCT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c") 350 | //end 351 | 352 | //db.sbtest8.distinct("c",{_id: {$gt: 5523412, $lt: 5523512}}).sort() 353 | 354 | int startId = rand.nextInt(numMaxInserts)+1; 355 | int endId = startId + oltpRangeSize - 1; 356 | 357 | BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId)); 358 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 359 | List lstDistinct = coll.distinct("c", query); 360 | //System.out.println(lstDistinct.toString()); 361 | 362 | globalRangeQueries.incrementAndGet(); 363 | } 364 | 365 | for (int i=1; i <= oltpIndexUpdates; i++) { 366 | //for i=1, oltp_index_updates do 367 | // rs = db_query("UPDATE " .. table_name .. " SET k=k+1 WHERE id=" .. sb_rand(1, oltp_table_size)) 368 | //end 369 | 370 | //db.sbtest8.update({_id: 5523412}, {$inc: {k: 1}}, false, false) 371 | 372 | int startId = rand.nextInt(numMaxInserts)+1; 373 | 374 | WriteResult wrUpdate = coll.update(new BasicDBObject("_id", startId), new BasicDBObject("$inc", new BasicDBObject("k",1)), false, false); 375 | 376 | //System.out.println(wrUpdate.toString()); 377 | } 378 | 379 | for (int i=1; i <= oltpNonIndexUpdates; i++) { 380 | //for i=1, oltp_non_index_updates do 381 | // c_val = sb_rand_str("###########-###########-###########-###########-###########-###########-###########-###########-###########-###########") 382 | // query = "UPDATE " .. table_name .. " SET c='" .. c_val .. "' WHERE id=" .. sb_rand(1, oltp_table_size) 383 | // rs = db_query(query) 384 | // if rs then 385 | // print(query) 386 | // end 387 | //end 388 | 389 | //db.sbtest8.update({_id: 5523412}, {$set: {c: "hello there"}}, false, false) 390 | 391 | int startId = rand.nextInt(numMaxInserts)+1; 392 | 393 | String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########"); 394 | 395 | WriteResult wrUpdate = coll.update(new BasicDBObject("_id", startId), new BasicDBObject("$set", new BasicDBObject("c",cVal)), false, false); 396 | 397 | //System.out.println(wrUpdate.toString()); 398 | } 399 | 400 | for (int i=1; i <= oltpInserts; i++) { 401 | //i = sb_rand(1, oltp_table_size) 402 | //rs = db_query("DELETE FROM " .. table_name .. " WHERE id=" .. i) 403 | 404 | //db.sbtest8.remove({_id: 5523412}) 405 | 406 | int startId = rand.nextInt(numMaxInserts)+1; 407 | 408 | WriteResult wrRemove = coll.remove(new BasicDBObject("_id", startId)); 409 | 410 | //c_val = sb_rand_str([[###########-###########-###########-###########-###########-###########-###########-###########-###########-###########]]) 411 | //pad_val = sb_rand_str([[###########-###########-###########-###########-###########]]) 412 | //rs = db_query("INSERT INTO " .. table_name .. " (id, k, c, pad) VALUES " .. string.format("(%d, %d, '%s', '%s')",i, sb_rand(1, oltp_table_size) , c_val, pad_val)) 413 | 414 | BasicDBObject doc = new BasicDBObject(); 415 | doc.put("_id",startId); 416 | doc.put("k",rand.nextInt(numMaxInserts)+1); 417 | String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########"); 418 | doc.put("c",cVal); 419 | String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########"); 420 | doc.put("pad",padVal); 421 | WriteResult wrInsert = coll.insert(doc); 422 | } 423 | 424 | globalSysbenchTransactions.incrementAndGet(); 425 | numTransactions += 1; 426 | 427 | long timeEnd = System.currentTimeMillis(); 428 | globalNewBatchQty.addAndGet(1); 429 | globalNewBatchLatency.addAndGet(timeEnd-timeStart); 430 | } 431 | 432 | catch (Exception e) { 433 | globalExceptions.incrementAndGet(); 434 | // //logMe("Writer thread %d : EXCEPTION",threadNumber); 435 | // //e.printStackTrace(); 436 | } 437 | 438 | finally { 439 | 440 | } 441 | 442 | } 443 | 444 | globalWriterThreads.decrementAndGet(); 445 | } 446 | } 447 | 448 | 449 | public static String sysbenchString(java.util.Random rand, String thisMask) { 450 | StringBuilder sb = new StringBuilder(); 451 | for (int i = 0, n = thisMask.length() ; i < n ; i++) { 452 | char c = thisMask.charAt(i); 453 | if (c == '#') { 454 | sb.append(String.valueOf(rand.nextInt(10))); 455 | } else if (c == '@') { 456 | sb.append((char) (rand.nextInt(26) + 'a')); 457 | } else { 458 | sb.append(c); 459 | } 460 | } 461 | return sb.toString(); 462 | } 463 | 464 | 465 | // reporting thread, outputs information to console and file 466 | class MyReporter implements Runnable { 467 | public void run() 468 | { 469 | long t0 = System.currentTimeMillis(); 470 | long lastInserts = 0; 471 | long thisInserts = 0; 472 | long lastDeletes = 0; 473 | long thisDeletes = 0; 474 | long lastUpdates = 0; 475 | long thisUpdates = 0; 476 | long thisExceptions = 0; 477 | long lastExceptions = 0; 478 | long lastPointQueries = 0; 479 | long thisPointQueries = 0; 480 | long lastRangeQueries = 0; 481 | long thisRangeQueries = 0; 482 | long lastSysbenchTransactions = 0; 483 | long thisSysbenchTransactions = 0; 484 | long lastMs = t0; 485 | long intervalNumber = 0; 486 | long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 487 | long runEndMillis = Long.MAX_VALUE; 488 | if (runSeconds > 0) 489 | runEndMillis = t0 + (1000 * runSeconds); 490 | 491 | while ((System.currentTimeMillis() < runEndMillis) && (thisInserts < numMaxInserts)) 492 | { 493 | try { 494 | Thread.sleep(100); 495 | } catch (Exception e) { 496 | e.printStackTrace(); 497 | } 498 | 499 | long now = System.currentTimeMillis(); 500 | 501 | if ((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) 502 | { 503 | thisInserts = globalInserts.get(); 504 | thisSysbenchTransactions = globalSysbenchTransactions.get(); 505 | thisExceptions = globalExceptions.get(); 506 | 507 | intervalNumber++; 508 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 509 | 510 | long elapsed = now - t0; 511 | long thisIntervalMs = now - lastMs; 512 | long thisWriterThreads = globalWriterThreads.get(); 513 | 514 | long thisIntervalSysbenchTransactions = thisSysbenchTransactions - lastSysbenchTransactions; 515 | double thisIntervalSysbenchTransactionsPerSecond = thisIntervalSysbenchTransactions/(double)thisIntervalMs*1000.0; 516 | double thisSysbenchTransactionsPerSecond = thisSysbenchTransactions/(double)elapsed*1000.0; 517 | 518 | long thisIntervalInserts = thisInserts - lastInserts; 519 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 520 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 521 | 522 | long thisIntervalExceptions = thisExceptions - lastExceptions; 523 | 524 | long thisBatchQty = globalNewBatchQty.getAndSet(0); 525 | long thisBatchLatency = globalNewBatchLatency.getAndSet(0); 526 | double thisIntervalLatency = 0.0; 527 | 528 | if (thisBatchQty > 0) 529 | { 530 | thisIntervalLatency = (double)thisBatchLatency / (double)thisBatchQty; 531 | } 532 | 533 | logMe("%,d seconds : cum tps=%,.2f : int tps=%,.2f : cum ips=%,.2f : int ips=%,.2f : cum excp=%,d : int excpt=%,d : writers=%,d : int lat=%,.2f", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisExceptions, thisIntervalExceptions, thisWriterThreads, thisIntervalLatency); 534 | 535 | try { 536 | if (outputHeader) 537 | { 538 | writer.write("elap_secs\tcum_tps\tint_tps\tcum_ips\tint_ips\tcum_excp\tint_excp\tint_lat\n"); 539 | outputHeader = false; 540 | } 541 | 542 | String statusUpdate = ""; 543 | 544 | statusUpdate = String.format("%d\t%.2f\t%.2f\t%.2f\t%.2f\t%d\t%d\t%.2f\n", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisExceptions, thisIntervalExceptions, thisIntervalLatency); 545 | 546 | writer.write(statusUpdate); 547 | writer.flush(); 548 | } catch (IOException e) { 549 | e.printStackTrace(); 550 | } 551 | 552 | lastInserts = thisInserts; 553 | lastSysbenchTransactions = thisSysbenchTransactions; 554 | lastExceptions = thisExceptions; 555 | 556 | lastMs = now; 557 | } 558 | } 559 | 560 | // shutdown all the writers 561 | allDone = 1; 562 | } 563 | } 564 | 565 | 566 | public static void logMe(String format, Object... args) { 567 | System.out.println(Thread.currentThread() + String.format(format, args)); 568 | } 569 | } 570 | -------------------------------------------------------------------------------- /src/jmongosysbenchload.java: -------------------------------------------------------------------------------- 1 | //import com.mongodb.Mongo; 2 | import java.io.BufferedWriter; 3 | import java.io.File; 4 | import java.io.FileWriter; 5 | import java.io.IOException; 6 | import java.io.Writer; 7 | import java.util.Arrays; 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | import java.util.logging.Level; 11 | import java.util.logging.Logger; 12 | 13 | import com.mongodb.BasicDBObject; 14 | import com.mongodb.CommandResult; 15 | import com.mongodb.DB; 16 | import com.mongodb.DBCollection; 17 | import com.mongodb.DBObject; 18 | import com.mongodb.MongoClient; 19 | import com.mongodb.MongoClientOptions; 20 | import com.mongodb.MongoClientURI; 21 | import com.mongodb.MongoCredential; 22 | import com.mongodb.ServerAddress; 23 | import com.mongodb.WriteConcern; 24 | 25 | public class jmongosysbenchload { 26 | public static AtomicLong globalInserts = new AtomicLong(0); 27 | public static AtomicLong globalNewBatchQty = new AtomicLong(0); 28 | public static AtomicLong globalNewBatchLatency = new AtomicLong(0); 29 | public static AtomicLong globalWriterThreads = new AtomicLong(0); 30 | 31 | public static Writer writer = null; 32 | public static boolean outputHeader = true; 33 | 34 | public static int numCollections; 35 | public static String dbName; 36 | public static int writerThreads; 37 | public static Integer numMaxInserts; 38 | public static int documentsPerInsert; 39 | public static long insertsPerFeedback; 40 | public static long secondsPerFeedback; 41 | public static String logFileName; 42 | public static String trustStore; 43 | public static String trustStorePassword; 44 | public static String connectionString; 45 | 46 | public static int allDone = 0; 47 | 48 | public jmongosysbenchload() { 49 | } 50 | 51 | public static void main (String[] args) throws Exception { 52 | if (args.length != 11) { 53 | logMe("*** ERROR : CONFIGURATION ISSUE ***"); 54 | logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [trust store file] [trust store password] [connection string]"); 55 | System.exit(1); 56 | } 57 | 58 | numCollections = Integer.valueOf(args[0]); 59 | dbName = args[1]; 60 | writerThreads = Integer.valueOf(args[2]); 61 | numMaxInserts = Integer.valueOf(args[3]); 62 | documentsPerInsert = Integer.valueOf(args[4]); 63 | insertsPerFeedback = Long.valueOf(args[5]); 64 | secondsPerFeedback = Long.valueOf(args[6]); 65 | logFileName = args[7]; 66 | trustStore = args[8]; 67 | trustStorePassword = args[9]; 68 | connectionString = args[10]; 69 | 70 | logMe("Application Parameters"); 71 | logMe("--------------------------------------------------"); 72 | logMe(" %d collections",numCollections); 73 | logMe(" database name = %s",dbName); 74 | logMe(" %d writer thread(s)",writerThreads); 75 | logMe(" %,d documents per collection",numMaxInserts); 76 | logMe(" Documents Per Insert = %d",documentsPerInsert); 77 | logMe(" Feedback every %,d seconds(s)",secondsPerFeedback); 78 | logMe(" Feedback every %,d inserts(s)",insertsPerFeedback); 79 | logMe(" logging to file %s",logFileName); 80 | logMe(" connection string = %s",connectionString); 81 | 82 | //if (useSSL.equals("true")) { 83 | // System.setProperty("javax.net.ssl.trustStore", trustStore); 84 | // System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); 85 | // } 86 | 87 | logMe("setting mongodb driver log level to WARNING"); 88 | java.util.logging.Logger.getLogger("org.mongodb.driver").setLevel(Level.WARNING); 89 | 90 | MongoClient m = new MongoClient(new MongoClientURI(connectionString)); 91 | 92 | DB db = m.getDB(dbName); 93 | 94 | // determine server type : mongo or tokumx 95 | //DBObject checkServerCmd = new BasicDBObject(); 96 | //CommandResult commandResult = db.command("buildInfo"); 97 | 98 | // check if tokumxVersion exists, otherwise assume mongo 99 | //if (commandResult.toString().contains("tokumxVersion")) { 100 | // indexTechnology = "tokumx"; 101 | //} 102 | //else 103 | //{ 104 | // indexTechnology = "mongo"; 105 | //} 106 | 107 | //logMe(" index technology = %s",indexTechnology); 108 | 109 | logMe("--------------------------------------------------"); 110 | 111 | try { 112 | writer = new BufferedWriter(new FileWriter(new File(logFileName))); 113 | } catch (IOException e) { 114 | e.printStackTrace(); 115 | } 116 | 117 | jmongosysbenchload t = new jmongosysbenchload(); 118 | 119 | Thread reporterThread = new Thread(t.new MyReporter()); 120 | reporterThread.start(); 121 | 122 | Thread[] tWriterThreads = new Thread[writerThreads]; 123 | 124 | for (int collectionNumber = 0; collectionNumber < numCollections; collectionNumber++) { 125 | // if necessary, wait for an available slot for this loader 126 | boolean waitingForSlot = true; 127 | while (globalWriterThreads.get() >= writerThreads) { 128 | if (waitingForSlot) { 129 | logMe(" collection %d is waiting for an available loader slot",collectionNumber+1); 130 | waitingForSlot = false; 131 | } 132 | try { 133 | Thread.sleep(5000); 134 | } catch (Exception e) { 135 | e.printStackTrace(); 136 | } 137 | } 138 | 139 | // start the loader 140 | for (int i=0; i 0) { 158 | try { 159 | Thread.sleep(5000); 160 | } catch (Exception e) { 161 | e.printStackTrace(); 162 | } 163 | } 164 | 165 | // all the writers are finished 166 | allDone = 1; 167 | 168 | if (reporterThread.isAlive()) 169 | reporterThread.join(); 170 | 171 | try { 172 | if (writer != null) { 173 | writer.close(); 174 | } 175 | } catch (IOException e) { 176 | e.printStackTrace(); 177 | } 178 | 179 | m.close(); 180 | 181 | logMe("Done!"); 182 | } 183 | 184 | class MyWriter implements Runnable { 185 | int collectionNumber; 186 | int threadCount; 187 | int threadNumber; 188 | int numTables; 189 | int numMaxInserts; 190 | DB db; 191 | 192 | java.util.Random rand; 193 | 194 | MyWriter(int collectionNumber, int threadCount, int threadNumber, int numMaxInserts, DB db) { 195 | this.collectionNumber = collectionNumber; 196 | this.threadCount = threadCount; 197 | this.threadNumber = threadNumber; 198 | this.numMaxInserts = numMaxInserts; 199 | this.db = db; 200 | rand = new java.util.Random((long) collectionNumber); 201 | } 202 | public void run() { 203 | String collectionName = "sbtest" + Integer.toString(collectionNumber); 204 | 205 | /* 206 | if (indexTechnology.toLowerCase().equals("tokumx")) { 207 | DBObject cmd = new BasicDBObject(); 208 | cmd.put("create", collectionName); 209 | cmd.put("compression", compressionType); 210 | cmd.put("readPageSize", basementSize); 211 | //cmd.put("basementSize", basementSize); 212 | CommandResult result = db.command(cmd); 213 | //logMe(result.toString()); 214 | } else if (indexTechnology.toLowerCase().equals("mongo")) { 215 | // nothing special to do for a regular mongo collection 216 | 217 | } else { 218 | // unknown index technology, abort 219 | logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology); 220 | System.exit(1); 221 | } 222 | */ 223 | 224 | logMe("Writer thread %d : creating collection %s",threadNumber, collectionName); 225 | 226 | DBCollection coll = db.getCollection(collectionName); 227 | 228 | BasicDBObject idxOptions = new BasicDBObject(); 229 | idxOptions.put("background",false); 230 | 231 | //if (indexTechnology.toLowerCase().equals("tokumx")) { 232 | // idxOptions.put("compression",compressionType); 233 | // idxOptions.put("readPageSize",basementSize); 234 | //} 235 | 236 | logMe("Writer thread %d : creating collection %s secondary index",threadNumber, collectionName); 237 | 238 | coll.createIndex(new BasicDBObject("k", 1), idxOptions); 239 | 240 | long numInserts = 0; 241 | long id = 0; 242 | 243 | try { 244 | logMe("Writer thread %d : started to load collection %s",threadNumber, collectionName); 245 | 246 | BasicDBObject[] aDocs = new BasicDBObject[documentsPerInsert]; 247 | 248 | int numRounds = numMaxInserts / documentsPerInsert; 249 | 250 | for (int roundNum = 0; roundNum < numRounds; roundNum++) { 251 | for (int i = 0; i < documentsPerInsert; i++) { 252 | id++; 253 | BasicDBObject doc = new BasicDBObject(); 254 | doc.put("_id",id); 255 | doc.put("k",rand.nextInt(numMaxInserts)+1); 256 | String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########"); 257 | doc.put("c",cVal); 258 | String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########"); 259 | doc.put("pad",padVal); 260 | aDocs[i]=doc; 261 | } 262 | 263 | long timeStart = System.currentTimeMillis(); 264 | coll.insert(aDocs); 265 | long timeEnd = System.currentTimeMillis(); 266 | numInserts += documentsPerInsert; 267 | globalInserts.addAndGet(documentsPerInsert); 268 | globalNewBatchQty.addAndGet(1); 269 | globalNewBatchLatency.addAndGet(timeEnd-timeStart); 270 | } 271 | 272 | } catch (Exception e) { 273 | logMe("Writer thread %d : EXCEPTION",threadNumber); 274 | e.printStackTrace(); 275 | } 276 | 277 | globalWriterThreads.decrementAndGet(); 278 | } 279 | } 280 | 281 | 282 | public static String sysbenchString(java.util.Random rand, String thisMask) { 283 | StringBuilder sb = new StringBuilder(); 284 | for (int i = 0, n = thisMask.length() ; i < n ; i++) { 285 | char c = thisMask.charAt(i); 286 | if (c == '#') { 287 | sb.append(String.valueOf(rand.nextInt(10))); 288 | } else if (c == '@') { 289 | sb.append((char) (rand.nextInt(26) + 'a')); 290 | } else { 291 | sb.append(c); 292 | } 293 | } 294 | return sb.toString(); 295 | } 296 | 297 | 298 | // reporting thread, outputs information to console and file 299 | class MyReporter implements Runnable { 300 | public void run() 301 | { 302 | long t0 = System.currentTimeMillis(); 303 | long lastInserts = 0; 304 | long lastMs = t0; 305 | long intervalNumber = 0; 306 | long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 307 | long nextFeedbackInserts = lastInserts + insertsPerFeedback; 308 | long thisInserts = 0; 309 | 310 | while (allDone == 0) 311 | { 312 | try { 313 | Thread.sleep(100); 314 | } catch (Exception e) { 315 | e.printStackTrace(); 316 | } 317 | 318 | long now = System.currentTimeMillis(); 319 | thisInserts = globalInserts.get(); 320 | if (((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) || 321 | ((thisInserts >= nextFeedbackInserts) && (insertsPerFeedback > 0))) 322 | { 323 | intervalNumber++; 324 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 325 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 326 | 327 | long thisBatchQty = globalNewBatchQty.getAndSet(0); 328 | long thisBatchLatency = globalNewBatchLatency.getAndSet(0); 329 | double thisIntervalLatency = 0.0; 330 | 331 | if (thisBatchQty > 0) 332 | { 333 | thisIntervalLatency = (double)thisBatchLatency / (double)thisBatchQty; 334 | } 335 | 336 | long elapsed = now - t0; 337 | long thisIntervalMs = now - lastMs; 338 | 339 | long thisIntervalInserts = thisInserts - lastInserts; 340 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 341 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 342 | 343 | if (secondsPerFeedback > 0) 344 | { 345 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : int lat=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency); 346 | } else { 347 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : int lat=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency); 348 | } 349 | 350 | try { 351 | if (outputHeader) 352 | { 353 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\tint_lat\n"); 354 | outputHeader = false; 355 | } 356 | 357 | String statusUpdate = ""; 358 | 359 | if (secondsPerFeedback > 0) 360 | { 361 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency); 362 | } else { 363 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisIntervalLatency); 364 | } 365 | writer.write(statusUpdate); 366 | writer.flush(); 367 | } catch (IOException e) { 368 | e.printStackTrace(); 369 | } 370 | 371 | lastInserts = thisInserts; 372 | 373 | lastMs = now; 374 | } 375 | } 376 | 377 | // output final numbers... 378 | long now = System.currentTimeMillis(); 379 | thisInserts = globalInserts.get(); 380 | intervalNumber++; 381 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 382 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 383 | long elapsed = now - t0; 384 | long thisIntervalMs = now - lastMs; 385 | long thisIntervalInserts = thisInserts - lastInserts; 386 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 387 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 388 | if (secondsPerFeedback > 0) 389 | { 390 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 391 | } else { 392 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 393 | } 394 | try { 395 | if (outputHeader) 396 | { 397 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\n"); 398 | outputHeader = false; 399 | } 400 | String statusUpdate = ""; 401 | if (secondsPerFeedback > 0) 402 | { 403 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 404 | } else { 405 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 406 | } 407 | writer.write(statusUpdate); 408 | writer.flush(); 409 | } catch (IOException e) { 410 | e.printStackTrace(); 411 | } 412 | 413 | } 414 | } 415 | 416 | 417 | public static void logMe(String format, Object... args) { 418 | System.out.println(Thread.currentThread() + String.format(format, args)); 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /src/jmongosysbenchshardedexecute.java: -------------------------------------------------------------------------------- 1 | //import com.mongodb.Mongo; 2 | import java.io.BufferedWriter; 3 | import java.io.File; 4 | import java.io.FileWriter; 5 | import java.io.IOException; 6 | import java.io.Writer; 7 | import java.util.Arrays; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.concurrent.atomic.AtomicLong; 11 | 12 | import com.mongodb.AggregationOutput; 13 | import com.mongodb.BasicDBObject; 14 | import com.mongodb.CommandResult; 15 | import com.mongodb.DB; 16 | import com.mongodb.DBCollection; 17 | import com.mongodb.DBCursor; 18 | import com.mongodb.DBObject; 19 | import com.mongodb.MongoClient; 20 | import com.mongodb.MongoClientOptions; 21 | import com.mongodb.MongoClientURI; 22 | import com.mongodb.MongoCredential; 23 | import com.mongodb.ServerAddress; 24 | import com.mongodb.WriteConcern; 25 | import com.mongodb.WriteResult; 26 | 27 | public class jmongosysbenchshardedexecute { 28 | public static AtomicLong globalInserts = new AtomicLong(0); 29 | public static AtomicLong globalDeletes = new AtomicLong(0); 30 | public static AtomicLong globalUpdates = new AtomicLong(0); 31 | public static AtomicLong globalPointQueries = new AtomicLong(0); 32 | public static AtomicLong globalRangeQueries = new AtomicLong(0); 33 | public static AtomicLong globalSysbenchTransactions = new AtomicLong(0); 34 | public static AtomicLong globalWriterThreads = new AtomicLong(0); 35 | public static AtomicLong globalExceptions = new AtomicLong(0); 36 | 37 | public static Writer writer = null; 38 | public static boolean outputHeader = true; 39 | 40 | public static int numCollections; 41 | public static String dbName; 42 | public static int writerThreads; 43 | public static Integer numMaxInserts; 44 | public static long secondsPerFeedback; 45 | public static String logFileName; 46 | public static String indexTechnology; 47 | public static String autoCommit; 48 | public static int runSeconds; 49 | public static String myWriteConcern; 50 | public static Integer maxTPS; 51 | public static Integer maxThreadTPS; 52 | public static String serverName; 53 | public static int serverPort; 54 | public static String userName; 55 | public static String passWord; 56 | public static String readPreference; 57 | public static String trustStore; 58 | public static String trustStorePassword; 59 | public static String useSSL; 60 | public static int maxShardKey; 61 | public static int docsPerShard; 62 | 63 | public static int oltpRangeSize; 64 | public static int oltpPointSelects; 65 | public static int oltpSimpleRanges; 66 | public static int oltpSumRanges; 67 | public static int oltpOrderRanges; 68 | public static int oltpDistinctRanges; 69 | public static int oltpIndexUpdates; 70 | public static int oltpNonIndexUpdates; 71 | public static int oltpInserts; 72 | 73 | public static boolean bIsTokuMX = false; 74 | 75 | public static int allDone = 0; 76 | 77 | public static long rngSeed = 0; 78 | 79 | public jmongosysbenchshardedexecute() { 80 | } 81 | 82 | public static void main (String[] args) throws Exception { 83 | if (args.length != 30) { 84 | logMe("*** ERROR : CONFIGURATION ISSUE ***"); 85 | logMe("jsysbenchshardedexecute [number of collections] [database name] [number of writer threads] [documents per collection] [seconds feedback] "+ 86 | "[log file name] [auto commit Y/N] [runtime (seconds)] [range size] [point selects] "+ 87 | "[simple ranges] [sum ranges] [order ranges] [distinct ranges] [index updates] [non index updates] [inserts] [writeconcern] "+ 88 | "[max tps] [server] [port] [seed] [username] [password] [read preference] [trust store] [trust store password] [use ssl] [max shard key] [num docs per shard]"); 89 | System.exit(1); 90 | } 91 | 92 | numCollections = Integer.valueOf(args[0]); 93 | dbName = args[1]; 94 | writerThreads = Integer.valueOf(args[2]); 95 | numMaxInserts = Integer.valueOf(args[3]); 96 | secondsPerFeedback = Long.valueOf(args[4]); 97 | logFileName = args[5]; 98 | autoCommit = args[6]; 99 | runSeconds = Integer.valueOf(args[7]); 100 | oltpRangeSize = Integer.valueOf(args[8]); 101 | oltpPointSelects = Integer.valueOf(args[9]); 102 | oltpSimpleRanges = Integer.valueOf(args[10]); 103 | oltpSumRanges = Integer.valueOf(args[11]); 104 | oltpOrderRanges = Integer.valueOf(args[12]); 105 | oltpDistinctRanges = Integer.valueOf(args[13]); 106 | oltpIndexUpdates = Integer.valueOf(args[14]); 107 | oltpNonIndexUpdates = Integer.valueOf(args[15]); 108 | oltpInserts = Integer.valueOf(args[16]); 109 | myWriteConcern = args[17]; 110 | maxTPS = Integer.valueOf(args[18]); 111 | serverName = args[19]; 112 | serverPort = Integer.valueOf(args[20]); 113 | rngSeed = Long.valueOf(args[21]); 114 | userName = args[22]; 115 | passWord = args[23]; 116 | readPreference = args[24]; 117 | trustStore = args[25]; 118 | trustStorePassword = args[26]; 119 | useSSL = args[27].toLowerCase(); 120 | maxShardKey = Integer.valueOf(args[28]); 121 | docsPerShard = Integer.valueOf(args[29]); 122 | 123 | // override 124 | numMaxInserts = maxShardKey * docsPerShard; 125 | 126 | maxThreadTPS = (maxTPS / writerThreads) + 1; 127 | 128 | WriteConcern myWC = new WriteConcern(); 129 | if (myWriteConcern.toLowerCase().equals("acknowledged")) { 130 | myWC = WriteConcern.ACKNOWLEDGED; 131 | } 132 | else if ((myWriteConcern.toLowerCase().equals("unacknowledged"))) { 133 | myWC = WriteConcern.UNACKNOWLEDGED; 134 | } 135 | else if ((myWriteConcern.toLowerCase().equals("w1"))) { 136 | myWC = WriteConcern.W1; 137 | } 138 | else if ((myWriteConcern.toLowerCase().equals("w2"))) { 139 | myWC = WriteConcern.W2; 140 | } 141 | else if ((myWriteConcern.toLowerCase().equals("w3"))) { 142 | myWC = WriteConcern.W3; 143 | } 144 | else { 145 | logMe("*** ERROR : WRITE CONCERN ISSUE ***"); 146 | logMe(" write concern %s is not supported",myWriteConcern); 147 | System.exit(1); 148 | } 149 | 150 | logMe("Application Parameters"); 151 | logMe("-------------------------------------------------------------------------------------------------"); 152 | logMe(" collections = %d",numCollections); 153 | logMe(" database name = %s",dbName); 154 | logMe(" writer threads = %d",writerThreads); 155 | logMe(" documents per collection = %,d",numMaxInserts); 156 | logMe(" feedback seconds = %,d",secondsPerFeedback); 157 | logMe(" log file = %s",logFileName); 158 | logMe(" auto commit = %s",autoCommit); 159 | logMe(" run seconds = %d",runSeconds); 160 | logMe(" oltp range size = %d",oltpRangeSize); 161 | logMe(" oltp point selects = %d",oltpPointSelects); 162 | logMe(" oltp simple ranges = %d",oltpSimpleRanges); 163 | logMe(" oltp sum ranges = %d",oltpSumRanges); 164 | logMe(" oltp order ranges = %d",oltpOrderRanges); 165 | logMe(" oltp distinct ranges = %d",oltpDistinctRanges); 166 | logMe(" oltp index updates = %d",oltpIndexUpdates); 167 | logMe(" oltp non index updates = %d",oltpNonIndexUpdates); 168 | logMe(" oltp inserts = %d",oltpInserts); 169 | logMe(" write concern = %s",myWriteConcern); 170 | logMe(" maximum tps (global) = %d",maxTPS); 171 | logMe(" maximum tps (per thread) = %d",maxThreadTPS); 172 | logMe(" Server:Port = %s:%d",serverName,serverPort); 173 | logMe(" seed = %d",rngSeed); 174 | logMe(" userName = %s",userName); 175 | logMe(" read preference = %s",readPreference); 176 | logMe(" use SSL = %s",useSSL); 177 | logMe(" max shard key = %s",maxShardKey); 178 | logMe(" num docs per shard = %s",docsPerShard); 179 | 180 | /* 181 | MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build(); 182 | ServerAddress srvrAdd = new ServerAddress(serverName,serverPort); 183 | 184 | // Credential login is optional. 185 | MongoClient m; 186 | if (userName.isEmpty() || userName.equalsIgnoreCase("none")) { 187 | m = new MongoClient(srvrAdd); 188 | } else { 189 | MongoCredential credential = MongoCredential.createCredential(userName, dbName, passWord.toCharArray()); 190 | m = new MongoClient(srvrAdd, Arrays.asList(credential)); 191 | } 192 | */ 193 | 194 | //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&replicaSet=rs0&readpreference=%s&maxPoolSize=4096"; 195 | //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=4096&serverSelectionTimeoutMS=30000&replicaSet=rs0"; 196 | String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=4096"; 197 | String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference); 198 | //logMe(" connection string = %s",connectionString); 199 | 200 | //if (useSSL.equals("true")) { 201 | // System.setProperty("javax.net.ssl.trustStore", trustStore); 202 | // System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); 203 | // } 204 | 205 | //MongoClient m = new MongoClient(new MongoClientURI(connectionString),clientOptions); 206 | MongoClient m = new MongoClient(new MongoClientURI(connectionString)); 207 | 208 | //logMe("mongoOptions | " + m.getMongoOptions().toString()); 209 | //logMe("mongoWriteConcern | " + m.getWriteConcern().toString()); 210 | 211 | DB db = m.getDB(dbName); 212 | 213 | // determine server type : mongo or tokumx 214 | DBObject checkServerCmd = new BasicDBObject(); 215 | CommandResult commandResult = db.command("buildInfo"); 216 | 217 | // check if tokumxVersion exists, otherwise assume mongo 218 | if (commandResult.toString().contains("tokumxVersion")) { 219 | indexTechnology = "tokumx"; 220 | } 221 | else 222 | { 223 | indexTechnology = "mongo"; 224 | } 225 | 226 | logMe(" index technology = %s",indexTechnology); 227 | logMe("-------------------------------------------------------------------------------------------------"); 228 | 229 | try { 230 | writer = new BufferedWriter(new FileWriter(new File(logFileName))); 231 | } catch (IOException e) { 232 | e.printStackTrace(); 233 | } 234 | 235 | if ((!indexTechnology.toLowerCase().equals("tokumx")) && (!indexTechnology.toLowerCase().equals("mongo"))) { 236 | // unknown index technology, abort 237 | logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology); 238 | System.exit(1); 239 | } 240 | 241 | if (indexTechnology.toLowerCase().equals("tokumx")) { 242 | bIsTokuMX = true; 243 | } 244 | 245 | jmongosysbenchshardedexecute t = new jmongosysbenchshardedexecute(); 246 | 247 | Thread[] tWriterThreads = new Thread[writerThreads]; 248 | 249 | for (int i=0; i= maxThreadTPS) { 313 | // pause until a second has passed 314 | while (System.currentTimeMillis() < nextMs) { 315 | try { 316 | Thread.sleep(20); 317 | } catch (Exception e) { 318 | e.printStackTrace(); 319 | } 320 | } 321 | numLastTransactions = numTransactions; 322 | nextMs = System.currentTimeMillis() + 1000; 323 | } 324 | 325 | String collectionName = "sbtest" + Integer.toString(rand.nextInt(numCollections)+1); 326 | DBCollection coll = db.getCollection(collectionName); 327 | 328 | try { 329 | // pick shard key 330 | int thisShardKey = rand.nextInt(maxShardKey); 331 | 332 | for (int i=1; i <= oltpPointSelects; i++) { 333 | //for i=1, oltp_point_selects do 334 | // rs = db_query("SELECT c FROM ".. table_name .." WHERE id=" .. sb_rand(1, oltp_table_size)) 335 | //end 336 | 337 | // db.sbtest8.find({_id: 554312}, {c: 1, _id: 0}) 338 | 339 | int startId = rand.nextInt(docsPerShard)+1; 340 | 341 | //BasicDBObject query = new BasicDBObject("_id", startId); 342 | BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId); 343 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 344 | 345 | DBObject myDoc = coll.findOne(query, columns); 346 | //System.out.println(myDoc); 347 | 348 | globalPointQueries.incrementAndGet(); 349 | } 350 | 351 | for (int i=1; i <= oltpSimpleRanges; i++) { 352 | //for i=1, oltp_simple_ranges do 353 | // range_start = sb_rand(1, oltp_table_size) 354 | // rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1) 355 | //end 356 | 357 | //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0}) 358 | 359 | int startId = rand.nextInt(docsPerShard)+1; 360 | int endId = startId + oltpRangeSize - 1; 361 | 362 | //BasicDBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", startId).append("$lte", endId)); 363 | List queryCriteria = new ArrayList(); 364 | queryCriteria.add(new BasicDBObject("shardKey", thisShardKey)); 365 | queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId))); 366 | 367 | BasicDBObject query = new BasicDBObject("$and", queryCriteria); 368 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 369 | DBCursor cursor = coll.find(query, columns); 370 | try { 371 | while(cursor.hasNext()) { 372 | cursor.next(); 373 | //System.out.println(cursor.next()); 374 | } 375 | } finally { 376 | cursor.close(); 377 | } 378 | 379 | globalRangeQueries.incrementAndGet(); 380 | } 381 | 382 | for (int i=1; i <= oltpSumRanges; i++) { 383 | //for i=1, oltp_sum_ranges do 384 | // range_start = sb_rand(1, oltp_table_size) 385 | // rs = db_query("SELECT SUM(K) FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1) 386 | //end 387 | 388 | //db.sbtest8.aggregate([ {$match: {_id: {$gt: 5523412, $lt: 5523512}}}, { $group: { _id: null, total: { $sum: "$k"}} } ]) 389 | 390 | int startId = rand.nextInt(docsPerShard)+1; 391 | int endId = startId + oltpRangeSize - 1; 392 | 393 | // create our pipeline operations, first with the $match 394 | List queryCriteria = new ArrayList(); 395 | queryCriteria.add(new BasicDBObject("shardKey", thisShardKey)); 396 | queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId))); 397 | DBObject match = new BasicDBObject("$match", new BasicDBObject("$and", queryCriteria)); 398 | 399 | // build the $projection operation 400 | DBObject fields = new BasicDBObject("k", 1); 401 | fields.put("_id", 0); 402 | DBObject project = new BasicDBObject("$project", fields ); 403 | 404 | // Now the $group operation 405 | DBObject groupFields = new BasicDBObject( "_id", null); 406 | groupFields.put("average", new BasicDBObject( "$sum", "$k")); 407 | DBObject group = new BasicDBObject("$group", groupFields); 408 | 409 | // run aggregation 410 | AggregationOutput output = coll.aggregate( match, project, group ); 411 | 412 | //System.out.println(output.getCommandResult()); 413 | 414 | globalRangeQueries.incrementAndGet(); 415 | } 416 | 417 | for (int i=1; i <= oltpOrderRanges; i++) { 418 | //for i=1, oltp_order_ranges do 419 | // range_start = sb_rand(1, oltp_table_size) 420 | // rs = db_query("SELECT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c") 421 | //end 422 | 423 | //db.sbtest8.find({_id: {$gte: 5523412, $lte: 5523512}}, {c: 1, _id: 0}).sort({c: 1}) 424 | 425 | int startId = rand.nextInt(docsPerShard)+1; 426 | int endId = startId + oltpRangeSize - 1; 427 | 428 | List queryCriteria = new ArrayList(); 429 | queryCriteria.add(new BasicDBObject("shardKey", thisShardKey)); 430 | queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId))); 431 | 432 | BasicDBObject query = new BasicDBObject("$and", queryCriteria); 433 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 434 | DBCursor cursor = coll.find(query, columns).sort(new BasicDBObject("c",1)); 435 | try { 436 | while(cursor.hasNext()) { 437 | cursor.next(); 438 | //System.out.println(cursor.next()); 439 | } 440 | } finally { 441 | cursor.close(); 442 | } 443 | 444 | globalRangeQueries.incrementAndGet(); 445 | } 446 | 447 | for (int i=1; i <= oltpDistinctRanges; i++) { 448 | //for i=1, oltp_distinct_ranges do 449 | // range_start = sb_rand(1, oltp_table_size) 450 | // rs = db_query("SELECT DISTINCT c FROM ".. table_name .." WHERE id BETWEEN " .. range_start .. " AND " .. range_start .. "+" .. oltp_range_size - 1 .. " ORDER BY c") 451 | //end 452 | 453 | //db.sbtest8.distinct("c",{_id: {$gt: 5523412, $lt: 5523512}}).sort() 454 | 455 | int startId = rand.nextInt(docsPerShard)+1; 456 | int endId = startId + oltpRangeSize - 1; 457 | 458 | List queryCriteria = new ArrayList(); 459 | queryCriteria.add(new BasicDBObject("shardKey", thisShardKey)); 460 | queryCriteria.add(new BasicDBObject("orderId", new BasicDBObject("$gte", startId).append("$lte", endId))); 461 | 462 | BasicDBObject query = new BasicDBObject("$and", queryCriteria); 463 | BasicDBObject columns = new BasicDBObject("c", 1).append("_id", 0); 464 | List lstDistinct = coll.distinct("c", query); 465 | //System.out.println(lstDistinct.toString()); 466 | 467 | globalRangeQueries.incrementAndGet(); 468 | } 469 | 470 | //'A {multi:false} update on a sharded collection must either contain an exact match on _id or must target a single shard but this update targeted _id (and have the collection default collation) or must target a single shard (and have the simple collation), but this update targeted multiple shards.' 471 | for (int i=1; i <= oltpIndexUpdates; i++) { 472 | //for i=1, oltp_index_updates do 473 | // rs = db_query("UPDATE " .. table_name .. " SET k=k+1 WHERE id=" .. sb_rand(1, oltp_table_size)) 474 | //end 475 | 476 | //db.sbtest8.update({_id: 5523412}, {$inc: {k: 1}}, false, false) 477 | 478 | int startId = rand.nextInt(docsPerShard)+1; 479 | 480 | BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId); 481 | //System.out.println(query.toString()); 482 | 483 | WriteResult wrUpdate = coll.update(query, new BasicDBObject("$inc", new BasicDBObject("k",1)), false, false); 484 | 485 | //System.out.println(wrUpdate.toString()); 486 | } 487 | 488 | for (int i=1; i <= oltpNonIndexUpdates; i++) { 489 | //for i=1, oltp_non_index_updates do 490 | // c_val = sb_rand_str("###########-###########-###########-###########-###########-###########-###########-###########-###########-###########") 491 | // query = "UPDATE " .. table_name .. " SET c='" .. c_val .. "' WHERE id=" .. sb_rand(1, oltp_table_size) 492 | // rs = db_query(query) 493 | // if rs then 494 | // print(query) 495 | // end 496 | //end 497 | 498 | //db.sbtest8.update({_id: 5523412}, {$set: {c: "hello there"}}, false, false) 499 | 500 | int startId = rand.nextInt(docsPerShard)+1; 501 | 502 | BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId); 503 | 504 | String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########"); 505 | 506 | WriteResult wrUpdate = coll.update(query, new BasicDBObject("$set", new BasicDBObject("c",cVal)), false, false); 507 | 508 | //System.out.println(wrUpdate.toString()); 509 | } 510 | 511 | for (int i=1; i <= oltpInserts; i++) { 512 | //i = sb_rand(1, oltp_table_size) 513 | //rs = db_query("DELETE FROM " .. table_name .. " WHERE id=" .. i) 514 | 515 | //db.sbtest8.remove({_id: 5523412}) 516 | 517 | int startId = rand.nextInt(docsPerShard)+1; 518 | 519 | BasicDBObject query = new BasicDBObject("shardKey", thisShardKey).append("orderId", startId); 520 | 521 | DBObject priorDoc = coll.findAndRemove(query); 522 | 523 | while (priorDoc == null) { 524 | //WriteResult wrRemove = coll.remove(new BasicDBObject("_id", startId)); 525 | priorDoc = coll.findAndRemove(query); 526 | //System.out.println(startId); 527 | } 528 | 529 | //c_val = sb_rand_str([[###########-###########-###########-###########-###########-###########-###########-###########-###########-###########]]) 530 | //pad_val = sb_rand_str([[###########-###########-###########-###########-###########]]) 531 | //rs = db_query("INSERT INTO " .. table_name .. " (id, k, c, pad) VALUES " .. string.format("(%d, %d, '%s', '%s')",i, sb_rand(1, oltp_table_size) , c_val, pad_val)) 532 | 533 | BasicDBObject doc = new BasicDBObject(); 534 | doc.put("_id",priorDoc.get("_id")); 535 | doc.put("shardKey",thisShardKey); 536 | doc.put("orderId",startId); 537 | doc.put("k",rand.nextInt(docsPerShard)+1); 538 | String cVal = sysbenchString(rand, "###########-###########-###########-###########-###########-###########-###########-###########-###########-###########"); 539 | doc.put("c",cVal); 540 | String padVal = sysbenchString(rand, "###########-###########-###########-###########-###########"); 541 | doc.put("pad",padVal); 542 | WriteResult wrInsert = coll.insert(doc); 543 | } 544 | 545 | globalSysbenchTransactions.incrementAndGet(); 546 | numTransactions += 1; 547 | } 548 | 549 | catch (Exception e) { 550 | globalExceptions.incrementAndGet(); 551 | //logMe("Writer thread %d : EXCEPTION",threadNumber); 552 | //e.printStackTrace(); 553 | try { 554 | Thread.sleep(1000); 555 | } catch (Exception eSleep) { 556 | eSleep.printStackTrace(); 557 | } 558 | } 559 | 560 | finally { 561 | 562 | } 563 | 564 | } 565 | 566 | globalWriterThreads.decrementAndGet(); 567 | } 568 | } 569 | 570 | 571 | public static String sysbenchString(java.util.Random rand, String thisMask) { 572 | StringBuilder sb = new StringBuilder(); 573 | for (int i = 0, n = thisMask.length() ; i < n ; i++) { 574 | char c = thisMask.charAt(i); 575 | if (c == '#') { 576 | sb.append(String.valueOf(rand.nextInt(10))); 577 | } else if (c == '@') { 578 | sb.append((char) (rand.nextInt(26) + 'a')); 579 | } else { 580 | sb.append(c); 581 | } 582 | } 583 | return sb.toString(); 584 | } 585 | 586 | 587 | // reporting thread, outputs information to console and file 588 | class MyReporter implements Runnable { 589 | public void run() 590 | { 591 | long t0 = System.currentTimeMillis(); 592 | long lastInserts = 0; 593 | long thisInserts = 0; 594 | long lastDeletes = 0; 595 | long thisDeletes = 0; 596 | long lastUpdates = 0; 597 | long thisUpdates = 0; 598 | long thisExceptions = 0; 599 | long lastExceptions = 0; 600 | long lastPointQueries = 0; 601 | long thisPointQueries = 0; 602 | long lastRangeQueries = 0; 603 | long thisRangeQueries = 0; 604 | long lastSysbenchTransactions = 0; 605 | long thisSysbenchTransactions = 0; 606 | long lastMs = t0; 607 | long intervalNumber = 0; 608 | long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 609 | long runEndMillis = Long.MAX_VALUE; 610 | if (runSeconds > 0) 611 | runEndMillis = t0 + (1000 * runSeconds); 612 | 613 | // modified the following, only running time based for now 614 | //while ((System.currentTimeMillis() < runEndMillis) && (thisInserts < numMaxInserts)) 615 | while (System.currentTimeMillis() < runEndMillis) 616 | { 617 | try { 618 | Thread.sleep(100); 619 | } catch (Exception e) { 620 | e.printStackTrace(); 621 | } 622 | 623 | long now = System.currentTimeMillis(); 624 | 625 | if ((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) 626 | { 627 | thisInserts = globalInserts.get(); 628 | thisSysbenchTransactions = globalSysbenchTransactions.get(); 629 | thisExceptions = globalExceptions.get(); 630 | 631 | intervalNumber++; 632 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 633 | 634 | long elapsed = now - t0; 635 | long thisIntervalMs = now - lastMs; 636 | long thisWriterThreads = globalWriterThreads.get(); 637 | 638 | long thisIntervalSysbenchTransactions = thisSysbenchTransactions - lastSysbenchTransactions; 639 | double thisIntervalSysbenchTransactionsPerSecond = thisIntervalSysbenchTransactions/(double)thisIntervalMs*1000.0; 640 | double thisSysbenchTransactionsPerSecond = thisSysbenchTransactions/(double)elapsed*1000.0; 641 | 642 | long thisIntervalInserts = thisInserts - lastInserts; 643 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 644 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 645 | 646 | long thisIntervalExceptions = thisExceptions - lastExceptions; 647 | 648 | logMe("%,d seconds : cum tps=%,.2f : int tps=%,.2f : cum excpt=%,d : int excpt=%,d : writers=%,d", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisExceptions, thisIntervalExceptions, thisWriterThreads); 649 | 650 | try { 651 | if (outputHeader) 652 | { 653 | writer.write("elap_secs\tcum_tps\tint_tps\tcum_excp\tint_excp\n"); 654 | outputHeader = false; 655 | } 656 | 657 | String statusUpdate = ""; 658 | 659 | statusUpdate = String.format("%d\t%.2f\t%.2f\t%d\t%d\n", elapsed / 1000l, thisSysbenchTransactionsPerSecond, thisIntervalSysbenchTransactionsPerSecond, thisExceptions, thisIntervalExceptions); 660 | 661 | writer.write(statusUpdate); 662 | writer.flush(); 663 | } catch (IOException e) { 664 | e.printStackTrace(); 665 | } 666 | 667 | lastInserts = thisInserts; 668 | lastSysbenchTransactions = thisSysbenchTransactions; 669 | lastExceptions = thisExceptions; 670 | 671 | lastMs = now; 672 | } 673 | } 674 | 675 | // shutdown all the writers 676 | allDone = 1; 677 | } 678 | } 679 | 680 | 681 | public static void logMe(String format, Object... args) { 682 | System.out.println(Thread.currentThread() + String.format(format, args)); 683 | } 684 | } 685 | -------------------------------------------------------------------------------- /src/jmongosysbenchshardedload.java: -------------------------------------------------------------------------------- 1 | //import com.mongodb.Mongo; 2 | import java.io.BufferedWriter; 3 | import java.io.File; 4 | import java.io.FileWriter; 5 | import java.io.IOException; 6 | import java.io.Writer; 7 | import java.util.Arrays; 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | import com.mongodb.BasicDBObject; 11 | import com.mongodb.CommandResult; 12 | import com.mongodb.DB; 13 | import com.mongodb.DBCollection; 14 | import com.mongodb.DBObject; 15 | import com.mongodb.MongoClient; 16 | import com.mongodb.MongoClientOptions; 17 | import com.mongodb.MongoClientURI; 18 | import com.mongodb.MongoCredential; 19 | import com.mongodb.ServerAddress; 20 | import com.mongodb.WriteConcern; 21 | 22 | public class jmongosysbenchshardedload { 23 | public static AtomicLong globalInserts = new AtomicLong(0); 24 | public static AtomicLong globalWriterThreads = new AtomicLong(0); 25 | 26 | public static Writer writer = null; 27 | public static boolean outputHeader = true; 28 | 29 | public static int numCollections; 30 | public static String dbName; 31 | public static int writerThreads; 32 | public static Integer numMaxInserts; 33 | public static int documentsPerInsert; 34 | public static long insertsPerFeedback; 35 | public static long secondsPerFeedback; 36 | public static String compressionType; 37 | public static int basementSize; 38 | public static String logFileName; 39 | public static String indexTechnology; 40 | public static String myWriteConcern; 41 | public static String serverName; 42 | public static int serverPort; 43 | public static String userName; 44 | public static String passWord; 45 | public static String trustStore; 46 | public static String trustStorePassword; 47 | public static String useSSL; 48 | public static int maxShardKey; 49 | public static int docsPerShard; 50 | 51 | public static int allDone = 0; 52 | 53 | public jmongosysbenchshardedload() { 54 | } 55 | 56 | public static void main (String[] args) throws Exception { 57 | if (args.length != 20) { 58 | logMe("*** ERROR : CONFIGURATION ISSUE ***"); 59 | logMe("jsysbenchload [number of collections] [database name] [number of writer threads] [documents per collection] [documents per insert] [inserts feedback] [seconds feedback] [log file name] [compression type] [basement node size (bytes)] [writeconcern] [server] [port] [username] [password] [trust store file] [trust store password] [use ssl] [max shard key] [docs per shard]"); 60 | System.exit(1); 61 | } 62 | 63 | numCollections = Integer.valueOf(args[0]); 64 | dbName = args[1]; 65 | writerThreads = Integer.valueOf(args[2]); 66 | numMaxInserts = Integer.valueOf(args[3]); 67 | documentsPerInsert = Integer.valueOf(args[4]); 68 | insertsPerFeedback = Long.valueOf(args[5]); 69 | secondsPerFeedback = Long.valueOf(args[6]); 70 | logFileName = args[7]; 71 | compressionType = args[8]; 72 | basementSize = Integer.valueOf(args[9]); 73 | myWriteConcern = args[10]; 74 | serverName = args[11]; 75 | serverPort = Integer.valueOf(args[12]); 76 | userName = args[13]; 77 | passWord = args[14]; 78 | trustStore = args[15]; 79 | trustStorePassword = args[16]; 80 | useSSL = args[17].toLowerCase(); 81 | maxShardKey = Integer.valueOf(args[18]); 82 | docsPerShard = Integer.valueOf(args[19]); 83 | 84 | // override numMaxInserts 85 | numMaxInserts = maxShardKey * docsPerShard; 86 | 87 | WriteConcern myWC = new WriteConcern(); 88 | if (myWriteConcern.toLowerCase().equals("acknowledged")) { 89 | myWC = WriteConcern.ACKNOWLEDGED; 90 | } 91 | else if ((myWriteConcern.toLowerCase().equals("unacknowledged"))) { 92 | myWC = WriteConcern.UNACKNOWLEDGED; 93 | } 94 | else if ((myWriteConcern.toLowerCase().equals("w1"))) { 95 | myWC = WriteConcern.W1; 96 | } 97 | else if ((myWriteConcern.toLowerCase().equals("w2"))) { 98 | myWC = WriteConcern.W2; 99 | } 100 | else if ((myWriteConcern.toLowerCase().equals("w3"))) { 101 | myWC = WriteConcern.W3; 102 | } 103 | else { 104 | logMe("*** ERROR : WRITE CONCERN ISSUE ***"); 105 | logMe(" write concern %s is not supported",myWriteConcern); 106 | System.exit(1); 107 | } 108 | 109 | logMe("Application Parameters"); 110 | logMe("--------------------------------------------------"); 111 | logMe(" %d collections",numCollections); 112 | logMe(" database name = %s",dbName); 113 | logMe(" %d writer thread(s)",writerThreads); 114 | logMe(" %,d documents per collection",numMaxInserts); 115 | logMe(" Documents Per Insert = %d",documentsPerInsert); 116 | logMe(" Feedback every %,d seconds(s)",secondsPerFeedback); 117 | logMe(" Feedback every %,d inserts(s)",insertsPerFeedback); 118 | logMe(" logging to file %s",logFileName); 119 | logMe(" write concern = %s",myWriteConcern); 120 | logMe(" Server:Port = %s:%d",serverName,serverPort); 121 | logMe(" Username = %s",userName); 122 | logMe(" Use SSL = %s",useSSL); 123 | logMe(" Maximum Shard Key = %s",maxShardKey); 124 | logMe(" Documents Per Shard = %s",docsPerShard); 125 | 126 | /* 127 | MongoClientOptions clientOptions = new MongoClientOptions.Builder().connectionsPerHost(2048).socketTimeout(60000).writeConcern(myWC).build(); 128 | ServerAddress srvrAdd = new ServerAddress(serverName,serverPort); 129 | 130 | // Credential login is optional. 131 | MongoClient m; 132 | if (userName.isEmpty() || userName.equalsIgnoreCase("none")) { 133 | m = new MongoClient(srvrAdd); 134 | } else { 135 | MongoCredential credential = MongoCredential.createCredential(userName, dbName, passWord.toCharArray()); 136 | m = new MongoClient(srvrAdd, Arrays.asList(credential)); 137 | } 138 | */ 139 | 140 | //String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&replicaSet=rs0&readpreference=%s"; 141 | String template = "mongodb://%s:%s@%s:%s/admin?ssl=%s&readpreference=%s&maxPoolSize=2000"; 142 | //String readPreference = "secondaryPreferred"; 143 | String readPreference = "primary"; 144 | String connectionString = String.format(template, userName, passWord, serverName, serverPort, useSSL, readPreference); 145 | logMe(" connection string = %s",connectionString); 146 | 147 | //if (useSSL.equals("true")) { 148 | // System.setProperty("javax.net.ssl.trustStore", trustStore); 149 | // System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); 150 | // } 151 | 152 | MongoClient m = new MongoClient(new MongoClientURI(connectionString)); 153 | 154 | //logMe("mongoOptions | " + m.getMongoOptions().toString()); 155 | //logMe("mongoWriteConcern | " + m.getWriteConcern().toString()); 156 | 157 | DB db = m.getDB(dbName); 158 | 159 | // determine server type : mongo or tokumx 160 | DBObject checkServerCmd = new BasicDBObject(); 161 | CommandResult commandResult = db.command("buildInfo"); 162 | 163 | // check if tokumxVersion exists, otherwise assume mongo 164 | if (commandResult.toString().contains("tokumxVersion")) { 165 | indexTechnology = "tokumx"; 166 | } 167 | else 168 | { 169 | indexTechnology = "mongo"; 170 | } 171 | 172 | logMe(" index technology = %s",indexTechnology); 173 | 174 | if (indexTechnology.toLowerCase().equals("tokumx")) { 175 | logMe(" + compression type = %s",compressionType); 176 | logMe(" + basement node size (bytes) = %d",basementSize); 177 | } 178 | 179 | logMe("--------------------------------------------------"); 180 | 181 | try { 182 | writer = new BufferedWriter(new FileWriter(new File(logFileName))); 183 | } catch (IOException e) { 184 | e.printStackTrace(); 185 | } 186 | 187 | if ((!indexTechnology.toLowerCase().equals("tokumx")) && (!indexTechnology.toLowerCase().equals("mongo"))) { 188 | // unknown index technology, abort 189 | logMe(" *** Unknown Indexing Technology %s, shutting down",indexTechnology); 190 | System.exit(1); 191 | } 192 | 193 | jmongosysbenchshardedload t = new jmongosysbenchshardedload(); 194 | 195 | Thread reporterThread = new Thread(t.new MyReporter()); 196 | reporterThread.start(); 197 | 198 | Thread[] tWriterThreads = new Thread[writerThreads]; 199 | 200 | for (int collectionNumber = 0; collectionNumber < numCollections; collectionNumber++) { 201 | // if necessary, wait for an available slot for this loader 202 | boolean waitingForSlot = true; 203 | while (globalWriterThreads.get() >= writerThreads) { 204 | if (waitingForSlot) { 205 | logMe(" collection %d is waiting for an available loader slot",collectionNumber+1); 206 | waitingForSlot = false; 207 | } 208 | try { 209 | Thread.sleep(5000); 210 | } catch (Exception e) { 211 | e.printStackTrace(); 212 | } 213 | } 214 | 215 | // start the loader 216 | for (int i=0; i 0) { 234 | try { 235 | Thread.sleep(5000); 236 | } catch (Exception e) { 237 | e.printStackTrace(); 238 | } 239 | } 240 | 241 | // wait for writer threads to terminate 242 | //for (int i=0; i nextFeedbackMillis) && (secondsPerFeedback > 0)) || 403 | ((thisInserts >= nextFeedbackInserts) && (insertsPerFeedback > 0))) 404 | { 405 | intervalNumber++; 406 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 407 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 408 | 409 | long elapsed = now - t0; 410 | long thisIntervalMs = now - lastMs; 411 | 412 | long thisIntervalInserts = thisInserts - lastInserts; 413 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 414 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 415 | 416 | if (secondsPerFeedback > 0) 417 | { 418 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 419 | } else { 420 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 421 | } 422 | 423 | try { 424 | if (outputHeader) 425 | { 426 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\n"); 427 | outputHeader = false; 428 | } 429 | 430 | String statusUpdate = ""; 431 | 432 | if (secondsPerFeedback > 0) 433 | { 434 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 435 | } else { 436 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 437 | } 438 | writer.write(statusUpdate); 439 | writer.flush(); 440 | } catch (IOException e) { 441 | e.printStackTrace(); 442 | } 443 | 444 | lastInserts = thisInserts; 445 | 446 | lastMs = now; 447 | } 448 | } 449 | 450 | // output final numbers... 451 | long now = System.currentTimeMillis(); 452 | thisInserts = globalInserts.get(); 453 | intervalNumber++; 454 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 455 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 456 | long elapsed = now - t0; 457 | long thisIntervalMs = now - lastMs; 458 | long thisIntervalInserts = thisInserts - lastInserts; 459 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 460 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 461 | if (secondsPerFeedback > 0) 462 | { 463 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 464 | } else { 465 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 466 | } 467 | try { 468 | if (outputHeader) 469 | { 470 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\n"); 471 | outputHeader = false; 472 | } 473 | String statusUpdate = ""; 474 | if (secondsPerFeedback > 0) 475 | { 476 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 477 | } else { 478 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond); 479 | } 480 | writer.write(statusUpdate); 481 | writer.flush(); 482 | } catch (IOException e) { 483 | e.printStackTrace(); 484 | } 485 | 486 | } 487 | } 488 | 489 | 490 | public static void logMe(String format, Object... args) { 491 | System.out.println(Thread.currentThread() + String.format(format, args)); 492 | } 493 | } 494 | --------------------------------------------------------------------------------