├── .gitignore ├── LICENSE ├── README.md ├── TODO ├── iibench-runner.bash ├── parse-results.bash ├── run.simple.bash └── src └── jmongoiibench.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | .classpath 3 | .project 4 | .vscode/* 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # Specific Files # 12 | clean.bash 13 | create-trust-store.bash 14 | instance-resize.bash 15 | instance-status.bash 16 | rds-truststore.jks 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | iibench-mongodb 2 | =============== 3 | 4 | iiBench Benchmark for MongoDB and DocumentDB 5 | 6 | 7 | Requirements 8 | ===================== 9 | 10 | * Java 11 11 | * The MongoDB Java driver jars must exist and be in the CLASSPATH. If you don't already have the MongoDB Java driver, then execute the following two commands: 12 | 13 | ```bash 14 | wget https://repo.maven.apache.org/maven2/org/mongodb/mongodb-driver-sync/4.8.1/mongodb-driver-sync-4.8.1.jar 15 | wget https://repo.maven.apache.org/maven2/org/mongodb/mongodb-driver-core/4.8.1/mongodb-driver-core-4.8.1.jar 16 | wget https://repo.maven.apache.org/maven2/org/mongodb/bson/4.8.1/bson-4.8.1.jar 17 | export CLASSPATH=$PWD/mongo-java-sync-4.8.1.jar:$PWD/mongo-driver-core-4.8.1.jar:$PWD/mongodb-driver-core-4.8.1.jar$CLASSPATH 18 | 19 | ``` 20 | 21 | 22 | Running the benchmark 23 | ===================== 24 | 25 | In the default configuration the benchmark will run for 2 days or 10 million, whichever comes first. 26 | 27 | ```bash 28 | git clone https://github.com/tmcallaghan/iibench-mongodb.git 29 | cd iibench-mongodb 30 | 31 | ``` 32 | 33 | *[optionally edit run.simple.bash to modify the benchmark behavior]* 34 | * Customize MONGO_URI for your specific connection needs 35 | 36 | ```bash 37 | ./run.simple.bash 38 | 39 | ``` 40 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - add an optional padding field to get big quickly 2 | - user defined size 3 | - user defined compressible amount 4 | - improve command line argument passing 5 | - use "--argument value" syntax 6 | - add defaults 7 | - additional query workload features 8 | - randomly select query type (based on indexes) 9 | - compare other features in Launchpad version -------------------------------------------------------------------------------- /iibench-runner.bash: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # check for required host environment parameters 4 | MONGO_SERVER_TEST=${DOCDB_HOST:?Environment variable not set or empty} 5 | MONGO_USERNAME_TEST=${DOCDB_USERNAME:?Environment variable not set or empty} 6 | MONGO_PASSWORD_TEST=${DOCDB_PASSWORD:?Environment variable not set or empty} 7 | 8 | 9 | instanceClasses="db.r6g.large db.r6g.xlarge db.r6g.2xlarge db.r6g.4xlarge db.r6g.8xlarge db.r6g.12xlarge db.r6g.16xlarge db.r5.large db.r5.xlarge db.r5.2xlarge db.r5.4xlarge db.r5.8xlarge db.r5.12xlarge db.r5.16xlarge db.r5.24xlarge" 10 | #instanceClasses="db.r6g.large db.r6g.8xlarge db.r5.2xlarge" 11 | 12 | DBINSTANCE='ddb4-max-insert-speed2' 13 | sleepSeconds=5 14 | finalSleepSeconds=15 15 | 16 | for thisInstanceClass in $instanceClasses; do 17 | echo "Modifying to instance type ${thisInstanceClass}" 18 | aws docdb modify-db-instance --db-instance-identifier $DBINSTANCE --db-instance-class $thisInstanceClass --apply-immediately 19 | 20 | T="$(date +%s)" 21 | 22 | instanceStatus='unknown' 23 | instancePendingModifiedValues=1 24 | 25 | while true ; do 26 | instanceInfo=`aws docdb describe-db-instances --db-instance-identifier $DBINSTANCE` 27 | 28 | instanceStatus=`echo $instanceInfo | jq -r '.DBInstances[0].DBInstanceStatus'` 29 | instanceClass=`echo $instanceInfo | jq -r '.DBInstances[0].DBInstanceClass'` 30 | instanceAvailabilityZone=`echo $instanceInfo | jq -r '.DBInstances[0].AvailabilityZone'` 31 | instancePendingModifiedValues=`echo $instanceInfo | jq '.DBInstances[0].PendingModifiedValues | length'` 32 | 33 | T2="$(($(date +%s)-T))" 34 | 35 | thisDuration=`printf "%02d:%02d:%02d:%02d" "$((T2/86400))" "$((T2/3600%24))" "$((T2/60%60))" "$((T2%60))"` 36 | 37 | echo "${thisDuration} | status = ${instanceStatus}, instance = ${instanceClass}, az = ${instanceAvailabilityZone}, pending modifications = ${instancePendingModifiedValues}" 38 | 39 | if [[ "$instanceStatus" == "available" ]] && [[ $instancePendingModifiedValues -eq "0" ]] ; then 40 | break 41 | fi 42 | 43 | sleep $sleepSeconds 44 | done 45 | 46 | # instance type now set, execute the benchmark 47 | export LOG_NAME_EXTRA="-${thisInstanceClass}" 48 | 49 | # one more sleep before the benchmark begins 50 | sleep $finalSleepSeconds 51 | 52 | ./run.simple.bash 53 | done 54 | -------------------------------------------------------------------------------- /parse-results.bash: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | for resultFile in mongoiibench*.txt; do 4 | instanceType=`echo $resultFile | cut -f 10 -d - | cut -f 1-3 -d .` 5 | perfInfo=`cat $resultFile | grep 'inserts : ' | tail -n3 | head -n1` 6 | cumIps=`echo $perfInfo | cut -f 3 -d : | cut -f 2 -d =` 7 | #echo "$instanceType | $cumIps" 8 | printf "%20s | %15s\n" "$instanceType" "$cumIps" 9 | done 10 | 11 | -------------------------------------------------------------------------------- /run.simple.bash: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # name of the server to connect to 4 | export MONGO_SERVER=${DOCDB_HOST:?Environment variable not set or empty} 5 | 6 | # port of the server to connect to 7 | export MONGO_PORT=27017 8 | 9 | # user 10 | export MONGO_USERNAME=${DOCDB_USERNAME:?Environment variable not set or empty} 11 | 12 | # password 13 | export MONGO_PASSWORD=${DOCDB_PASSWORD:?Environment variable not set or empty} 14 | 15 | # maximum size of connection pool 16 | export MAX_POOL_SIZE=4096 17 | 18 | # URI used for connection, customize as needed 19 | export MONGO_URI="mongodb://$MONGO_USERNAME:$MONGO_PASSWORD@$MONGO_SERVER/?replicaSet=rs0&readPreference=secondaryPreferred&retryWrites=false&maxPoolSize=$MAX_POOL_SIZE" 20 | 21 | # run the benchmark for this many inserts (or the number of minutes defined by RUN_MINUTES) 22 | # valid values : integer > 0 23 | export MAX_ROWS=10000000 24 | 25 | # total number of documents to insert per "batch" 26 | # valid values : integer > 0 27 | export NUM_DOCUMENTS_PER_INSERT=100 28 | 29 | # total number of simultaneous insertion threads 30 | # valid values : integer > 0 31 | export NUM_LOADER_THREADS=8 32 | 33 | # number of secondary indexes to maintain 34 | # valid values : integer >= 0 and <= 3 35 | export NUM_SECONDARY_INDEXES=1 36 | 37 | # number of additional character fields (semi-compressible) to add to each inserted document 38 | # valid values : integer >= 0 39 | export NUM_CHAR_FIELDS=10 40 | 41 | # size (in bytes) of each additional semi-compressible character field 42 | # valid values : integer >= 0 43 | export LENGTH_CHAR_FIELDS=1024 44 | 45 | # database in which to run the benchmark 46 | # valid values : character 47 | #export DB_NAME=iibench 48 | export DB_NAME=iibench4 49 | export COLLECTION_NAME=test1 50 | 51 | # suppress output from exceptions 52 | export SUPPRESS_EXCEPTIONS=0 53 | 54 | # run the benchmark for this many minutes (or the number of inserts defined by MAX_ROWS) 55 | # valid values : intever > 0 56 | export RUN_MINUTES=2880 57 | export RUN_SECONDS=$[RUN_MINUTES*60] 58 | 59 | # total number of documents to insert per second, allows for the benchmark to be rate limited 60 | # valid values : integer > 0 61 | export MAX_INSERTS_PER_SECOND=999999 62 | 63 | # display performance information every time the client application inserts this many documents 64 | # valid values : integer > 0, set to -1 if using NUM_SECONDS_PER_FEEDBACK 65 | export NUM_INSERTS_PER_FEEDBACK=-1 66 | 67 | # display performance information every time the client application has run for this many seconds 68 | # valid values : integer > 0, set to -1 if using NUM_INSERTS_PER_FEEDBACK 69 | export NUM_SECONDS_PER_FEEDBACK=10 70 | 71 | 72 | # percentage of highly compressible data (repeated character "a") in character field 73 | # valid values : integer >= 0 and <= 100 74 | export PERCENT_COMPRESSIBLE=80 75 | 76 | 77 | # the following 4 parameters allow an insert plus query workload benchmark 78 | 79 | # number of queries to perform per QUERY_INTERVAL_SECONDS seconds 80 | # valid values : integer > 0, set to zero for insert only workload 81 | export QUERIES_PER_INTERVAL=0 82 | 83 | # number of seconds during which to perform QUERIES_PER_INTERVAL queries 84 | # valid values : integer > 0 85 | export QUERY_INTERVAL_SECONDS=15 86 | 87 | # number of documents to return per query 88 | # valid values : integer > 0 89 | export QUERY_LIMIT=10 90 | 91 | # wait this many inserts to begin the query workload 92 | # valid values : integer > 0 93 | export QUERY_NUM_DOCS_BEGIN=1000000 94 | 95 | # create the collection 96 | # valid values : Y/N 97 | export CREATE_COLLECTION=Y 98 | 99 | # number of lines to tail from output 100 | TAIL_LINES=10 101 | 102 | mongoJars="$PWD/mongodb-driver-sync-4.8.1.jar:$PWD/mongodb-driver-core-4.8.1.jar:$PWD/bson-4.8.1.jar" 103 | 104 | #javac -cp ${mongoJars}:$CLASSPATH:$PWD/src src/jmongoiibench.java -Xlint:deprecation 105 | javac --release 11 -cp ${mongoJars}:$CLASSPATH:$PWD/src src/jmongoiibench.java 106 | 107 | YMDHMS=`date +'%Y%m%d-%H%M%S'` 108 | 109 | export LOG_NAME=mongoiibench-${YMDHMS}-${MAX_ROWS}-${NUM_DOCUMENTS_PER_INSERT}-${MAX_INSERTS_PER_SECOND}-${NUM_LOADER_THREADS}-${QUERIES_PER_INTERVAL}-${QUERY_INTERVAL_SECONDS}${LOG_NAME_EXTRA}.txt 110 | export BENCHMARK_TSV=${LOG_NAME}.tsv 111 | 112 | rm -f $LOG_NAME 113 | rm -f $BENCHMARK_TSV 114 | 115 | T="$(date +%s)" 116 | java -cp ${mongoJars}:$CLASSPATH:$PWD/src jmongoiibench $DB_NAME $COLLECTION_NAME $NUM_LOADER_THREADS $MAX_ROWS $NUM_DOCUMENTS_PER_INSERT $NUM_INSERTS_PER_FEEDBACK $NUM_SECONDS_PER_FEEDBACK $BENCHMARK_TSV $RUN_SECONDS $QUERIES_PER_INTERVAL $QUERY_INTERVAL_SECONDS $QUERY_LIMIT $QUERY_NUM_DOCS_BEGIN $MAX_INSERTS_PER_SECOND $NUM_CHAR_FIELDS $LENGTH_CHAR_FIELDS $NUM_SECONDARY_INDEXES $PERCENT_COMPRESSIBLE $CREATE_COLLECTION $SUPPRESS_EXCEPTIONS $MONGO_URI | tee -a $LOG_NAME 117 | echo "" | tee -a $LOG_NAME 118 | T="$(($(date +%s)-T))" 119 | printf "`date` | iibench duration = %02d:%02d:%02d:%02d\n" "$((T/86400))" "$((T/3600%24))" "$((T/60%60))" "$((T%60))" | tee -a $LOG_NAME 120 | echo "" 121 | echo "************************************************************************" 122 | echo "final $TAIL_LINES interval(s)" 123 | echo "************************************************************************" 124 | tail -n $TAIL_LINES $LOG_NAME 125 | 126 | cat $LOG_NAME | grep '170 seconds' >> results.txt 127 | 128 | -------------------------------------------------------------------------------- /src/jmongoiibench.java: -------------------------------------------------------------------------------- 1 | import com.mongodb.client.MongoClient; 2 | import com.mongodb.client.MongoClients; 3 | import com.mongodb.client.MongoCollection; 4 | import com.mongodb.client.MongoCursor; 5 | import com.mongodb.client.MongoDatabase; 6 | import com.mongodb.client.model.Projections; 7 | import com.mongodb.client.model.Indexes; 8 | import com.mongodb.client.model.BulkWriteOptions; 9 | import com.mongodb.client.model.IndexOptions; 10 | import com.mongodb.client.model.WriteModel; 11 | import com.mongodb.client.model.InsertOneModel; 12 | 13 | import org.bson.Document; 14 | import org.bson.conversions.Bson; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.io.BufferedWriter; 19 | import java.io.FileWriter; 20 | import java.io.File; 21 | import java.io.Writer; 22 | import java.io.IOException; 23 | import java.util.concurrent.atomic.AtomicLong; 24 | import java.text.SimpleDateFormat; 25 | import java.time.Instant; 26 | import java.util.Date; 27 | import java.util.TimeZone; 28 | 29 | public class jmongoiibench { 30 | public static AtomicLong globalInserts = new AtomicLong(0); 31 | public static AtomicLong globalWriterThreads = new AtomicLong(0); 32 | public static AtomicLong globalQueryThreads = new AtomicLong(0); 33 | public static AtomicLong globalQueriesExecuted = new AtomicLong(0); 34 | public static AtomicLong globalQueriesTimeMs = new AtomicLong(0); 35 | public static AtomicLong globalQueriesStarted = new AtomicLong(0); 36 | public static AtomicLong globalInsertExceptions = new AtomicLong(0); 37 | 38 | public static Writer writer = null; 39 | public static boolean outputHeader = true; 40 | 41 | public static int numCashRegisters = 1000; 42 | public static int numProducts = 10000; 43 | public static int numCustomers = 100000; 44 | public static double maxPrice = 500.0; 45 | 46 | public static String dbName; 47 | public static String collName; 48 | public static int writerThreads; 49 | public static long numMaxInserts; 50 | public static int documentsPerInsert; 51 | public static long insertsPerFeedback; 52 | public static long secondsPerFeedback; 53 | public static String logFileName; 54 | public static Long numSeconds; 55 | public static Integer queriesPerInterval; 56 | public static Integer queryIntervalSeconds; 57 | public static double queriesPerMinute; 58 | public static Long msBetweenQueries; 59 | public static Integer queryLimit; 60 | public static Integer queryBeginNumDocs; 61 | public static Integer maxInsertsPerSecond; 62 | public static Integer maxThreadInsertsPerSecond; 63 | public static String createCollection; 64 | public static int numCharFields; 65 | public static int lengthCharFields; 66 | public static int numSecondaryIndexes; 67 | public static int percentCompressible; 68 | public static int numCompressibleCharacters; 69 | public static int numUncompressibleCharacters; 70 | public static boolean suppressExceptions = false; 71 | public static int exceptionPauseMs = 1000; 72 | 73 | public static int randomStringLength = 4*1024*1024; 74 | public static String randomStringHolder; 75 | public static int compressibleStringLength = 4*1024*1024; 76 | public static String compressibleStringHolder; 77 | public static String connectionString; 78 | 79 | public static int allDone = 0; 80 | 81 | public jmongoiibench() { 82 | } 83 | 84 | public static void main (String[] args) throws Exception { 85 | if (args.length != 21) { 86 | logMe("*** ERROR : CONFIGURATION ISSUE ***"); 87 | logMe("jmongoiibench [database name] [collection name] [number of writer threads] [documents per collection] [documents per insert] " 88 | +"[inserts feedback] [seconds feedback] [log file name] [number of seconds to run] [queries per interval] [interval (seconds)] " 89 | +"[query limit] [inserts for begin query] [max inserts per second] [num char fields] [length char fields] [num secondary indexes] " 90 | +"[percent compressible] [create collection] [suppress exceptions (0 or 1)] [connection string]"); 91 | System.exit(1); 92 | } 93 | 94 | dbName = args[0]; 95 | collName = args[1]; 96 | writerThreads = Integer.valueOf(args[2]); 97 | numMaxInserts = Long.valueOf(args[3]); 98 | documentsPerInsert = Integer.valueOf(args[4]); 99 | insertsPerFeedback = Long.valueOf(args[5]); 100 | secondsPerFeedback = Long.valueOf(args[6]); 101 | logFileName = args[7]; 102 | numSeconds = Long.valueOf(args[8]); 103 | queriesPerInterval = Integer.valueOf(args[9]); 104 | queryIntervalSeconds = Integer.valueOf(args[10]); 105 | queryLimit = Integer.valueOf(args[11]); 106 | queryBeginNumDocs = Integer.valueOf(args[12]); 107 | maxInsertsPerSecond = Integer.valueOf(args[13]); 108 | numCharFields = Integer.valueOf(args[14]); 109 | lengthCharFields = Integer.valueOf(args[15]); 110 | numSecondaryIndexes = Integer.valueOf(args[16]); 111 | percentCompressible = Integer.valueOf(args[17]); 112 | createCollection = args[18].toLowerCase(); 113 | if (Integer.valueOf(args[19]) == 1) { 114 | suppressExceptions = true; 115 | }; 116 | connectionString = args[20]; 117 | 118 | maxThreadInsertsPerSecond = (maxInsertsPerSecond / writerThreads); 119 | 120 | if ((numSecondaryIndexes < 0) || (numSecondaryIndexes > 3)) { 121 | logMe("*** ERROR : INVALID NUMBER OF SECONDARY INDEXES, MUST BE >=0 and <= 3 ***"); 122 | logMe(" %d secondary indexes is not supported",numSecondaryIndexes); 123 | System.exit(1); 124 | } 125 | 126 | if ((queriesPerInterval <= 0) || (queryIntervalSeconds <= 0)) 127 | { 128 | queriesPerMinute = 0.0; 129 | msBetweenQueries = 0l; 130 | } 131 | else 132 | { 133 | queriesPerMinute = (double)queriesPerInterval * (60.0 / (double)queryIntervalSeconds); 134 | msBetweenQueries = (long)((1000.0 * (double)queryIntervalSeconds) / (double)queriesPerInterval); 135 | } 136 | 137 | if ((percentCompressible < 0) || (percentCompressible > 100)) { 138 | logMe("*** ERROR : INVALID PERCENT COMPRESSIBLE, MUST BE >=0 and <= 100 ***"); 139 | logMe(" %d secondary indexes is not supported",percentCompressible); 140 | System.exit(1); 141 | } 142 | 143 | numCompressibleCharacters = (int) (((double) percentCompressible / 100.0) * (double) lengthCharFields); 144 | numUncompressibleCharacters = (int) (((100.0 - (double) percentCompressible) / 100.0) * (double) lengthCharFields); 145 | 146 | logMe("Application Parameters"); 147 | logMe("--------------------------------------------------"); 148 | logMe(" namespace = %s.%s",dbName,collName); 149 | logMe(" %d writer thread(s)",writerThreads); 150 | logMe(" %,d documents per collection",numMaxInserts); 151 | logMe(" %d character fields",numCharFields); 152 | logMe(" %d bytes per character field",lengthCharFields); 153 | logMe(" %d secondary indexes",numSecondaryIndexes); 154 | logMe(" Documents Per Insert = %d",documentsPerInsert); 155 | logMe(" Maximum of %,d insert(s) per second",maxInsertsPerSecond); 156 | logMe(" Maximum of %,d insert(s) per second per writer thread",maxThreadInsertsPerSecond); 157 | logMe(" Feedback every %,d seconds(s)",secondsPerFeedback); 158 | logMe(" Feedback every %,d inserts(s)",insertsPerFeedback); 159 | logMe(" logging to file %s",logFileName); 160 | logMe(" Run for %,d second(s)",numSeconds); 161 | logMe(" Extra character fields are %d percent compressible",percentCompressible); 162 | if (queriesPerMinute > 0.0) 163 | { 164 | logMe(" Attempting %,.2f queries per minute",queriesPerMinute); 165 | logMe(" Queries limited to %,d document(s)",queryLimit); 166 | logMe(" Starting queries after %,d document(s) inserted",queryBeginNumDocs); 167 | } 168 | else 169 | { 170 | logMe(" NO queries, insert only benchmark"); 171 | } 172 | 173 | //String truststore = "./rds-truststore.jks"; 174 | //String truststorePassword = "secret"; 175 | 176 | //System.setProperty("javax.net.ssl.trustStore", truststore); 177 | //System.setProperty("javax.net.ssl.trustStorePassword", truststorePassword); 178 | 179 | MongoClient m = MongoClients.create(connectionString); 180 | 181 | MongoDatabase db = m.getDatabase(dbName); 182 | 183 | logMe("--------------------------------------------------"); 184 | 185 | if (writerThreads > 1) { 186 | numMaxInserts = numMaxInserts / writerThreads; 187 | } 188 | 189 | try { 190 | writer = new BufferedWriter(new FileWriter(new File(logFileName))); 191 | } catch (IOException e) { 192 | e.printStackTrace(); 193 | } 194 | 195 | // test if collection exists 196 | boolean collectionExists = db.listCollectionNames().into(new ArrayList()).contains(collName); 197 | 198 | if (createCollection.equals("n")) 199 | { 200 | logMe("Skipping collection creation"); 201 | } 202 | else 203 | { 204 | // create the collection 205 | MongoCollection coll = db.getCollection(collName); 206 | 207 | // drop the collection, if it exists 208 | if (collectionExists) { 209 | logMe(" *** dropping collection " + dbName + "." + collName); 210 | coll.drop(); 211 | } 212 | 213 | IndexOptions idxOptions = new IndexOptions().background(false); 214 | 215 | if (numSecondaryIndexes >= 1) { 216 | logMe(" *** creating secondary index on price + customerid"); 217 | coll.createIndex(Indexes.ascending("price", "customerid"), idxOptions); 218 | } 219 | if (numSecondaryIndexes >= 2) { 220 | logMe(" *** creating secondary index on cashregisterid + price + customerid"); 221 | coll.createIndex(Indexes.ascending("cashregisterid", "price", "customerid"), idxOptions); 222 | } 223 | if (numSecondaryIndexes >= 3) { 224 | logMe(" *** creating secondary index on price + dateandtime + customerid"); 225 | coll.createIndex(Indexes.ascending("price", "dateandtime", "customerid"), idxOptions); 226 | } 227 | } 228 | 229 | java.util.Random rand = new java.util.Random(); 230 | 231 | // create random string holder 232 | logMe(" creating %,d bytes of random character data...",randomStringLength); 233 | char[] tempString = new char[randomStringLength]; 234 | for (int i = 0 ; i < randomStringLength ; i++) { 235 | tempString[i] = (char) (rand.nextInt(26) + 'a'); 236 | } 237 | randomStringHolder = new String(tempString); 238 | 239 | // create compressible string holder 240 | logMe(" creating %,d bytes of compressible character data...",compressibleStringLength); 241 | char[] tempStringCompressible = new char[compressibleStringLength]; 242 | for (int i = 0 ; i < compressibleStringLength ; i++) { 243 | tempStringCompressible[i] = 'a'; 244 | } 245 | compressibleStringHolder = new String(tempStringCompressible); 246 | 247 | 248 | jmongoiibench t = new jmongoiibench(); 249 | 250 | Thread reporterThread = new Thread(t.new MyReporter()); 251 | reporterThread.start(); 252 | 253 | Thread queryThread = new Thread(t.new MyQuery(1, 1, numMaxInserts, db, collName)); 254 | if (queriesPerMinute > 0.0) { 255 | queryThread.start(); 256 | } 257 | 258 | Thread[] tWriterThreads = new Thread[writerThreads]; 259 | 260 | // start the loaders 261 | for (int i=0; i coll = db.getCollection(collectionName); 321 | 322 | long numInserts = 0; 323 | long numLastInserts = 0; 324 | long nextMs = System.currentTimeMillis() + 1000; 325 | 326 | try { 327 | logMe("Writer thread %d : started to load collection %s",threadNumber, collectionName); 328 | 329 | List> bulkOperations = new ArrayList<>(); 330 | BulkWriteOptions bwOptions = new BulkWriteOptions().ordered(false); 331 | 332 | int numRounds = (int) numMaxInserts / documentsPerInsert; 333 | 334 | for (int roundNum = 0; roundNum < numRounds; roundNum++) { 335 | if ((numInserts - numLastInserts) >= maxInsertsPerSecond) { 336 | // pause until a second has passed 337 | while (System.currentTimeMillis() < nextMs) { 338 | try { 339 | Thread.sleep(20); 340 | } catch (Exception e) { 341 | e.printStackTrace(); 342 | } 343 | } 344 | numLastInserts = numInserts; 345 | nextMs = System.currentTimeMillis() + 1000; 346 | } 347 | 348 | for (int i = 0; i < documentsPerInsert; i++) { 349 | int thisCustomerId = rand.nextInt(numCustomers); 350 | double thisPrice= ((rand.nextDouble() * maxPrice) + (double) thisCustomerId) / 100.0; 351 | Document doc = new Document(); 352 | doc.put("dateandtime", System.currentTimeMillis()); 353 | doc.put("cashregisterid", rand.nextInt(numCashRegisters)); 354 | doc.put("customerid", thisCustomerId); 355 | doc.put("productid", rand.nextInt(numProducts)); 356 | doc.put("price", thisPrice); 357 | for (int charField = 1; charField <= numCharFields; charField++) { 358 | int startPosition = rand.nextInt(randomStringLength-lengthCharFields); 359 | doc.put("cf"+Integer.toString(charField), randomStringHolder.substring(startPosition,startPosition+numUncompressibleCharacters) + compressibleStringHolder.substring(startPosition,startPosition+numCompressibleCharacters)); 360 | } 361 | bulkOperations.add(new InsertOneModel<>(doc)); 362 | } 363 | 364 | try { 365 | coll.bulkWrite(bulkOperations, bwOptions); 366 | bulkOperations.clear(); 367 | numInserts += documentsPerInsert; 368 | globalInserts.addAndGet(documentsPerInsert); 369 | 370 | } catch (Exception e) { 371 | if (!suppressExceptions) { 372 | logMe("Writer thread %d : EXCEPTION",threadNumber); 373 | e.printStackTrace(); 374 | } 375 | globalInsertExceptions.incrementAndGet(); 376 | 377 | try { 378 | Thread.sleep(exceptionPauseMs); 379 | } catch (Exception e1) { 380 | e1.printStackTrace(); 381 | } 382 | } 383 | 384 | if (allDone == 1) 385 | break; 386 | } 387 | 388 | } catch (Exception e) { 389 | logMe("Writer thread %d : EXCEPTION",threadNumber); 390 | e.printStackTrace(); 391 | } 392 | 393 | long numWriters = globalWriterThreads.decrementAndGet(); 394 | if (numWriters == 0) 395 | allDone = 1; 396 | } 397 | } 398 | 399 | 400 | class MyQuery implements Runnable { 401 | int threadCount; 402 | int threadNumber; 403 | long numMaxInserts; 404 | MongoDatabase db; 405 | String collectionName; 406 | 407 | java.util.Random rand; 408 | 409 | MyQuery(int threadCount, int threadNumber, long numMaxInserts, MongoDatabase db, String collectionName) { 410 | this.threadCount = threadCount; 411 | this.threadNumber = threadNumber; 412 | this.numMaxInserts = numMaxInserts; 413 | this.db = db; 414 | this.collectionName = collectionName; 415 | rand = new java.util.Random((long) threadNumber); 416 | } 417 | public void run() { 418 | long t0 = System.currentTimeMillis(); 419 | long nextQueryMillis = t0; 420 | boolean outputWaiting = true; 421 | boolean outputStarted = true; 422 | 423 | MongoCollection coll = db.getCollection(collectionName); 424 | 425 | int whichQuery = 0; 426 | 427 | Bson projectionFields = Projections.fields( 428 | Projections.include("cashregisterid", "dateandtime", "customerid", "price"), 429 | Projections.excludeId()); 430 | 431 | try { 432 | logMe("Query thread %d : ready to query collection %s",threadNumber, collectionName); 433 | 434 | while (allDone == 0) { 435 | long thisNow = System.currentTimeMillis(); 436 | 437 | // wait until my next runtime 438 | if (thisNow > nextQueryMillis) { 439 | nextQueryMillis = thisNow + msBetweenQueries; 440 | 441 | // check if number of inserts reached 442 | if (globalInserts.get() >= queryBeginNumDocs) { 443 | if (outputStarted) 444 | { 445 | logMe("Query thread %d : now running",threadNumber,queryBeginNumDocs); 446 | outputStarted = false; 447 | // set query start time 448 | globalQueriesStarted.set(thisNow); 449 | } 450 | 451 | whichQuery++; 452 | if (whichQuery > 3) { 453 | whichQuery = 1; 454 | } 455 | 456 | int thisCustomerId = rand.nextInt(numCustomers); 457 | double thisPrice = ((rand.nextDouble() * maxPrice) + (double) thisCustomerId) / 100.0; 458 | int thisCashRegisterId = rand.nextInt(numCashRegisters); 459 | long thisRandomTime = t0 + (long) ((double) (thisNow - t0) * rand.nextDouble()); 460 | 461 | Document query = new Document(); 462 | Document keys = new Document(); 463 | 464 | if (whichQuery == 1) { 465 | // query 1 466 | /* 467 | db.purchases_index.find({$or: [ {price: , dateandtime: , customerid: {$gte: }}, 468 | {price: , dateandtime: {$gt: }}, 469 | {price: {$gt: }} ]}, 470 | {price:1, dateandtime:1, customerid:1, _id:0}).limit(1000); 471 | */ 472 | 473 | Document query1a = new Document(); 474 | query1a.put("price", thisPrice); 475 | query1a.put("dateandtime", thisRandomTime); 476 | query1a.put("customerid", new Document("$gte", thisCustomerId)); 477 | 478 | Document query1b = new Document(); 479 | query1b.put("price", thisPrice); 480 | query1b.put("dateandtime", new Document("$gt", thisRandomTime)); 481 | 482 | Document query1c = new Document(); 483 | query1c.put("price", new Document("$gt", thisPrice)); 484 | 485 | ArrayList list1 = new ArrayList(); 486 | list1.add(query1a); 487 | list1.add(query1b); 488 | list1.add(query1c); 489 | 490 | query.put("$or", list1); 491 | 492 | keys.put("price",1); 493 | keys.put("dateandtime",1); 494 | keys.put("customerid",1); 495 | keys.put("_id",0); 496 | 497 | } else if (whichQuery == 2) { 498 | // query 2 499 | /* 500 | db.purchases_index.find({$or: [ {price: , customerid: {$gte: } }, 501 | {price: {$gt: }} ]}, 502 | {price:1, customerid:1, _id:0}).limit(1000); 503 | */ 504 | 505 | Document query2a = new Document(); 506 | query2a.put("price", thisPrice); 507 | query2a.put("customerid", new Document("$gte", thisCustomerId)); 508 | 509 | Document query2b = new Document(); 510 | query2b.put("price", new Document("$gt", thisPrice)); 511 | 512 | ArrayList list2 = new ArrayList(); 513 | list2.add(query2a); 514 | list2.add(query2b); 515 | 516 | query.put("$or", list2); 517 | 518 | keys.put("price",1); 519 | keys.put("customerid",1); 520 | keys.put("_id",0); 521 | 522 | } else if (whichQuery == 3) { 523 | // query 3 524 | /* 525 | db.purchases_index.find({$or: [ {cashregisterid: , price: , customerid: {$gte: } }, 526 | {cashregisterid: , price: {$gt: }}, 527 | {cashregisterid: {$gt: }} ]}, 528 | {cashregisterid:1, price:1, customerid:1, _id:0}).limit(1000);; 529 | */ 530 | 531 | Document query3a = new Document(); 532 | query3a.put("cashregisterid", thisCashRegisterId); 533 | query3a.put("price", thisPrice); 534 | query3a.put("customerid", new Document("$gte", thisCustomerId)); 535 | 536 | Document query3b = new Document(); 537 | query3b.put("cashregisterid", thisCashRegisterId); 538 | query3b.put("price", new Document("$gt", thisPrice)); 539 | 540 | Document query3c = new Document(); 541 | query3c.put("cashregisterid", new Document("$gt", thisCashRegisterId)); 542 | 543 | ArrayList list3 = new ArrayList(); 544 | list3.add(query3a); 545 | list3.add(query3b); 546 | list3.add(query3c); 547 | 548 | query.put("$or", list3); 549 | 550 | keys.put("cashregisterid",1); 551 | keys.put("price",1); 552 | keys.put("customerid",1); 553 | keys.put("_id",0); 554 | 555 | } 556 | 557 | long now = System.currentTimeMillis(); 558 | MongoCursor cursor = coll.find(query).projection(projectionFields).limit(queryLimit).iterator(); 559 | try { 560 | while(cursor.hasNext()) { 561 | cursor.next(); 562 | } 563 | } finally { 564 | cursor.close(); 565 | } 566 | long elapsed = System.currentTimeMillis() - now; 567 | 568 | globalQueriesExecuted.incrementAndGet(); 569 | globalQueriesTimeMs.addAndGet(elapsed); 570 | } else { 571 | if (outputWaiting) 572 | { 573 | logMe("Query thread %d : waiting for %,d document insert(s) before starting",threadNumber,queryBeginNumDocs); 574 | outputWaiting = false; 575 | } 576 | } 577 | } 578 | } 579 | 580 | } catch (Exception e) { 581 | logMe("Query thread %d : EXCEPTION",threadNumber); 582 | e.printStackTrace(); 583 | } 584 | } 585 | } 586 | 587 | 588 | // reporting thread, outputs information to console and file 589 | class MyReporter implements Runnable { 590 | public void run() 591 | { 592 | long t0 = System.currentTimeMillis(); 593 | long lastInserts = 0; 594 | long lastQueriesNum = 0; 595 | long lastQueriesMs = 0; 596 | long lastMs = t0; 597 | long intervalNumber = 0; 598 | long nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 599 | long nextFeedbackInserts = lastInserts + insertsPerFeedback; 600 | long thisInserts = 0; 601 | long thisQueriesNum = 0; 602 | long thisQueriesMs = 0; 603 | long thisQueriesStarted = 0; 604 | long endDueToTime = System.currentTimeMillis() + (1000 * numSeconds); 605 | 606 | while (allDone == 0) 607 | { 608 | try { 609 | Thread.sleep(100); 610 | } catch (Exception e) { 611 | e.printStackTrace(); 612 | } 613 | 614 | long now = System.currentTimeMillis(); 615 | 616 | if (now >= endDueToTime) 617 | { 618 | allDone = 1; 619 | } 620 | 621 | thisInserts = globalInserts.get(); 622 | thisQueriesNum = globalQueriesExecuted.get(); 623 | thisQueriesMs = globalQueriesTimeMs.get(); 624 | thisQueriesStarted = globalQueriesStarted.get(); 625 | if (((now > nextFeedbackMillis) && (secondsPerFeedback > 0)) || 626 | ((thisInserts >= nextFeedbackInserts) && (insertsPerFeedback > 0))) 627 | { 628 | intervalNumber++; 629 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 630 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 631 | 632 | long elapsed = now - t0; 633 | long thisIntervalMs = now - lastMs; 634 | 635 | long thisIntervalInserts = thisInserts - lastInserts; 636 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 637 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 638 | 639 | long thisIntervalQueriesNum = thisQueriesNum - lastQueriesNum; 640 | long thisIntervalQueriesMs = thisQueriesMs - lastQueriesMs; 641 | double thisIntervalQueryAvgMs = 0; 642 | double thisQueryAvgMs = 0; 643 | double thisIntervalAvgQPM = 0; 644 | double thisAvgQPM = 0; 645 | 646 | long thisInsertExceptions = globalInsertExceptions.get(); 647 | 648 | if (thisIntervalQueriesNum > 0) { 649 | thisIntervalQueryAvgMs = thisIntervalQueriesMs/(double)thisIntervalQueriesNum; 650 | } 651 | if (thisQueriesNum > 0) { 652 | thisQueryAvgMs = thisQueriesMs/(double)thisQueriesNum; 653 | } 654 | 655 | if (thisQueriesStarted > 0) 656 | { 657 | long adjustedElapsed = now - thisQueriesStarted; 658 | if (adjustedElapsed > 0) 659 | { 660 | thisAvgQPM = (double)thisQueriesNum/((double)adjustedElapsed/1000.0/60.0); 661 | } 662 | if (thisIntervalMs > 0) 663 | { 664 | thisIntervalAvgQPM = (double)thisIntervalQueriesNum/((double)thisIntervalMs/1000.0/60.0); 665 | } 666 | } 667 | 668 | if (secondsPerFeedback > 0) 669 | { 670 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f : exceptions=%,d", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 671 | } else { 672 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f : exceptions=%,d", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 673 | } 674 | 675 | try { 676 | if (outputHeader) 677 | { 678 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\tcum_qry_avg\tint_qry_avg\tcum_qpm\tint_qpm\texceptions\n"); 679 | outputHeader = false; 680 | } 681 | 682 | String statusUpdate = ""; 683 | 684 | if (secondsPerFeedback > 0) 685 | { 686 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%,d\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 687 | } else { 688 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%,d\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM, thisInsertExceptions); 689 | } 690 | writer.write(statusUpdate); 691 | writer.flush(); 692 | } catch (IOException e) { 693 | e.printStackTrace(); 694 | } 695 | 696 | lastInserts = thisInserts; 697 | lastQueriesNum = thisQueriesNum; 698 | lastQueriesMs = thisQueriesMs; 699 | 700 | lastMs = now; 701 | } 702 | } 703 | 704 | // output final numbers... 705 | long now = System.currentTimeMillis(); 706 | thisInserts = globalInserts.get(); 707 | thisQueriesNum = globalQueriesExecuted.get(); 708 | thisQueriesMs = globalQueriesTimeMs.get(); 709 | thisQueriesStarted = globalQueriesStarted.get(); 710 | intervalNumber++; 711 | nextFeedbackMillis = t0 + (1000 * secondsPerFeedback * (intervalNumber + 1)); 712 | nextFeedbackInserts = (intervalNumber + 1) * insertsPerFeedback; 713 | long elapsed = now - t0; 714 | long thisIntervalMs = now - lastMs; 715 | long thisIntervalInserts = thisInserts - lastInserts; 716 | double thisIntervalInsertsPerSecond = thisIntervalInserts/(double)thisIntervalMs*1000.0; 717 | double thisInsertsPerSecond = thisInserts/(double)elapsed*1000.0; 718 | long thisIntervalQueriesNum = thisQueriesNum - lastQueriesNum; 719 | long thisIntervalQueriesMs = thisQueriesMs - lastQueriesMs; 720 | double thisIntervalQueryAvgMs = 0; 721 | double thisQueryAvgMs = 0; 722 | double thisIntervalAvgQPM = 0; 723 | double thisAvgQPM = 0; 724 | if (thisIntervalQueriesNum > 0) { 725 | thisIntervalQueryAvgMs = thisIntervalQueriesMs/(double)thisIntervalQueriesNum; 726 | } 727 | if (thisQueriesNum > 0) { 728 | thisQueryAvgMs = thisQueriesMs/(double)thisQueriesNum; 729 | } 730 | if (thisQueriesStarted > 0) 731 | { 732 | long adjustedElapsed = now - thisQueriesStarted; 733 | if (adjustedElapsed > 0) 734 | { 735 | thisAvgQPM = (double)thisQueriesNum/((double)adjustedElapsed/1000.0/60.0); 736 | } 737 | if (thisIntervalMs > 0) 738 | { 739 | thisIntervalAvgQPM = (double)thisIntervalQueriesNum/((double)thisIntervalMs/1000.0/60.0); 740 | } 741 | } 742 | if (secondsPerFeedback > 0) 743 | { 744 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f", thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 745 | } else { 746 | logMe("%,d inserts : %,d seconds : cum ips=%,.2f : int ips=%,.2f : cum avg qry=%,.2f : int avg qry=%,.2f : cum avg qpm=%,.2f : int avg qpm=%,.2f", intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 747 | } 748 | try { 749 | if (outputHeader) 750 | { 751 | writer.write("tot_inserts\telap_secs\tcum_ips\tint_ips\tcum_qry_avg\tint_qry_avg\tcum_qpm\tint_qpm\n"); 752 | outputHeader = false; 753 | } 754 | String statusUpdate = ""; 755 | if (secondsPerFeedback > 0) 756 | { 757 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n",thisInserts, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 758 | } else { 759 | statusUpdate = String.format("%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n",intervalNumber * insertsPerFeedback, elapsed / 1000l, thisInsertsPerSecond, thisIntervalInsertsPerSecond, thisQueryAvgMs, thisIntervalQueryAvgMs, thisAvgQPM, thisIntervalAvgQPM); 760 | } 761 | writer.write(statusUpdate); 762 | writer.flush(); 763 | } catch (IOException e) { 764 | e.printStackTrace(); 765 | } 766 | 767 | } 768 | } 769 | 770 | 771 | public static void logMe(String format, Object... args) { 772 | Date currentUtcTime = Date.from(Instant.now()); 773 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); 774 | sdf.setTimeZone(TimeZone.getTimeZone("Etc/UTC")); 775 | // System.out.println("Current UTC time is " + sdf.format(currentUtcTime)); 776 | 777 | System.out.println(Thread.currentThread().toString() + ' ' + sdf.format(currentUtcTime) + " | " + String.format(format, args)); 778 | } 779 | } 780 | --------------------------------------------------------------------------------