├── .arcconfig ├── .gitignore ├── DataModel.md ├── LICENSE ├── NOTICES ├── README.md ├── bin └── linkbench ├── config ├── Distribution.dat ├── Distribution_2011.dat ├── FBWorkload.properties ├── LinkConfigMongoDb2.properties ├── LinkConfigMongoDbBasic.properties ├── LinkConfigMysql.properties ├── LinkConfigPgsql.properties └── LinkConfigRocksDb.properties ├── docs ├── ftdc_00e7bb.png └── statistics.md ├── mongo-setup.js ├── pgsql-ddl.sql ├── pom.xml └── src ├── main └── java │ └── com │ └── facebook │ └── LinkBench │ ├── Config.java │ ├── ConfigUtil.java │ ├── GraphStore.java │ ├── InvertibleShuffler.java │ ├── Level.java │ ├── Link.java │ ├── LinkBenchConfigError.java │ ├── LinkBenchDriver.java │ ├── LinkBenchLoad.java │ ├── LinkBenchOp.java │ ├── LinkBenchRequest.java │ ├── LinkBenchTask.java │ ├── LinkCount.java │ ├── LinkStore.java │ ├── LinkStoreMongoDb2.java │ ├── LinkStoreMongoDbBasic.java │ ├── LinkStoreMysql.java │ ├── LinkStorePgsql.java │ ├── LinkStoreSql.java │ ├── LinkWriteResult.java │ ├── Logger.java │ ├── MemoryLinkStore.java │ ├── Node.java │ ├── NodeLoader.java │ ├── NodeStore.java │ ├── Phase.java │ ├── RealDistribution.java │ ├── Shuffler.java │ ├── Timer.java │ ├── distributions │ ├── AccessDistributions.java │ ├── ApproxHarmonic.java │ ├── GeometricDistribution.java │ ├── Harmonic.java │ ├── ID2Chooser.java │ ├── ID2Chooser.java.new │ ├── LinkDistributions.java │ ├── LogNormalDistribution.java │ ├── PiecewiseLinearDistribution.java │ ├── ProbabilityDistribution.java │ ├── UniformDistribution.java │ └── ZipfDistribution.java │ ├── generators │ ├── DataGenerator.java │ ├── MotifDataGenerator.java │ └── UniformDataGenerator.java │ ├── stats │ ├── HdrLatencyHistogram.java │ ├── LatencyHistogram.java │ ├── LatencyHistogramFactory.java │ ├── LatencyStats.java │ ├── RunningMean.java │ └── SampledStats.java │ └── util │ └── ClassLoadUtil.java └── test └── java └── com └── facebook └── LinkBench ├── DistributionTestBase.java ├── DummyLinkStore.java ├── DummyLinkStoreTest.java ├── GeneratedDataDump.java ├── GeomDistTest.java ├── GraphStoreTestBase.java ├── HarmonicTest.java ├── ID2ChooserTest.java ├── InvertibleShufflerTest.java ├── LinkStoreTestBase.java ├── LogNormalTest.java ├── MemoryGraphStoreTest.java ├── MemoryLinkStoreTest.java ├── MemoryNodeStoreTest.java ├── MongoDbGraphStoreTest2.java ├── MongoDbGraphStoreTestBasic.java ├── MongoDbLinkStoreTest2.java ├── MongoDbLinkStoreTestBasic.java ├── MongoDbNodeStoreTest2.java ├── MongoDbNodeStoreTestBasic.java ├── MongoDbTestConfig2.java ├── MongoDbTestConfigBasic.java ├── MySqlGraphStoreTest.java ├── MySqlLinkStoreTest.java ├── MySqlNodeStoreTest.java ├── MySqlTestConfig.java ├── NodeStoreTestBase.java ├── PgsqlGraphStoreTest.java ├── PgsqlLinkStoreTest.java ├── PgsqlNodeStoreTest.java ├── PgsqlTestConfig.java ├── PiecewiseDistTest.java ├── TestAccessDistribution.java ├── TestDataGen.java ├── TestHdrLatencyHistogram.java ├── TestLatencyHistogramFactory.java ├── TestRealDistribution.java ├── TestStats.java ├── TimerTest.java ├── UniformDistTest.java ├── ZipfDistTest.java └── testtypes ├── MongoDbTest.java ├── MySqlTest.java ├── PgsqlTest.java ├── ProviderTest.java ├── RocksDbTest.java └── SlowTest.java /.arcconfig: -------------------------------------------------------------------------------- 1 | { 2 | "project_id" : "linkbench", 3 | "conduit_uri" : "https://reviews.facebook.net/", 4 | "copyright_holder" : "", 5 | "lint.engine" : "ArcanistSingleLintEngine", 6 | "lint.engine.single.linter" : "ArcanistTextLinter" 7 | } 8 | 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | out 3 | .project 4 | .classpath 5 | .settings 6 | *.iws 7 | *.ipr 8 | *.iml 9 | .idea 10 | .DS_Store 11 | .vscode/ 12 | -------------------------------------------------------------------------------- /NOTICES: -------------------------------------------------------------------------------- 1 | Several third party components are included with the LinkBench source 2 | distribution. 3 | 4 | Apache Commons CLI, Apache Commons Math, Apache log4j, Apache Hadoop, 5 | and Apache HBase are licensed under the Apache 2.0 license, which is 6 | available at: 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | 11 | JUnit is licensed under the Common Public License v1.0, which is 12 | available at: 13 | 14 | http://junit.sourceforge.net/cpl-v10.html 15 | 16 | MySQL Connector/J is licensed under the General Public License v2.0 with 17 | the MySQL FOSS exception. Further license information is included in 18 | the Connector/J source distribution. The license can also be viewed at: 19 | 20 | http://www.mysql.com/about/legal/licensing/foss-exception/ 21 | -------------------------------------------------------------------------------- /bin/linkbench: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # The command script 20 | # 21 | # Environment Variables 22 | # 23 | # JAVA_HOME The java implementation to use. Overrides JAVA_HOME. 24 | # 25 | # HEAPSIZE The maximum amount of heap to use, in MB. 26 | # Default is 1000. 27 | # 28 | # OPTS Extra Java runtime options. 29 | # 30 | # CONF_DIR Alternate conf dir. Default is ./config. 31 | # 32 | # This script creates the benchmark data and then runs the workload 33 | # on it 34 | 35 | bin=`dirname "$0"` 36 | bin=`cd "$bin"; pwd` 37 | 38 | # Export LINKBENCH_HOME so that LinkBench Java code can access env var 39 | export LINKBENCH_HOME=`dirname $bin` 40 | 41 | if [ "$JAVA_HOME" = "" ]; then 42 | JAVA=`which java` 43 | if [ ! -x $JAVA ]; then 44 | echo "Error: java not found, set JAVA_HOME or add java to PATH." 45 | exit 1 46 | fi 47 | else 48 | JAVA=$JAVA_HOME/bin/java 49 | fi 50 | 51 | echo "Using java at: $JAVA" 52 | 53 | JAVA_HEAP_MAX=-Xmx1000m 54 | 55 | # check envvars which might override default args 56 | if [ "$HEAPSIZE" != "" ]; then 57 | #echo "run with heapsize $HEAPSIZE" 58 | JAVA_HEAP_MAX="-Xmx""$HEAPSIZE""m" 59 | #echo $JAVA_HEAP_MAX 60 | fi 61 | 62 | # CLASSPATH initially contains $CONF_DIR 63 | CLASSPATH="${CONF_DIR}" 64 | CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar 65 | 66 | # so that filenames w/ spaces are handled correctly in loops below 67 | IFS= 68 | 69 | # add latest jar to CLASSPATH 70 | CLASSPATH=${CLASSPATH}:target/FacebookLinkBench.jar; 71 | 72 | # restore ordinary behaviour 73 | unset IFS 74 | 75 | # figure out which class to run 76 | CLASS='com.facebook.LinkBench.LinkBenchDriver' 77 | 78 | # run it 79 | exec "$JAVA" $JAVA_HEAP_MAX $OPTS $JMX_OPTS -classpath "$CLASSPATH" $CLASS \ 80 | "$@" 81 | -------------------------------------------------------------------------------- /config/LinkConfigMongoDb2.properties: -------------------------------------------------------------------------------- 1 | # Sample MongoDB LinkBench configuration file. 2 | # 3 | # This file contains settings for the data store, as well as controlling 4 | # benchmark output and behavior. The workload is defined in a separate 5 | # file. 6 | # 7 | # At a minimum to use this file, you will need to fill in MongoDB 8 | # connection information. 9 | 10 | ########################## 11 | # Workload Configuration # 12 | ########################## 13 | 14 | # Path for workload properties file. Properties in this file will override 15 | # those in workload properties file. 16 | # Can be absolute path, or relative path from LinkBench home directory 17 | workload_file = config/FBWorkload.properties 18 | 19 | ################################# 20 | # # 21 | # Data Source Configuration # 22 | # # 23 | ################################# 24 | 25 | # Implementation of LinkStore and NodeStore to use 26 | linkstore = com.facebook.LinkBench.LinkStoreMongoDb2 27 | nodestore = com.facebook.LinkBench.LinkStoreMongoDb2 28 | 29 | # MySQL connection information 30 | host = localhost 31 | user = root 32 | password = pw 33 | port = 27017 34 | # If a url is provided, it is used and host/port and also user/password are ignored. 35 | # Note that if you don't provide a url, the replicaSet option will default to "replset" and you must call your replica set exactly that. 36 | url = mongodb://localhost:27017/test?retryWrites=True&replicaSet=replset 37 | # Same with authentication: 38 | # url = mongodb://username:password@localhost:27017/test?retryWrites=True&replicaSet=replset 39 | # dbprefix: the database prefix to use 40 | dbprefix = linkdb 41 | # number of databases to use 42 | dbcount = 1 43 | # comma seperated list of integers to denote the load on each database 44 | # the sum must add up to 100, if empty, equal load to all DB assumed 45 | dbload = 46 | 47 | # database table names. 48 | # In MongoDB, "collections" are the equivalent of "tables" in SQL, but we 49 | # call them "tables" here to keep the naming conventions consistent across different database benchmarks. 50 | linktable = linktable 51 | # counttable not required for all databases 52 | counttable = counttable 53 | nodetable = nodetable 54 | 55 | ############################### 56 | # # 57 | # Logging and Stats Setup # 58 | # # 59 | ############################### 60 | 61 | # This controls logging output. Settings are, in order of increasing 62 | # verbosity: 63 | # ERROR: only output serious errors 64 | # WARN: output warnings 65 | # INFO: output additional information such as progress 66 | # DEBUG: output high-level debugging information 67 | # TRACE: output more detailed lower-level debugging information 68 | debuglevel = INFO 69 | 70 | # display frequency of per-thread progress in seconds 71 | progressfreq = 300 72 | 73 | # display frequency of per-thread stats (latency, etc) in seconds 74 | displayfreq = 1800 75 | 76 | # display global load update (% complete, etc) after this many links loaded 77 | load_progress_interval = 50000 78 | 79 | # display global update on request phase (% complete, etc) after this many ops 80 | req_progress_interval = 10000 81 | 82 | # max number of samples to store for each per-thread statistic 83 | maxsamples = 10000 84 | 85 | ############################### 86 | # # 87 | # Load Phase Configuration # 88 | # # 89 | ############################### 90 | 91 | # number of threads to run during load phase 92 | loaders = 10 93 | 94 | # whether to generate graph nodes during load process 95 | generate_nodes = true 96 | 97 | # partition loading work into chunks of id1s of this size 98 | loader_chunk_size = 2048 99 | 100 | # seed for initial data load random number generation (optional) 101 | # load_random_seed = 12345 102 | 103 | ################################## 104 | # # 105 | # Request Phase Configuration # 106 | # # 107 | ################################## 108 | 109 | # number of threads to run during request phase 110 | requesters = 20 111 | 112 | # read + write requests per thread 113 | requests = 500000 114 | 115 | # request rate per thread. <= 0 means unthrottled requests, > 0 limits 116 | # the average request rate to that number of requests per second per thread, 117 | # with the inter-request intervals governed by an exponential distribution 118 | requestrate = 0 119 | 120 | # max duration in seconds for request phase of benchmark 121 | maxtime = 100000 122 | 123 | # warmup time in seconds. The benchmark is run for a warmup period 124 | # during which no statistics are recorded. This allows database caches, 125 | # etc to warm up. 126 | warmup_time = 0 127 | 128 | # seed for request random number generation (optional) 129 | # request_random_seed = 12345 130 | 131 | # maximum number of failures per requester to tolerate before aborting 132 | # negative number means never abort 133 | max_failed_requests = 100 134 | 135 | ############################### 136 | # # 137 | # MongoDB Tuning # 138 | # # 139 | ############################### 140 | 141 | # Optional tuning parameters 142 | # 143 | # TODO: Add any useful tuning parameters. 144 | -------------------------------------------------------------------------------- /config/LinkConfigMongoDbBasic.properties: -------------------------------------------------------------------------------- 1 | # Sample MongoDB LinkBench configuration file. 2 | # 3 | # This config file uses LinkStoreMongoDbBasic. This is a basic port of 4 | # LinkStoreMysql to MongoDB. See LinkStoreMongoDb2 for a more optimized 5 | # version. 6 | # 7 | # Indexes for LinkStoreMongoDbBasic: 8 | # 9 | # use linkdb0 10 | # db.linktable.createIndex({id1: 1, link_type: 1, id2: 1}, {unique: true}); 11 | # db.linktable.createIndex({id1: 1, link_type: 1, time: 1, visibility: 1}); 12 | # db.counttable.createIndex({id: 1, link_type: 1}, {unique: true}); 13 | # db.nodetable.createIndex({id: 1}, {unique: true}); 14 | # 15 | # You need to create the above before running this. 16 | # 17 | # This file contains settings for the data store, as well as controlling 18 | # benchmark output and behavior. The workload is defined in a separate 19 | # file. 20 | # 21 | # At a minimum to use this file, you will need to fill in MongoDB 22 | # connection information. 23 | 24 | ########################## 25 | # Workload Configuration # 26 | ########################## 27 | 28 | # Path for workload properties file. Properties in this file will override 29 | # those in workload properties file. 30 | # Can be absolute path, or relative path from LinkBench home directory 31 | workload_file = config/FBWorkload.properties 32 | 33 | ################################# 34 | # # 35 | # Data Source Configuration # 36 | # # 37 | ################################# 38 | 39 | # Implementation of LinkStore and NodeStore to use 40 | linkstore = com.facebook.LinkBench.LinkStoreMongoDbBasic 41 | nodestore = com.facebook.LinkBench.LinkStoreMongoDbBasic 42 | 43 | # MongoDB connection information 44 | host = localhost 45 | user = linkbench 46 | password = linkbench 47 | port = 27017 48 | # If a url is provided, it is used and host/port and also user/password are ignored. 49 | # Note that if you don't provide a url, the replicaSet option will default to "replset" and you must call your replica set exactly that. 50 | url = mongodb://localhost:27017/test?retryWrites=True&replicaSet=replset 51 | # Same with authentication: 52 | # url = mongodb://username:password@localhost:27017/test?retryWrites=True&replicaSet=replset 53 | # dbprefix: the database prefix to use 54 | dbprefix = linkdb 55 | # number of databases to use 56 | dbcount = 1 57 | # comma seperated list of integers to denote the load on each database 58 | # the sum must add up to 100, if empty, equal load to all DB assumed 59 | dbload = 60 | 61 | # database table names. 62 | # In MongoDB, "collections" are the equivalent of "tables" in SQL, but we 63 | # call them "tables" here to keep the naming conventions consistent across different database benchmarks. 64 | linktable = linktable 65 | # counttable not required for all databases 66 | counttable = counttable 67 | nodetable = nodetable 68 | 69 | ############################### 70 | # # 71 | # Logging and Stats Setup # 72 | # # 73 | ############################### 74 | 75 | # This controls logging output. Settings are, in order of increasing 76 | # verbosity: 77 | # ERROR: only output serious errors 78 | # WARN: output warnings 79 | # INFO: output additional information such as progress 80 | # DEBUG: output high-level debugging information 81 | # TRACE: output more detailed lower-level debugging information 82 | debuglevel = INFO 83 | 84 | # display frequency of per-thread progress in seconds 85 | progressfreq = 300 86 | 87 | # display frequency of per-thread stats (latency, etc) in seconds 88 | displayfreq = 1800 89 | 90 | # display global load update (% complete, etc) after this many links loaded 91 | load_progress_interval = 50000 92 | 93 | # display global update on request phase (% complete, etc) after this many ops 94 | req_progress_interval = 10000 95 | 96 | # max number of samples to store for each per-thread statistic 97 | maxsamples = 10000 98 | 99 | ############################### 100 | # # 101 | # Load Phase Configuration # 102 | # # 103 | ############################### 104 | 105 | # number of threads to run during load phase 106 | loaders = 10 107 | 108 | # whether to generate graph nodes during load process 109 | generate_nodes = true 110 | 111 | # partition loading work into chunks of id1s of this size 112 | loader_chunk_size = 2048 113 | 114 | # seed for initial data load random number generation (optional) 115 | # load_random_seed = 12345 116 | 117 | ################################## 118 | # # 119 | # Request Phase Configuration # 120 | # # 121 | ################################## 122 | 123 | # number of threads to run during request phase 124 | requesters = 20 125 | 126 | # read + write requests per thread 127 | requests = 500000 128 | 129 | # request rate per thread. <= 0 means unthrottled requests, > 0 limits 130 | # the average request rate to that number of requests per second per thread, 131 | # with the inter-request intervals governed by an exponential distribution 132 | requestrate = 0 133 | 134 | # max duration in seconds for request phase of benchmark 135 | maxtime = 100000 136 | 137 | # warmup time in seconds. The benchmark is run for a warmup period 138 | # during which no statistics are recorded. This allows database caches, 139 | # etc to warm up. 140 | warmup_time = 0 141 | 142 | # seed for request random number generation (optional) 143 | # request_random_seed = 12345 144 | 145 | # maximum number of failures per requester to tolerate before aborting 146 | # negative number means never abort 147 | max_failed_requests = 100 148 | 149 | # In cases where Linkbench implementation evolved, stay with the legacy code path if available 150 | # The point is to guard repeatability over time 151 | never_change = true 152 | 153 | ############################### 154 | # # 155 | # MongoDB Tuning # 156 | # # 157 | ############################### 158 | 159 | # Optional tuning parameters 160 | # 161 | # TODO: Add any useful tuning parameters. 162 | 163 | -------------------------------------------------------------------------------- /config/LinkConfigMysql.properties: -------------------------------------------------------------------------------- 1 | # Sample MySQL LinkBench configuration file. 2 | # 3 | # This file contains settings for the data store, as well as controlling 4 | # benchmark output and behavior. The workload is defined in a separate 5 | # file. 6 | # 7 | # At a minimum to use this file, you will need to fill in MySQL 8 | # connection information. 9 | 10 | ########################## 11 | # Workload Configuration # 12 | ########################## 13 | 14 | # Path for workload properties file. Properties in this file will override 15 | # those in workload properties file. 16 | # Can be absolute path, or relative path from LinkBench home directory 17 | workload_file = config/FBWorkload.properties 18 | 19 | ################################# 20 | # # 21 | # Data Source Configuration # 22 | # # 23 | ################################# 24 | 25 | # Implementation of LinkStore and NodeStore to use 26 | linkstore = com.facebook.LinkBench.LinkStoreMysql 27 | nodestore = com.facebook.LinkBench.LinkStoreMysql 28 | 29 | # MySQL connection information 30 | host = yourhostname.here 31 | user = MySQLuser 32 | password = MySQLpass 33 | port = 3306 34 | # dbprefix: the database prefix to use 35 | dbprefix = linkdb 36 | # number of databases to use 37 | dbcount = 1 38 | # comma seperated list of integers to denote the load on each database 39 | # the sum must add up to 100, if empty, equal load to all DB assumed 40 | dbload = 41 | 42 | # database table names 43 | linktable = linktable 44 | # counttable not required for all databases 45 | counttable = counttable 46 | nodetable = nodetable 47 | 48 | ############################### 49 | # # 50 | # Logging and Stats Setup # 51 | # # 52 | ############################### 53 | 54 | # This controls logging output. Settings are, in order of increasing 55 | # verbosity: 56 | # ERROR: only output serious errors 57 | # WARN: output warnings 58 | # INFO: output additional information such as progress 59 | # DEBUG: output high-level debugging information 60 | # TRACE: output more detailed lower-level debugging information 61 | debuglevel = INFO 62 | 63 | # display frequency of per-thread progress in seconds 64 | progressfreq = 300 65 | 66 | # display frequency of per-thread stats (latency, etc) in seconds 67 | displayfreq = 1800 68 | 69 | # display global load update (% complete, etc) after this many links loaded 70 | load_progress_interval = 50000 71 | 72 | # display global update on request phase (% complete, etc) after this many ops 73 | req_progress_interval = 10000 74 | 75 | # max number of samples to store for each per-thread statistic 76 | maxsamples = 10000 77 | 78 | ############################### 79 | # # 80 | # Load Phase Configuration # 81 | # # 82 | ############################### 83 | 84 | # number of threads to run during load phase 85 | loaders = 10 86 | 87 | # whether to generate graph nodes during load process 88 | generate_nodes = true 89 | 90 | # partition loading work into chunks of id1s of this size 91 | loader_chunk_size = 2048 92 | 93 | # seed for initial data load random number generation (optional) 94 | # load_random_seed = 12345 95 | 96 | ################################## 97 | # # 98 | # Request Phase Configuration # 99 | # # 100 | ################################## 101 | 102 | # number of threads to run during request phase 103 | requesters = 100 104 | 105 | # read + write requests per thread 106 | requests = 500000 107 | 108 | # request rate per thread. <= 0 means unthrottled requests, > 0 limits 109 | # the average request rate to that number of requests per second per thread, 110 | # with the inter-request intervals governed by an exponential distribution 111 | requestrate = 0 112 | 113 | # max duration in seconds for request phase of benchmark 114 | maxtime = 100000 115 | 116 | # warmup time in seconds. The benchmark is run for a warmup period 117 | # during which no statistics are recorded. This allows database caches, 118 | # etc to warm up. 119 | warmup_time = 0 120 | 121 | # seed for request random number generation (optional) 122 | # request_random_seed = 12345 123 | 124 | # maximum number of failures per requester to tolerate before aborting 125 | # negative number means never abort 126 | max_failed_requests = 100 127 | 128 | ############################### 129 | # # 130 | # MySQL Tuning # 131 | # # 132 | ############################### 133 | 134 | # Optional tuning parameters 135 | 136 | # # of link inserts to batch together when loading 137 | # MySQL_bulk_insert_batch = 1024 138 | 139 | # optional tuning - disable binary logging during load phase 140 | # WARNING: do not use unless you know what you are doing, it can 141 | # break replication amongst other things 142 | # MySQL_disable_binlog_load = true 143 | 144 | -------------------------------------------------------------------------------- /config/LinkConfigPgsql.properties: -------------------------------------------------------------------------------- 1 | # Sample MySQL LinkBench configuration file. 2 | # 3 | # This file contains settings for the data store, as well as controlling 4 | # benchmark output and behavior. The workload is defined in a separate 5 | # file. 6 | # 7 | # At a minimum to use this file, you will need to fill in MySQL 8 | # connection information. 9 | 10 | ########################## 11 | # Workload Configuration # 12 | ########################## 13 | 14 | # Path for workload properties file. Properties in this file will override 15 | # those in workload properties file. 16 | # Can be absolute path, or relative path from LinkBench home directory 17 | workload_file = config/FBWorkload.properties 18 | 19 | ################################# 20 | # # 21 | # Data Source Configuration # 22 | # # 23 | ################################# 24 | 25 | # Implementation of LinkStore and NodeStore to use 26 | linkstore = com.facebook.LinkBench.LinkStorePgsql 27 | nodestore = com.facebook.LinkBench.LinkStorePgsql 28 | 29 | # MySQL connection information 30 | host = localhost 31 | user = postgres 32 | password = postgres 33 | port = 5432 34 | # dbprefix: the database prefix to use 35 | dbprefix = linkdb 36 | # number of databases to use 37 | dbcount = 1 38 | # comma seperated list of integers to denote the load on each database 39 | # the sum must add up to 100, if empty, equal load to all DB assumed 40 | dbload = 41 | 42 | # database table names 43 | linktable = linktable 44 | # counttable not required for all databases 45 | counttable = counttable 46 | nodetable = nodetable 47 | 48 | ############################### 49 | # # 50 | # Logging and Stats Setup # 51 | # # 52 | ############################### 53 | 54 | # This controls logging output. Settings are, in order of increasing 55 | # verbosity: 56 | # ERROR: only output serious errors 57 | # WARN: output warnings 58 | # INFO: output additional information such as progress 59 | # DEBUG: output high-level debugging information 60 | # TRACE: output more detailed lower-level debugging information 61 | debuglevel = INFO 62 | 63 | # display frequency of per-thread progress in seconds 64 | progressfreq = 300 65 | 66 | # display frequency of per-thread stats (latency, etc) in seconds 67 | displayfreq = 1800 68 | 69 | # display global load update (% complete, etc) after this many links loaded 70 | load_progress_interval = 50000 71 | 72 | # display global update on request phase (% complete, etc) after this many ops 73 | req_progress_interval = 10000 74 | 75 | # max number of samples to store for each per-thread statistic 76 | maxsamples = 10000 77 | 78 | ############################### 79 | # # 80 | # Load Phase Configuration # 81 | # # 82 | ############################### 83 | 84 | # number of threads to run during load phase 85 | loaders = 4 86 | 87 | # whether to generate graph nodes during load process 88 | generate_nodes = true 89 | 90 | # partition loading work into chunks of id1s of this size 91 | loader_chunk_size = 2048 92 | 93 | # seed for initial data load random number generation (optional) 94 | # load_random_seed = 12345 95 | 96 | ################################## 97 | # # 98 | # Request Phase Configuration # 99 | # # 100 | ################################## 101 | 102 | # number of threads to run during request phase 103 | requesters = 8 104 | 105 | # read + write requests per thread 106 | requests = 500000 107 | 108 | # request rate per thread. <= 0 means unthrottled requests, > 0 limits 109 | # the average request rate to that number of requests per second per thread, 110 | # with the inter-request intervals governed by an exponential distribution 111 | requestrate = 0 112 | 113 | # max duration in seconds for request phase of benchmark 114 | maxtime = 100000 115 | 116 | # warmup time in seconds. The benchmark is run for a warmup period 117 | # during which no statistics are recorded. This allows database caches, 118 | # etc to warm up. 119 | warmup_time = 0 120 | 121 | # seed for request random number generation (optional) 122 | # request_random_seed = 12345 123 | 124 | # maximum number of failures per requester to tolerate before aborting 125 | # negative number means never abort 126 | max_failed_requests = 100 127 | 128 | ############################### 129 | # # 130 | # PostgreSQL Tuning # 131 | # # 132 | ############################### 133 | 134 | # Optional tuning parameters 135 | 136 | -------------------------------------------------------------------------------- /config/LinkConfigRocksDb.properties: -------------------------------------------------------------------------------- 1 | # Sample RocksDb LinkBench configuration file. 2 | # 3 | # This file contains settings for the data store, as well as controlling 4 | # benchmark output and behavior. The workload is defined in a separate 5 | # file. 6 | # 7 | 8 | ########################## 9 | # Workload Configuration # 10 | ########################## 11 | 12 | # Path for workload properties file. Properties in this file will override 13 | # those in workload properties file. 14 | # Can be absolute path, or relative path from LinkBench home directory 15 | workload_file = config/FBWorkload.properties 16 | 17 | ################################# 18 | # # 19 | # Data Source Configuration # 20 | # # 21 | ################################# 22 | 23 | # Implementation of LinkStore and NodeStore to use 24 | linkstore = com.facebook.LinkBench.LinkStoreRocksDb 25 | nodestore = com.facebook.LinkBench.LinkStoreRocksDb 26 | 27 | # RocksDb connection information 28 | host = yourhostname.here 29 | port = 9090 30 | # dbprefix: the database prefix to use 31 | dbprefix = linkdb 32 | # number of databases to use 33 | dbcount = 1 34 | # comma seperated list of integers to denote the load on each database 35 | # the sum must add up to 100, if empty, equal load to all DB assumed 36 | dbload = 37 | 38 | ############################### 39 | # # 40 | # Logging and Stats Setup # 41 | # # 42 | ############################### 43 | 44 | # This controls logging output. Settings are, in order of increasing 45 | # verbosity: 46 | # ERROR: only output serious errors 47 | # WARN: output warnings 48 | # INFO: output additional information such as progress 49 | # DEBUG: output high-level debugging information 50 | # TRACE: output more detailed lower-level debugging information 51 | debuglevel = INFO 52 | 53 | # display frequency of per-thread progress in seconds 54 | progressfreq = 300 55 | 56 | # display frequency of per-thread stats (latency, etc) in seconds 57 | displayfreq = 1800 58 | 59 | # display global load update (% complete, etc) after this many links loaded 60 | load_progress_interval = 50000 61 | 62 | # display global update on request phase (% complete, etc) after this many ops 63 | req_progress_interval = 10000 64 | 65 | # max number of samples to store for each per-thread statistic 66 | maxsamples = 10000 67 | 68 | ############################### 69 | # # 70 | # Load Phase Configuration # 71 | # # 72 | ############################### 73 | 74 | # number of threads to run during load phase 75 | loaders = 10 76 | 77 | # whether to generate graph nodes during load process 78 | generate_nodes = true 79 | 80 | # partition loading work into chunks of id1s of this size 81 | loader_chunk_size = 2048 82 | 83 | # seed for initial data load random number generation (optional) 84 | # load_random_seed = 12345 85 | 86 | ################################## 87 | # # 88 | # Request Phase Configuration # 89 | # # 90 | ################################## 91 | 92 | # number of threads to run during request phase 93 | requesters = 100 94 | 95 | # read + write requests per thread 96 | requests = 500000 97 | 98 | # request rate per thread. <= 0 means unthrottled requests, > 0 limits 99 | # the average request rate to that number of requests per second per thread, 100 | # with the inter-request intervals governed by an exponential distribution 101 | requestrate = 0 102 | 103 | # max duration in seconds for request phase of benchmark 104 | maxtime = 100000 105 | 106 | # warmup time in seconds. The benchmark is run for a warmup period 107 | # during which no statistics are recorded. This allows database caches, 108 | # etc to warm up. 109 | warmup_time = 0 110 | 111 | # seed for request random number generation (optional) 112 | # request_random_seed = 12345 113 | 114 | # maximum number of failures per requester to tolerate before aborting 115 | # negative number means never abort 116 | max_failed_requests = 100 117 | -------------------------------------------------------------------------------- /docs/ftdc_00e7bb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdcallag/linkbench/9ef9cc8785dbd0921abd0bbfea971680a493a4fe/docs/ftdc_00e7bb.png -------------------------------------------------------------------------------- /docs/statistics.md: -------------------------------------------------------------------------------- 1 | # Linkbench statistics 2 | 3 | ## LOAD phase 4 | 5 | These are notes from reverse engineering for myself what the load phase does and what the statistics 6 | mean. This is for the LinkStoreMongoDb, but should be true for any other store as well. (The example 7 | output is for revision 00e7bb4f5e46be31f371964186a31ac613a13a1c in mongodb/mongo repo, but our perf 8 | testing results aren't currently public.) 9 | 10 | ### What it does 11 | 12 | The load phase loads all 3 collections in parallel. However, the nodetable is populated by a single 13 | thread in order to generate a sequence of id1 values sequentially. So when we run with loaders=20, 14 | the 20 threads are generating counttable and linktable, and 1 additional thread is generating 15 | nodetable. This is why the csv table has 21 in the threads column, it is true. 16 | 17 | The nodetable is populated with inserts while the linktable and counttable are upserts. Upserts are 18 | updates in FTDC data. So we can now interpret the graph: 19 | 20 | ![Graph of MongoDB diagnostic data](ftdc_00e7bb.png) 21 | 22 | The red until 10:55:00 is inserts, so that's the thread filling the nodetable at about 25k 23 | insert/sec. During this time there are about 33k updates/sec (yellow). At 11:10:47 all loader 24 | threads except for Thread-0 have finished. That last thread continues until 11:25:20. Even if 15 25 | minutes is a lot, this last thread is only responsible for 0.02 % of the total amount of links. 26 | (...if we were to trust what it reports, which I don't necessarily do yet.) It must be some kind of 27 | remainder from how the id1 space is split up. Obviously MongoDB is not saturated at this point 28 | (average latency drops from 500ms to 200ms), rather the slowness must be inherent to the client. 29 | 30 | ### Understanding the results 31 | 32 | When each thread finishes working it prints out its own statistics. The last line is: 33 | 34 | INFO 2018-08-06 11:10:47,449 [Thread-11]: 9997952/10000000 id1s loaded (100.0% complete) at 7043.75 id1s/sec avg. 39929251 links loaded at 28130.94 links/sec avg. 35 | 36 | This can be misleading: Note that the nodetable has been populated a long time ago. The "id1s 37 | loaded" stat refers to the id1 range for which this thread has created links and counts. 38 | 39 | At the end, the main thread prints summary statistics. This line is straightforward to read: nr of 40 | links / total time: 41 | 42 | INFO 2018-08-06 11:25:21,376 [main]: LOAD PHASE COMPLETED. Loaded 10000000 nodes (Expected 10000000). Loaded 44118052 links (4.41 links per node). Took 2293.3 seconds. Links/second = 19238 43 | 44 | Note that the throughput is only regarding links/second and ignores the parallel work that happened 45 | to insert nodes and counts. Also note that the total time includes the 15 minutes from the last 46 | straggling thread. So this statistic maybe tells something about linkbench, but does not say much 47 | about mongodb writes/sec. We will continue to ignore it for MongoDB perf testing purposes. 48 | 49 | The main thread also prints these statistics, which are the ones that are output also as CSV and 50 | used by us in Evergreen graphs: 51 | 52 | INFO 2018-08-06 11:25:21,373 [main]: LOAD_NODE_BULK count = 9766 p25 = [10000,100000]ms p50 = [10000,100000]ms p75 = [10000,100000]ms p95 = [10000,100000]ms p99 = [10000,100000]ms max = 187266.326ms mean = 37250.397ms threads = 21 53 | INFO 2018-08-06 11:25:21,374 [main]: LOAD_LINKS_BULK count = 43093 p25 = [10000,100000]ms p50 = [10000,100000]ms p75 = [10000,100000]ms p95 = [10000,100000]ms p99 = [10000,100000]ms max = 14466876.608ms mean = 569744.311ms threads = 21 54 | INFO 2018-08-06 11:25:21,374 [main]: LOAD_COUNTS_BULK count = 11754 p25 = [10000,100000]ms p50 = [10000,100000]ms p75 = [10000,100000]ms p95 = [10000,100000]ms p99 = [10000,100000]ms max = 760687.825ms mean = 392135.426ms threads = 21 55 | 56 | The mean is the mean response time for a bulk write = for the entire batch. We use 1024 documents / 57 | batch. (The default for LinkStoreMongoDb.) Note however that the last batch for each thread is then 58 | incomplete, and has less than 1024 docs. If my math is right, and work is distributed evenly to 59 | begin with, in our case the last batch is 100M/20 mod 1024 = 288. So it's of course completing 60 | faster, but the latency for this smaller bulk write is added to the mean just like any other. As 61 | there are hundreds of 1024 sized batches per thread, this is an ignorable error. 62 | 63 | LOAD_LINKS_BULK mean latency is 569744.311ms. This translates to 556 ms / document. This is 64 | suspiciously slow, but the ftdc data seems to confirm that this is indeed the average latency for 65 | writes. Possibly 1024 is too large value for batches here, especially as we have 21 parallel 66 | threads. -------------------------------------------------------------------------------- /mongo-setup.js: -------------------------------------------------------------------------------- 1 | // 2 | // create the required databases and indexes 3 | // 4 | // use like: 5 | // mongo mongo-setup.js # for single linkdb0 database 6 | // mongo --eval 'var numdbs=10;' mongo-setup.js # for linkdb0 .. linkdb9 databases 7 | // 8 | if (typeof(numdbs) === 'undefined') { 9 | numdbs = 1; 10 | } 11 | for(var i =0; i < numdbs; i++) { 12 | db = db.getSiblingDB('linkdb' + i); 13 | db.createCollection("linktable"); 14 | db.createCollection("nodetable"); 15 | db.createCollection("counttable"); 16 | 17 | db.linktable.createIndex({id1: 1, link_type: 1, time: 1, visibility: 1}); 18 | db.counttable.createIndex({id1: 1, link_type: 1}) 19 | } 20 | -------------------------------------------------------------------------------- /pgsql-ddl.sql: -------------------------------------------------------------------------------- 1 | --create database for linkbench 2 | 3 | drop database if exists linkdb0; 4 | create database linkdb0 encoding='latin1' ; 5 | 6 | --drop user linkbench to create new one 7 | DROP USER IF EXISTS linkbench; 8 | 9 | -- You may want to set up a special database user account for benchmarking: 10 | CREATE USER linkbench password 'linkbench'; 11 | -- Grant all privileges on linkdb0 to this user 12 | GRANT ALL ON database linkdb0 TO linkbench; 13 | 14 | --add Schema keep the same query style 15 | DROP SCHEMA IF EXISTS linkdb0 CASCADE; 16 | CREATE SCHEMA linkdb0; 17 | 18 | --conn postgresql linkbench/password 19 | 20 | --FIXME:Need to make it partitioned by key id1 %16 21 | CREATE TABLE linkdb0.linktable ( 22 | id1 numeric(20) NOT NULL DEFAULT '0', 23 | id2 numeric(20) NOT NULL DEFAULT '0', 24 | link_type numeric(20) NOT NULL DEFAULT '0', 25 | visibility smallint NOT NULL DEFAULT '0', 26 | data varchar(255) NOT NULL DEFAULT '', 27 | time numeric(20) NOT NULL DEFAULT '0', 28 | version bigint NOT NULL DEFAULT '0', 29 | PRIMARY KEY (link_type, id1,id2) 30 | ); 31 | 32 | -- this is index for linktable 33 | CREATE INDEX id1_type on linkdb0.linktable(id1,link_type,visibility,time,id2,version,data); 34 | 35 | CREATE TABLE linkdb0.counttable ( 36 | id numeric(20) NOT NULL DEFAULT '0', 37 | link_type numeric(20) NOT NULL DEFAULT '0', 38 | count int NOT NULL DEFAULT '0', 39 | time numeric(20) NOT NULL DEFAULT '0', 40 | version numeric(20) NOT NULL DEFAULT '0', 41 | PRIMARY KEY (id,link_type) 42 | ); 43 | 44 | CREATE TABLE linkdb0.nodetable ( 45 | id BIGSERIAL NOT NULL, 46 | type int NOT NULL, 47 | version numeric NOT NULL, 48 | time int NOT NULL, 49 | data text NOT NULL, 50 | PRIMARY KEY(id) 51 | ); 52 | 53 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/ConfigUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.util.Properties; 21 | 22 | public class ConfigUtil { 23 | public static final String linkbenchHomeEnvVar = "LINKBENCH_HOME"; 24 | 25 | /** 26 | * @return null if not set, or if not valid path 27 | */ 28 | public static String findLinkBenchHome() { 29 | String linkBenchHome = System.getenv("LINKBENCH_HOME"); 30 | if (linkBenchHome != null && linkBenchHome.length() > 0) { 31 | File dir = new File(linkBenchHome); 32 | if (dir.exists() && dir.isDirectory()) { 33 | return linkBenchHome; 34 | } 35 | } 36 | return null; 37 | } 38 | 39 | public static Level getDebugLevel(Properties props) throws LinkBenchConfigError { 40 | if (props == null) { 41 | return Level.INFO; 42 | } 43 | String levStr = props.getProperty(Config.DEBUGLEVEL); 44 | 45 | if (levStr == null) { 46 | return Level.INFO; 47 | } 48 | 49 | Level l = Level.toLevel(levStr); 50 | if (l != null) { 51 | return l; 52 | } else { 53 | throw new LinkBenchConfigError("Invalid setting for debug level: " + levStr); 54 | } 55 | } 56 | 57 | public static void setupLogging(Properties props, String logFile) 58 | throws LinkBenchConfigError, IOException { 59 | Level logLevel = ConfigUtil.getDebugLevel(props); 60 | // System.err.println("setupLogging level is " + logLevel); 61 | Logger.getLogger().setLevel(logLevel); 62 | } 63 | 64 | /** 65 | * Look up key in props, failing if not present 66 | * @param props 67 | * @param key 68 | * @return 69 | * @throws LinkBenchConfigError thrown if key not present 70 | */ 71 | public static String getPropertyRequired(Properties props, String key) 72 | throws LinkBenchConfigError { 73 | String v = props.getProperty(key); 74 | if (v == null) { 75 | throw new LinkBenchConfigError("Expected configuration key " + key + 76 | " to be defined"); 77 | } 78 | return v; 79 | } 80 | 81 | public static String getString(Properties props, String key) { 82 | return props.getProperty(key); 83 | } 84 | 85 | public static int getInt(Properties props, String key) 86 | throws LinkBenchConfigError { 87 | return getInt(props, key, null); 88 | } 89 | 90 | /** 91 | * Retrieve a config key and convert to integer 92 | * @param props 93 | * @param key 94 | * @return a non-null string value 95 | * @throws LinkBenchConfigError if not present or not integer 96 | */ 97 | public static int getInt(Properties props, String key, Integer defaultVal) 98 | throws LinkBenchConfigError { 99 | if (defaultVal != null && !props.containsKey(key)) { 100 | return defaultVal; 101 | } 102 | String v = getPropertyRequired(props, key); 103 | try { 104 | return Integer.parseInt(v); 105 | } catch (NumberFormatException e) { 106 | throw new LinkBenchConfigError("Expected configuration key " + key + 107 | " to be integer, but was '" + v + "'"); 108 | } 109 | } 110 | 111 | public static long getLong(Properties props, String key) 112 | throws LinkBenchConfigError { 113 | return getLong(props, key, null); 114 | } 115 | 116 | /** 117 | * Retrieve a config key and convert to long integer 118 | * @param props 119 | * @param key 120 | * @param defaultVal default value if key not present 121 | * @return 122 | * @throws LinkBenchConfigError if not present or not integer 123 | */ 124 | public static long getLong(Properties props, String key, Long defaultVal) 125 | throws LinkBenchConfigError { 126 | if (defaultVal != null && !props.containsKey(key)) { 127 | return defaultVal; 128 | } 129 | String v = getPropertyRequired(props, key); 130 | try { 131 | return Long.parseLong(v); 132 | } catch (NumberFormatException e) { 133 | throw new LinkBenchConfigError("Expected configuration key " + key + 134 | " to be long integer, but was '" + v + "'"); 135 | } 136 | } 137 | 138 | 139 | public static double getDouble(Properties props, String key) 140 | throws LinkBenchConfigError { 141 | return getDouble(props, key, null); 142 | } 143 | 144 | /** 145 | * Retrieve a config key and convert to double 146 | * @param props 147 | * @param key 148 | * @param defaultVal default value if key not present 149 | * @return 150 | * @throws LinkBenchConfigError if not present or not double 151 | */ 152 | public static double getDouble(Properties props, String key, 153 | Double defaultVal) throws LinkBenchConfigError { 154 | if (defaultVal != null && !props.containsKey(key)) { 155 | return defaultVal; 156 | } 157 | String v = getPropertyRequired(props, key); 158 | try { 159 | return Double.parseDouble(v); 160 | } catch (NumberFormatException e) { 161 | throw new LinkBenchConfigError("Expected configuration key " + key + 162 | " to be double, but was '" + v + "'"); 163 | } 164 | } 165 | 166 | /** 167 | * Retrieve a config key and convert to boolean. 168 | * Valid boolean strings are "true" or "false", case insensitive 169 | * @param props 170 | * @param key 171 | * @return 172 | * @throws LinkBenchConfigError if not present or not boolean 173 | */ 174 | public static boolean getBool(Properties props, String key) 175 | throws LinkBenchConfigError { 176 | String v = getPropertyRequired(props, key).trim().toLowerCase(); 177 | // Parse manually since parseBoolean accepts many things as "false" 178 | if (v.equals("true")) { 179 | return true; 180 | } else if (v.equals("false")) { 181 | return false; 182 | } else { 183 | throw new LinkBenchConfigError("Expected configuration key " + key + 184 | " to be true or false, but was '" + v + "'"); 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/InvertibleShuffler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Random; 19 | 20 | /** 21 | * Shuffler designed to make computing permutation and inverse easy 22 | */ 23 | public class InvertibleShuffler { 24 | private final long[] params; 25 | private final int shuffleGroups; 26 | long n; 27 | long nRoundedUp; // n rounded up to next multiple of shuffleGroups 28 | long nRoundedDown; // n rounded down to next multiple of shuffleGroups 29 | int minGroupSize; 30 | 31 | public InvertibleShuffler(long seed, int shuffleGroups, long n) { 32 | this(new Random(seed), shuffleGroups, n); 33 | } 34 | public InvertibleShuffler(Random rng, int shuffleGroups, long n) { 35 | if (shuffleGroups > n) { 36 | // Can't have more shuffle groups than items 37 | shuffleGroups = (int)n; 38 | } 39 | this.shuffleGroups = shuffleGroups; 40 | this.n = n; 41 | this.params = new long[shuffleGroups]; 42 | this.minGroupSize = (int)n / shuffleGroups; 43 | 44 | for (int i = 0; i < shuffleGroups; i++) { 45 | // Positive long 46 | params[i] = Math.abs(rng.nextInt(minGroupSize)); 47 | } 48 | this.nRoundedDown = (n / shuffleGroups) * shuffleGroups; 49 | this.nRoundedUp = n == nRoundedDown ? n : nRoundedDown + shuffleGroups; 50 | } 51 | 52 | public long permute(long i) { 53 | return permute(i, false); 54 | } 55 | 56 | public long invertPermute(long i) { 57 | return permute(i, true); 58 | } 59 | 60 | public long permute(long i, boolean inverse) { 61 | if (i < 0 || i >= n) { 62 | throw new IllegalArgumentException("Bad index to permute: " + i 63 | + ": out of range [0:" + (n - 1) + "]"); 64 | } 65 | // Number of the group 66 | int group = (int) (i % shuffleGroups); 67 | 68 | // Whether this is a big or small group 69 | boolean bigGroup = group < n % shuffleGroups; 70 | 71 | // Calculate the (positive) rotation 72 | long rotate = params[group]; 73 | if (inverse) { 74 | // Reverse the rotation 75 | if (bigGroup) { 76 | rotate = minGroupSize + 1 - rotate; 77 | } else { 78 | rotate = minGroupSize - rotate; 79 | } 80 | assert(rotate >= 0); 81 | } 82 | 83 | long j = (i + shuffleGroups * rotate); 84 | long result; 85 | if (j < n) { 86 | result = j; 87 | } else { 88 | // Depending on the group there might be different numbers of 89 | // ids in the ring 90 | if (bigGroup) { 91 | result = j % nRoundedUp; 92 | } else { 93 | result = j % nRoundedDown; 94 | } 95 | if (result >= n) { 96 | result = group; 97 | } 98 | } 99 | assert(result % shuffleGroups == group); 100 | return result; 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Level.java: -------------------------------------------------------------------------------- 1 | 2 | package com.facebook.LinkBench; 3 | 4 | public enum Level { 5 | TRACE, DEBUG, INFO, WARN, ERROR; 6 | 7 | public boolean isGreaterOrEqual(Level o) { 8 | int res = compareTo(o); 9 | // System.out.println("Compare self(" + this + ") with (" + o + ") returns " + (res >= 0)); 10 | return res >= 0; 11 | } 12 | 13 | public static Level toLevel(String s) { 14 | if (s.equalsIgnoreCase("TRACE")) 15 | return TRACE; 16 | else if (s.equalsIgnoreCase("DEBUG")) 17 | return DEBUG; 18 | else if (s.equalsIgnoreCase("INFO")) 19 | return INFO; 20 | else if (s.equalsIgnoreCase("WARN")) 21 | return WARN; 22 | else if (s.equalsIgnoreCase("ERROR")) 23 | return ERROR; 24 | else 25 | return null; 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Link.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Arrays; 19 | 20 | 21 | public class Link { 22 | 23 | public Link(long id1, long link_type, long id2, 24 | byte visibility, byte[] data, int version, long time) { 25 | this.id1 = id1; 26 | this.link_type = link_type; 27 | this.id2 = id2; 28 | this.visibility = visibility; 29 | this.data = data; 30 | assert data.length <= 255 : "data.length must be <= 255"; 31 | this.version = version; 32 | this.time = time; 33 | } 34 | 35 | Link() { 36 | link_type = LinkStore.DEFAULT_LINK_TYPE; 37 | visibility = LinkStore.VISIBILITY_DEFAULT; 38 | } 39 | 40 | public boolean equals(Object other) { 41 | if (other instanceof Link) { 42 | Link o = (Link) other; 43 | return id1 == o.id1 && id2 == o.id2 && 44 | link_type == o.link_type && 45 | visibility == o.visibility && 46 | version == o.version && time == o.time && 47 | Arrays.equals(data, o.data); 48 | } else { 49 | return false; 50 | } 51 | } 52 | 53 | public String toString() { 54 | return String.format("Link(id1=%d, id2=%d, link_type=%d," + 55 | "visibility=%d, version=%d," + 56 | "time=%d, data=%s", id1, id2, link_type, 57 | visibility, version, time, data.toString()); 58 | } 59 | 60 | /** 61 | * Clone an existing link 62 | * @param l 63 | */ 64 | public Link clone() { 65 | Link l = new Link(); 66 | l.id1 = this.id1; 67 | l.link_type = this.link_type; 68 | l.id2 = this.id2; 69 | l.visibility = this.visibility; 70 | l.data = this.data.clone(); 71 | l.version = this.version; 72 | l.time = this.time; 73 | return l; 74 | } 75 | 76 | /** The node id of the source of directed edge */ 77 | public long id1; 78 | 79 | /** The node id of the target of directed edge */ 80 | public long id2; 81 | 82 | /** Type of link */ 83 | public long link_type; 84 | 85 | /** Visibility mode */ 86 | public byte visibility; 87 | 88 | /** Version of link */ 89 | public int version; 90 | 91 | /** time is the sort key for links. Often it contains a timestamp, 92 | but it can be used as a arbitrary user-defined sort key. */ 93 | public long time; 94 | 95 | /** Arbitrary payload data */ 96 | public byte[] data; 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkBenchConfigError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | public class LinkBenchConfigError extends RuntimeException { 19 | private static final long serialVersionUID = 1L; 20 | 21 | public LinkBenchConfigError(String msg) { 22 | super(msg); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkBenchOp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | // Various operation types for which we want to gather stats 19 | public enum LinkBenchOp { 20 | ADD_NODE, 21 | UPDATE_NODE, 22 | DELETE_NODE, 23 | GET_NODE, 24 | ADD_LINK, 25 | DELETE_LINK, 26 | UPDATE_LINK, 27 | COUNT_LINK, 28 | MULTIGET_LINK, 29 | GET_LINKS_LIST, 30 | LOAD_NODE_BULK, 31 | LOAD_LINK, 32 | LOAD_LINKS_BULK, 33 | LOAD_COUNTS_BULK, 34 | // Although the following are not truly operations, we need stats 35 | // for them 36 | RANGE_SIZE, // how big range scans are 37 | LOAD_LINKS_BULK_NLINKS, // how many links inserted in bulk 38 | LOAD_COUNTS_BULK_NLINKS, // how many counts inserted in bulk 39 | UNKNOWN; 40 | 41 | public String displayName() { 42 | return name(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkBenchTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | /** 19 | * The same as runnable, except run() can throw Exceptions 20 | * to be handled by the caller. 21 | */ 22 | public interface LinkBenchTask { 23 | public abstract void run() throws Exception; 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | public class LinkCount { 19 | 20 | public final long id1; 21 | public final long link_type; 22 | public long time; 23 | public long version; 24 | public long count; 25 | public LinkCount(long id1, long link_type, 26 | long time, long version, long init_count) { 27 | super(); 28 | this.id1 = id1; 29 | this.link_type = link_type; 30 | this.time = time; 31 | this.version = version; 32 | this.count = init_count; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.*; 19 | import java.sql.SQLException; 20 | import java.util.*; 21 | 22 | public abstract class LinkStore { 23 | // void createLinkTable(); 24 | public static final long DEFAULT_LINK_TYPE = 123456789; 25 | public static final long MAX_ID2 = Long.MAX_VALUE; 26 | public static final int DEFAULT_NODE_TYPE = 2048; 27 | 28 | // visibility 29 | public static final byte VISIBILITY_HIDDEN = 0; 30 | public static final byte VISIBILITY_DEFAULT = 1; 31 | public static final byte VISIBILITY_NOT_FOUND = 2; 32 | 33 | public static final int MAX_OPTYPES = LinkBenchOp.values().length; 34 | public static final int DEFAULT_LIMIT = 10000; 35 | 36 | public static final long MAX_LINK_DATA = 255; 37 | 38 | /** Controls the current setting for range limit */ 39 | protected int rangeLimit; 40 | 41 | /** The default constructor */ 42 | public LinkStore() { 43 | this.rangeLimit = DEFAULT_LIMIT; 44 | } 45 | 46 | public int getRangeLimit() { 47 | return rangeLimit; 48 | } 49 | 50 | public void setRangeLimit(int rangeLimit) { 51 | this.rangeLimit = rangeLimit; 52 | } 53 | 54 | /** initialize the store object */ 55 | public abstract void initialize(Properties p, Phase currentPhase, int threadId); 56 | 57 | /** 58 | * Do any cleanup. After this is called, store won't be reused 59 | */ 60 | public abstract void close(); 61 | 62 | // this is invoked when an error happens in case connection needs to be 63 | // cleaned up, reset, reopened, whatever 64 | public abstract void clearErrors(int threadID); 65 | 66 | /** 67 | * Add provided link to the store. If already exists, update with new data 68 | * @param dbid 69 | * @param a 70 | * @param noinverse 71 | * @return true if new link added, false if updated. Implementation is 72 | * optional, for informational purposes only. 73 | * @throws Exception 74 | */ 75 | public abstract LinkWriteResult addLink(String dbid, Link a, boolean noinverse) throws Exception; 76 | 77 | /** 78 | * Delete link identified by parameters from store 79 | * @param dbid 80 | * @param id1 81 | * @param link_type 82 | * @param id2 83 | * @param noinverse 84 | * @param expunge if true, delete permanently. If false, hide instead 85 | * @return true if row existed. Implementation is optional, for informational 86 | * purposes only. 87 | * @throws Exception 88 | */ 89 | public abstract boolean deleteLink(String dbid, long id1, long link_type, 90 | long id2, boolean noinverse, boolean expunge) throws Exception; 91 | 92 | /** 93 | * Update a link in the database, or add if not found 94 | * @param dbid 95 | * @param a 96 | * @param noinverse 97 | * @return true if link found, false if new link created. Implementation is 98 | * optional, for informational purposes only. 99 | * @throws Exception 100 | */ 101 | public abstract LinkWriteResult updateLink(String dbid, Link a, boolean noinverse) 102 | throws Exception; 103 | 104 | /** 105 | * lookup using id1, type, id2 106 | * Returns hidden links. 107 | * @param dbid 108 | * @param id1 109 | * @param link_type 110 | * @param id2 111 | * @return 112 | * @throws Exception 113 | */ 114 | public abstract Link getLink(String dbid, long id1, long link_type, long id2) 115 | throws Exception; 116 | 117 | /** 118 | * Lookup multiple links: same as getlink but retrieve 119 | * multiple ids 120 | * @return list of matching links found, in any order 121 | */ 122 | public Link[] multigetLinks(String dbid, long id1, long link_type, 123 | long id2s[]) 124 | throws Exception { 125 | // Default implementation 126 | ArrayList res = new ArrayList(id2s.length); 127 | for (int i = 0; i < id2s.length; i++) { 128 | Link l = getLink(dbid, id1, link_type, id2s[i]); 129 | if (l != null) { 130 | res.add(l); 131 | } 132 | } 133 | return res.toArray(new Link[res.size()]); 134 | } 135 | 136 | /** 137 | * lookup using just id1, type 138 | * Does not return hidden links 139 | * @param dbid 140 | * @param id1 141 | * @param link_type 142 | * @return list of links in descending order of time, or null 143 | * if no matching links 144 | * @throws Exception 145 | */ 146 | public abstract Link[] getLinkList(String dbid, long id1, long link_type) 147 | throws Exception; 148 | 149 | 150 | /** 151 | * lookup using just id1, type 152 | * Does not return hidden links 153 | * @param dbid 154 | * @param id1 155 | * @param link_type 156 | * @param minTimestamp 157 | * @param maxTimestamp 158 | * @param offset 159 | * @param limit 160 | * @return list of links in descending order of time, or null 161 | * if no matching links 162 | * @throws Exception 163 | */ 164 | public abstract Link[] getLinkList(String dbid, long id1, long link_type, 165 | long minTimestamp, long maxTimestamp, 166 | int offset, int limit) 167 | throws Exception; 168 | 169 | // count the #links 170 | public abstract long countLinks(String dbid, long id1, long link_type) throws Exception; 171 | 172 | /** 173 | * @return 0 if it doesn't support addBulkLinks and recalculateCounts methods 174 | * If it does support them, return the maximum number of links that 175 | * can be added at a time */ 176 | public int bulkLoadBatchSize() { 177 | return 0; 178 | } 179 | 180 | /** Add a batch of links without updating counts */ 181 | public void addBulkLinks(String dbid, List a, boolean noinverse) 182 | throws Exception { 183 | throw new UnsupportedOperationException("addBulkLinks not supported for " + 184 | "LinkStore subclass " + this.getClass().getName()); 185 | } 186 | 187 | /** Add a batch of counts */ 188 | public void addBulkCounts(String dbid, List a) 189 | throws Exception { 190 | throw new UnsupportedOperationException("addBulkCounts not supported for " + 191 | "LinkStore subclass " + this.getClass().getName()); 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkStoreMysql.java: -------------------------------------------------------------------------------- 1 | /* * LinkStore for MySQL 2 | * Author: Mark Callaghan 3 | * Date : Feb 2020 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.facebook.LinkBench; 18 | 19 | import java.io.IOException; 20 | import java.nio.ByteBuffer; 21 | import java.nio.CharBuffer; 22 | import java.nio.charset.Charset; 23 | import java.sql.Connection; 24 | import java.sql.DriverManager; 25 | import java.sql.ResultSet; 26 | import java.sql.SQLException; 27 | import java.sql.Statement; 28 | import java.sql.PreparedStatement; 29 | import java.util.Collections; 30 | import java.util.Date; 31 | import java.util.HashSet; 32 | import java.util.List; 33 | import java.util.ArrayList; 34 | import java.util.Properties; 35 | 36 | public class LinkStoreMysql extends LinkStoreSql { 37 | 38 | public LinkStoreMysql() { 39 | super(); 40 | } 41 | 42 | public LinkStoreMysql(Properties props) throws IOException, Exception { 43 | super(); 44 | initialize(props, Phase.LOAD, 0); 45 | } 46 | 47 | public void initialize(Properties props, Phase currentPhase, int threadId) { 48 | super.initialize(props, currentPhase, threadId); 49 | } 50 | 51 | protected PreparedStatement makeAddLinkIncCountPS() throws SQLException { 52 | String sql = "INSERT INTO " + init_dbid + "." + counttable + 53 | " (id, link_type, count, time, version)" + 54 | " VALUES (?, ?, ?, ?, 0)" + 55 | " ON DUPLICATE KEY UPDATE" + 56 | " count = count + ?," + 57 | " version = version + 1," + 58 | " time = ?"; 59 | 60 | logger.debug("addLinkIncCount PS: " + sql); 61 | return conn_ac0.prepareStatement(sql); 62 | } 63 | 64 | protected PreparedStatement makeGetLinkListPS() throws SQLException { 65 | String sql = "SELECT id1, id2, link_type," + 66 | " visibility, data, version, time" + 67 | " FROM " + init_dbid + "." + linktable + 68 | " FORCE INDEX(id1_type) " + 69 | " WHERE id1 = ? AND link_type = ? " + 70 | " AND time >= ?" + 71 | " AND time <= ?" + 72 | " AND visibility = " + LinkStore.VISIBILITY_DEFAULT + 73 | " ORDER BY time DESC" + 74 | " LIMIT ? OFFSET ?"; 75 | logger.debug("getLinkList PS: " + sql); 76 | return conn_ac1.prepareStatement(sql); 77 | } 78 | 79 | // This hardwires Linkbench to use the database "linkbench" 80 | protected String getJdbcUrl() { 81 | return "jdbc:mysql://"+ host + ":" + port + "/"; 82 | } 83 | 84 | protected String getJdbcClassName() { 85 | return "com.mysql.jdbc.Driver"; 86 | } 87 | 88 | protected String getJdbcOptions() { 89 | // It isn't clear that these make a big difference 90 | return "?elideSetAutoCommits=true" + 91 | "&useLocalTransactionState=true" + 92 | "&allowMultiQueries=true" + 93 | "&useLocalSessionState=true" + 94 | "&useAffectedRows=true" + 95 | "&useServerPrepStmts=true" + 96 | "&cachePrepStmts=true"+ 97 | "&cacheCallableStmts=true"+ 98 | "&alwaysSendSetIsolation=false"+ 99 | "&prepStmtCacheSqlLimit=4096"+ 100 | "&prepStmtCacheSize=1000"+ 101 | "&enableQueryTimeouts=false"+ 102 | "&callableStmtCacheSize=1000"+ 103 | "&metadataCacheSize=1000"+ 104 | "&cacheResultSetMetadata=true"; 105 | // Do not use -- https://bugs.mysql.com/bug.php?id=95139 106 | // "&cacheServerConfiguration=true"; 107 | } 108 | 109 | /** 110 | * Set of all JDBC SQLState strings that indicate a transient error 111 | * that should be handled by retrying 112 | */ 113 | protected HashSet populateRetrySQLStates() { 114 | // TODO are there more? 115 | HashSet states = new HashSet(); 116 | states.add("41000"); // ER_LOCK_WAIT_TIMEOUT 117 | states.add("40001"); // ER_LOCK_DEADLOCK 118 | return states; 119 | } 120 | 121 | protected boolean isDupKeyError(SQLException ex) { 122 | logger.trace("isDupKeyError for : " + ex + 123 | " with state " + ex.getSQLState()); 124 | return ex.getSQLState().equals("23000"); 125 | } 126 | 127 | @Override 128 | public void resetNodeStore(String dbid, long startID) throws SQLException { 129 | checkNodeTableConfigured(); 130 | // Truncate table deletes all data and allows us to reset autoincrement 131 | stmt_ac1.execute(String.format("TRUNCATE TABLE %s.%s;", dbid, nodetable)); 132 | 133 | stmt_ac1.execute(String.format("ALTER TABLE `%s`.`%s` " + 134 | "AUTO_INCREMENT = %d;", dbid, nodetable, startID)); 135 | } 136 | 137 | String getDefaultPort() { return "3306"; } 138 | 139 | protected void addLinkChangeCount(String dbid, Link l, int base_count, PreparedStatement pstmt) 140 | throws SQLException { 141 | 142 | if (Level.TRACE.isGreaterOrEqual(debuglevel)) 143 | logger.trace("addLink change count"); 144 | 145 | long now = (new Date()).getTime(); 146 | pstmt.setLong(1, l.id1); 147 | pstmt.setLong(2, l.link_type); 148 | pstmt.setLong(3, base_count); 149 | pstmt.setLong(4, now); 150 | pstmt.setLong(5, base_count); 151 | pstmt.setLong(6, now); 152 | 153 | int update_res = pstmt.executeUpdate(); 154 | // 1 means insert, 2 means update, 0 means no change, other values are not defined 155 | if (update_res != 1 && update_res != 2) { 156 | String e = "addLink increment count failed with res=" + 157 | update_res + " for id1=" + l.id1 + 158 | " id2=" + l.id2 + " link_type=" + l.link_type; 159 | logger.error(e); 160 | conn_ac0.rollback(); 161 | throw new RuntimeException(e); 162 | } 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkStorePgsql.java: -------------------------------------------------------------------------------- 1 | /* * LinkStore for PostgreSQL 2 | * Author : woonhak.kang (woonhak.kang@gmail.com) 3 | * Date : 01/26/2016 4 | * Author: Mark Callaghan 5 | * Date : Feb 2020 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | package com.facebook.LinkBench; 20 | 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | import java.nio.CharBuffer; 24 | import java.nio.charset.Charset; 25 | import java.sql.Connection; 26 | import java.sql.DriverManager; 27 | import java.sql.ResultSet; 28 | import java.sql.SQLException; 29 | import java.sql.Statement; 30 | import java.sql.PreparedStatement; 31 | import java.util.Collections; 32 | import java.util.Date; 33 | import java.util.HashSet; 34 | import java.util.List; 35 | import java.util.ArrayList; 36 | import java.util.Properties; 37 | 38 | public class LinkStorePgsql extends LinkStoreSql { 39 | 40 | public LinkStorePgsql() { 41 | super(); 42 | } 43 | 44 | public LinkStorePgsql(Properties props) throws IOException, Exception { 45 | super(); 46 | initialize(props, Phase.LOAD, 0); 47 | } 48 | 49 | public void initialize(Properties props, Phase currentPhase, int threadId) { 50 | super.initialize(props, currentPhase, threadId); 51 | } 52 | 53 | protected PreparedStatement makeAddLinkIncCountPS() throws SQLException { 54 | String sql = "INSERT INTO " + init_dbid + "." + counttable + 55 | "(id, link_type, count, time, version) " + 56 | "VALUES (?, ?, ?, ?, 0) " + 57 | "ON CONFLICT ON CONSTRAINT " + counttable + "_pkey DO UPDATE SET " + 58 | " count = " + init_dbid + "." + counttable +".count + ?" + 59 | ", version = " + init_dbid + "." + counttable +".version + 1 " + 60 | ", time = ?"; 61 | 62 | logger.debug("addLinkIncCount PS: " + sql); 63 | return conn_ac0.prepareStatement(sql); 64 | } 65 | 66 | protected PreparedStatement makeGetLinkListPS() throws SQLException { 67 | String sql = "SELECT id1, id2, link_type," + 68 | " visibility, data, version, time" + 69 | " FROM " + init_dbid + "." + linktable + 70 | " WHERE id1 = ? AND link_type = ? " + 71 | " AND time >= ?" + 72 | " AND time <= ?" + 73 | " AND visibility = " + LinkStore.VISIBILITY_DEFAULT + 74 | " ORDER BY time DESC" + 75 | " LIMIT ? OFFSET ?"; 76 | logger.debug("getLinkList PS: " + sql); 77 | return conn_ac1.prepareStatement(sql); 78 | } 79 | 80 | // This hardwires Linkbench to use the database "linkbench" 81 | protected String getJdbcUrl() { 82 | return "jdbc:postgresql://"+ host + ":" + port + "/linkbench"; 83 | } 84 | 85 | protected String getJdbcClassName() { 86 | return "org.postgresql.Driver"; 87 | } 88 | 89 | protected String getJdbcOptions() { 90 | return "?elideSetAutoCommits=true" + 91 | "&useLocalTransactionState=true" + 92 | "&allowMultiQueries=true" + 93 | "&useLocalSessionState=true" + 94 | "&useAffectedRows=true"; 95 | } 96 | 97 | /** 98 | * Set of all JDBC SQLState strings that indicate a transient error 99 | * that should be handled by retrying 100 | */ 101 | protected HashSet populateRetrySQLStates() { 102 | // TODO are there more? 103 | HashSet states = new HashSet(); 104 | states.add("41000"); // ER_LOCK_WAIT_TIMEOUT 105 | states.add("40001"); // ER_LOCK_DEADLOCK 106 | return states; 107 | } 108 | 109 | protected boolean isDupKeyError(SQLException ex) { 110 | // 23505 is unique_violation, see https://www.postgresql.org/docs/12/errcodes-appendix.html 111 | return ex.getSQLState().equals("23505"); 112 | } 113 | 114 | @Override 115 | public void resetNodeStore(String dbid, long startID) throws SQLException { 116 | checkNodeTableConfigured(); 117 | // Truncate table deletes all data and allows us to reset autoincrement 118 | stmt_ac1.execute(String.format("TRUNCATE TABLE %s.%s;", dbid, nodetable)); 119 | 120 | stmt_ac1.execute(String.format("ALTER SEQUENCE %s.%s_id_seq RESTART %d;", 121 | dbid, nodetable, startID)); 122 | } 123 | 124 | String getDefaultPort() { return "5432"; } 125 | 126 | protected void addLinkChangeCount(String dbid, Link l, int base_count, PreparedStatement pstmt) 127 | throws SQLException { 128 | 129 | if (Level.TRACE.isGreaterOrEqual(debuglevel)) 130 | logger.trace("addLink change count"); 131 | 132 | long now = (new Date()).getTime(); 133 | pstmt.setLong(1, l.id1); 134 | pstmt.setLong(2, l.link_type); 135 | pstmt.setLong(3, base_count); 136 | pstmt.setLong(4, now); 137 | pstmt.setLong(5, base_count); 138 | pstmt.setLong(6, now); 139 | 140 | int update_res = pstmt.executeUpdate(); 141 | if (update_res != 1) { 142 | String e = "addLink increment count failed with res=" + 143 | update_res + " for id1=" + l.id1 + 144 | " id2=" + l.id2 + " link_type=" + l.link_type; 145 | logger.error(e); 146 | conn_ac0.rollback(); 147 | throw new RuntimeException(e); 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/LinkWriteResult.java: -------------------------------------------------------------------------------- 1 | 2 | package com.facebook.LinkBench; 3 | 4 | public enum LinkWriteResult { 5 | LINK_INSERT, // Done via an insert 6 | LINK_UPDATE, // Done via an update 7 | LINK_NO_CHANGE, // Not done because data didn't change 8 | LINK_NOT_DONE // Operation wasn't done. Retry. 9 | } 10 | 11 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Logger.java: -------------------------------------------------------------------------------- 1 | 2 | package com.facebook.LinkBench; 3 | 4 | import java.sql.Timestamp; 5 | 6 | public class Logger { 7 | private static Logger logger = new Logger(Level.INFO); 8 | public static Logger getLogger() { return logger; } 9 | 10 | private Level level; 11 | 12 | public Logger(Level l) { 13 | level = l; 14 | } 15 | 16 | public Level getLevel() { return level; } 17 | public void setLevel(Level l) { level = l; } 18 | 19 | public void trace(StringBuilder s) { logif(Level.TRACE, s); } 20 | public void trace(String s) { logif(Level.TRACE, s); } 21 | 22 | public void debug(StringBuilder s) { logif(Level.DEBUG, s); } 23 | public void debug(String s) { logif(Level.DEBUG, s); } 24 | 25 | public void info(StringBuilder s) { logif(Level.INFO, s); } 26 | public void info(String s) { logif(Level.INFO, s); } 27 | 28 | public void warn(String s, Exception e) { logif(Level.WARN, s, e); } 29 | public void warn(Exception e) { logif(Level.WARN, e); } 30 | public void warn(StringBuilder s) { logif(Level.WARN, s); } 31 | public void warn(String s) { logif(Level.WARN, s); } 32 | 33 | public void error(String s, Exception e) { log(Level.ERROR, s + ": " + e.toString()); } 34 | public void error(Exception e) { log(Level.ERROR, e.toString()); } 35 | public void error(StringBuilder s) { log(Level.ERROR, s.toString()); } 36 | public void error(String s) { log(Level.ERROR, s); } 37 | 38 | private void logif(Level l, String s, Exception e) { 39 | if (l.isGreaterOrEqual(level)) 40 | log(l, s + ": " + e.toString()); 41 | } 42 | 43 | private void logif(Level l, Exception e) { 44 | if (l.isGreaterOrEqual(level)) 45 | log(l, e.toString()); 46 | } 47 | 48 | private void logif(Level l, StringBuilder s) { 49 | if (l.isGreaterOrEqual(level)) 50 | log(l, s.toString()); 51 | } 52 | 53 | private void logif(Level l, String s) { 54 | if (l.isGreaterOrEqual(level)) 55 | log(l, s); 56 | } 57 | 58 | private void log(Level l, String s) { 59 | Timestamp ts = new Timestamp(System.currentTimeMillis()); 60 | System.out.println(l + " " + ts + " [" + Thread.currentThread().getName() + "]: " + s); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Node.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Arrays; 19 | 20 | /** 21 | * Object node in social graph 22 | * @author tarmstrong 23 | */ 24 | public class Node { 25 | /** Unique identifier for node */ 26 | public long id; 27 | 28 | /** Type of node */ 29 | public int type; 30 | 31 | /** Version of node: typically updated on every change */ 32 | public long version; 33 | 34 | /** Last update time of node as UNIX timestamp */ 35 | public int time; 36 | 37 | /** Arbitrary payload data */ 38 | public byte data[]; 39 | 40 | public Node(long id, int type, long version, int time, 41 | byte data[]) { 42 | super(); 43 | this.id = id; 44 | this.type = type; 45 | this.version = version; 46 | this.time = time; 47 | this.data = data; 48 | } 49 | 50 | public Node clone() { 51 | return new Node(id, type, version, time, data); 52 | } 53 | @Override 54 | public boolean equals(Object other) { 55 | if (!(other instanceof Node)) { 56 | return false; 57 | } 58 | Node o = (Node) other; 59 | return id == o.id && type == o.type && version == o.version 60 | && time == o.time && Arrays.equals(data, o.data); 61 | } 62 | 63 | public String toString() { 64 | return "Node(" + "id=" + id + ",type=" + type + ",version=" + version + "," 65 | + "timestamp=" + time + ",data=" 66 | + Arrays.toString(data) + ")"; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/NodeStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.List; 21 | import java.util.Properties; 22 | 23 | /** 24 | * Some implementations of NodeStore may require that each 25 | * dbid be initialized with reset before any data is written to it (so as to 26 | * ensure that the starting id is actually specified. 27 | */ 28 | public interface NodeStore { 29 | // Limit data to 1MB 30 | public static final long MAX_NODE_DATA = 1024 * 1024; 31 | 32 | /** initialize the store object */ 33 | public void initialize(Properties p, 34 | Phase currentPhase, int threadId) throws IOException, Exception; 35 | 36 | /** 37 | * Reset node storage to a clean state in shard: 38 | * deletes all stored nodes 39 | * resets id allocation, with new IDs to be allocated starting from startID 40 | */ 41 | public void resetNodeStore(String dbid, long startID) throws Exception; 42 | 43 | /** 44 | * Adds a new node object to the database. 45 | * 46 | * This allocates a new id for the object and returns i. 47 | * 48 | * The benchmark assumes that, after resetStore() is called, 49 | * node IDs are allocated in sequence, i.e. startID, startID + 1, ... 50 | * Add node should return the next ID in the sequence. 51 | * 52 | * @param dbid the db shard to put that object in 53 | * @param node a node with all data aside from id filled in. The id 54 | * field is *not* updated to the new value by this function 55 | * @return the id allocated for the node 56 | */ 57 | public long addNode(String dbid, Node node) throws Exception; 58 | 59 | /** 60 | * Bulk loading to more efficiently load nodes. 61 | * Calling this is equivalent to calling addNode multiple times. 62 | * 63 | * @param dbid 64 | * @param nodes 65 | * @return the actual IDs allocated to the nodes 66 | * @throws Exception 67 | */ 68 | public long[] bulkAddNodes(String dbid, List nodes) throws Exception; 69 | 70 | /** 71 | * Preferred size of data to load 72 | * @return 73 | */ 74 | public int bulkLoadBatchSize(); 75 | public int bulkLoadBatchKB(); 76 | 77 | /** 78 | * Get a node of the specified type 79 | * @param dbid the db shard the id is mapped to 80 | * @param type the type of the object 81 | * @param id the id of the object 82 | * @return null if not found, a Node with all fields filled in otherwise 83 | */ 84 | public Node getNode(String dbid, int type, long id) throws Exception; 85 | 86 | /** 87 | * Update all parameters of the node specified. 88 | * @param dbid 89 | * @param node 90 | * @return true if the update was successful, false if not present 91 | */ 92 | public boolean updateNode(String dbid, Node node) throws Exception; 93 | 94 | /** 95 | * Delete the object specified by the arguments 96 | * @param dbid 97 | * @param type 98 | * @param id 99 | * @return true if the node was deleted, false if not present 100 | */ 101 | public boolean deleteNode(String dbid, int type, long id) throws Exception; 102 | 103 | public void clearErrors(int loaderId); 104 | 105 | /** 106 | * Close the node store and clean up any resources 107 | */ 108 | public void close(); 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Phase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | /** 19 | * Different phases of the benchmark 20 | * 21 | */ 22 | public enum Phase { 23 | LOAD, 24 | REQUEST 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Shuffler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | 19 | /* 20 | * A class to generate permutation of 0, 1, 2, ..(N-1) using O(1) memory. 21 | */ 22 | 23 | class Shuffler { 24 | /* Example to show how algorithm works: n=9; m=4. 25 | * 26 | * Starting with the original a = {0, 1, ..n}, apply the following 27 | * transformation to generate a permutation of a: 28 | * 29 | * 1. Divide 0..9 into multiple groups, each group has length m=4 30 | * a = 0 1 2 3|4 5 6 7|8 9 31 | * 32 | * 2. Move elements with same position in each group together 33 | * M(a) = 0 4 8|1 5 9|2 6|3 7 34 | * 35 | * 3. T(a) = position of i in the permutation in M(a) 36 | * T(a) = {0 3 6 8 1 4 7 9 2 5} 37 | */ 38 | 39 | //get T(a)[i] 40 | static long getPermutationValue(long i, long n, long m) { 41 | long minsize = n/m; 42 | long maxsize = (n%m == 0) ? minsize : minsize + 1; 43 | long n_maxsize_groups = n%m; 44 | long newgroupid = i%m, newidx = i/m; 45 | if (newgroupid < n_maxsize_groups) { 46 | return newgroupid*maxsize + newidx; 47 | } 48 | else { 49 | return n_maxsize_groups*maxsize + 50 | (newgroupid - n_maxsize_groups)*minsize + newidx; 51 | } 52 | } 53 | 54 | static long getPermutationValue(long i, long start, long end, long m) { 55 | return start + getPermutationValue(i - start, end - start, m); 56 | } 57 | 58 | //multiplication of transformations 59 | //apply multiple transformation T to make a more random permutation 60 | static long getPermutationValue(long i, long start, long end, long[] ms) { 61 | for (int j = 0; j < ms.length; ++j) { 62 | i = getPermutationValue(i, start, end, ms[j]); 63 | } 64 | return i; 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/Timer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Random; 19 | 20 | public class Timer { 21 | 22 | /** 23 | * Wait an amount of time since the last event determined by the 24 | * exponential distribution 25 | * @param rng random number generator to use 26 | * @param lastevent_time_ns last event time (units same as System.nanoTime()) 27 | * @param arrival_rate_ns arrival rate: events per nanosecond 28 | * @return time of the next event 29 | */ 30 | public static long waitExpInterval(Random rng, 31 | long lasteventTime_ns, double arrivalRate_ns) { 32 | long nextTime_ns = lasteventTime_ns + 33 | Math.round(-1 * Math.log(rng.nextDouble()) / arrivalRate_ns); 34 | Timer.waitUntil(nextTime_ns); 35 | return nextTime_ns; 36 | } 37 | 38 | /** 39 | * Wait until System.nanoTime() is > the argument 40 | * @param time_ns 41 | */ 42 | public static void waitUntil(long time_ns) { 43 | long now = System.nanoTime(); 44 | while (now < time_ns) { 45 | long wait = time_ns - now; 46 | try { 47 | Thread.sleep(wait / 1000000, (int)(wait % 1000)); 48 | } catch (InterruptedException ie) { 49 | // Restart loop 50 | } 51 | now = System.nanoTime(); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/distributions/ApproxHarmonic.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.distributions; 17 | 18 | /** 19 | * Approximations to harmonic numbers that speed up calculation time. 20 | */ 21 | public class ApproxHarmonic { 22 | private static final long APPROX_THRESHOLD = 100000; 23 | 24 | // Euler Mascheroni constant 25 | private static final double EULER_MASCHERONI = 26 | 0.5772156649015328606065120900824024310421; 27 | 28 | 29 | /** 30 | * Approximation to generalized harmonic for 0 >= m >= 1. 31 | * Designed to not take more than a couple of seconds to calculate, 32 | * and the have error of < 0.05% 33 | * @param n 34 | * @param m assume > 0 <= 35 | * @return 36 | */ 37 | public static double generalizedHarmonic(final long n, 38 | final double m) { 39 | if (n < 0) { 40 | throw new IllegalArgumentException("n must be non-negative"); 41 | } 42 | if (m < 0 || m > 1) { 43 | throw new IllegalArgumentException("m = " + m + " outside " + 44 | "range [0, 1]"); 45 | } 46 | if (n < APPROX_THRESHOLD) { 47 | // Approximation less accurate for small n, and full calculation 48 | // doesn't take as long 49 | return Harmonic.generalizedHarmonic(n, m); 50 | } 51 | 52 | if (m == 1) { 53 | // Standard approximation for regular harmonic numbers 54 | return Math.log(n) + EULER_MASCHERONI + 1 / (2 * n); 55 | } else { 56 | // Rough approximation for generalized harmonic for 57 | // m >= 0 and m <= 1 58 | 59 | // Standard integral of 1/(n^k) 60 | double integral = (1 / (1 - m)) * Math.pow(n, 1 - m); 61 | 62 | // Empirically derived correction factor that is good enough 63 | // to get to within 0.2% or so of exact number 64 | double correction = 0.58 - 1 / (1 - m); 65 | return integral + correction; 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/distributions/GeometricDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.distributions; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | import org.apache.commons.math3.util.FastMath; 22 | 23 | import com.facebook.LinkBench.Config; 24 | import com.facebook.LinkBench.ConfigUtil; 25 | 26 | /** 27 | * Geometric distribution 28 | * 29 | * NOTE: this generates values in the range [min, max). Since the 30 | * real geometric distribution generates values in range [min, inf), 31 | * we truncate anything >= max 32 | */ 33 | public class GeometricDistribution implements ProbabilityDistribution { 34 | 35 | /** The probability parameter that defines the distribution */ 36 | private double p = 0.0; 37 | 38 | /** Valid range */ 39 | private long min = 0, max = 0; 40 | 41 | private double scale = 0.0; 42 | 43 | public static final String PROB_PARAM_KEY = "prob"; 44 | 45 | @Override 46 | public void init(long min, long max, Properties props, String keyPrefix) { 47 | double parsedP = ConfigUtil.getDouble(props, keyPrefix + PROB_PARAM_KEY); 48 | 49 | double scaleVal = 1.0;; 50 | if (props.containsKey(Config.PROB_MEAN)) { 51 | scaleVal = (max - min) * ConfigUtil.getDouble(props, 52 | keyPrefix + Config.PROB_MEAN); 53 | } 54 | init(min, max, parsedP, scaleVal); 55 | } 56 | 57 | public void init(long min, long max, double p, double scale) { 58 | this.min = min; 59 | this.max = max; 60 | this.p = p; 61 | this.scale = scale; 62 | } 63 | 64 | @Override 65 | public double pdf(long id) { 66 | return scaledPdf(id, 1.0); 67 | } 68 | 69 | @Override 70 | public double expectedCount(long id) { 71 | return scaledPdf(id, scale); 72 | } 73 | 74 | private double scaledPdf(long id, double scaleFactor) { 75 | if (id < min || id >= max) return 0.0; 76 | long x = id - min; 77 | return FastMath.pow(1 - p, x) * scaleFactor * p; 78 | } 79 | 80 | @Override 81 | public double cdf(long id) { 82 | if (id < min) return 0.0; 83 | if (id >= max) return 1.0; 84 | return 1 - FastMath.pow(1 - p, id - min + 1); 85 | } 86 | 87 | @Override 88 | public long choose(Random rng) { 89 | return quantile(rng.nextDouble()); 90 | } 91 | 92 | @Override 93 | public long quantile(double r) { 94 | /* 95 | * Quantile function for geometric distribution over 96 | * range [0, inf) where 0 < r < 1 97 | * quantile(r) = ceiling(ln(1 - r) / ln (1 - p)) 98 | * Source: http://www.math.uah.edu/stat/bernoulli/Geometric.html 99 | */ 100 | if (r == 0.0) return min; // 0.0 must be handled specially 101 | 102 | long x = min + (long)FastMath.ceil( 103 | FastMath.log(1 - r) / FastMath.log(1 - p)); 104 | // truncate over max 105 | return Math.min(x, max - 1); 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/distributions/Harmonic.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.distributions; 2 | /* 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | /** 19 | * This code was derived and modified from the Apache Commons 20 | * Math 3.0 source release and modified for use in LinkBench 21 | * 22 | * @author tarmstrong 23 | */ 24 | import org.apache.commons.math3.util.FastMath; 25 | 26 | public class Harmonic { 27 | /** 28 | * Calculates the Nth generalized harmonic number. See 29 | * Harmonic 30 | * Series. 31 | * 32 | * @param n Term in the series to calculate (must be larger than 1) 33 | * @param m Exponent (special case {@code m = 1} is the harmonic series). 34 | * @return the nth generalized harmonic number. 35 | */ 36 | public static double generalizedHarmonic(final long n, final double m) { 37 | double value = 0; 38 | for (long k = n; k > 0; --k) { 39 | value += 1.0 / FastMath.pow(k, m); 40 | } 41 | return value; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/distributions/LogNormalDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.distributions; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | import org.apache.commons.math3.util.FastMath; 22 | 23 | import com.facebook.LinkBench.ConfigUtil; 24 | 25 | public class LogNormalDistribution implements ProbabilityDistribution { 26 | private long min; 27 | private long max; 28 | private double mu; // mean of the natural log of random variable 29 | private double sigma; // standard deviation of natural log of random variable 30 | 31 | public static final String CONFIG_MEDIAN = "median"; 32 | public static final String CONFIG_SIGMA = "sigma"; 33 | 34 | @Override 35 | public void init(long min, long max, Properties props, String keyPrefix) { 36 | double sigma = ConfigUtil.getDouble(props, CONFIG_SIGMA); 37 | double median = ConfigUtil.getDouble(props, CONFIG_MEDIAN); 38 | init(min, max, median, sigma); 39 | } 40 | 41 | /** 42 | * 43 | * @param min 44 | * @param max 45 | * @param median the median value of the distribution 46 | * @param sigma the standard deviation of the natural log of the variable 47 | * @param scale 48 | */ 49 | public void init(long min, long max, double median, double sigma) { 50 | this.min = min; 51 | this.max = max; 52 | this.mu = FastMath.log(median); 53 | this.sigma = sigma; 54 | } 55 | 56 | @Override 57 | public double pdf(long id) { 58 | throw new RuntimeException("pdf not implemented"); 59 | } 60 | 61 | @Override 62 | public double expectedCount(long id) { 63 | throw new RuntimeException("expectedCount not implemented"); 64 | } 65 | 66 | @Override 67 | public double cdf(long id) { 68 | if (id < min) return 0.0; 69 | if (id >= max) return 1.0; 70 | org.apache.commons.math3.distribution.LogNormalDistribution d = 71 | new org.apache.commons.math3.distribution.LogNormalDistribution(mu, sigma); 72 | return d.cumulativeProbability(id); 73 | } 74 | 75 | @Override 76 | public long choose(Random rng) { 77 | long choice = (long) Math.round(FastMath.exp((rng.nextGaussian() * sigma) + mu)); 78 | if (choice < min) 79 | return min; 80 | else if (choice >= max) 81 | return max - 1; 82 | else 83 | return choice; 84 | } 85 | 86 | @Override 87 | public long quantile(double p) { 88 | throw new RuntimeException("Quantile not implemented"); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/distributions/ProbabilityDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.distributions; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | /** 22 | * Probability distribution over a range of integers [min, max), ranked 23 | * in descending order of probability, where min is most probable and 24 | * max - 1 is least probable. 25 | * @author tarmstrong 26 | * 27 | */ 28 | public interface ProbabilityDistribution { 29 | 30 | /** 31 | * Initialize probability distribution for range [min, max) with additional 32 | * parameters pulled from properties dictionary by implementation. 33 | * @param min 34 | * @param max 35 | * @param props Properties dictionary for any extra parameters 36 | * @param keyPrefix In case there are multiple distributions with 37 | * different parameters in properties, this prefix can be 38 | * provided to distinguish when looking up keys 39 | */ 40 | public abstract void init(long min, long max, 41 | Properties props, String keyPrefix); 42 | 43 | /** 44 | * Probability density function, i.e. P(X = id) 45 | * @param id 46 | * @return 47 | */ 48 | public abstract double pdf(long id); 49 | 50 | /** 51 | * Probability density function scaled by an implementation-defined 52 | * factor (e.g. the number of trials, giving the expected number of values) 53 | * @param id 54 | * @return 55 | */ 56 | public abstract double expectedCount(long id); 57 | 58 | /** 59 | * Cumulative distribution function, i.e. for a random variable 60 | * X chosen accord to the distribution P(X <= id). 61 | * E.g. cdf(min - 1) = 0.0, and cdf(max - 1) = 1.0 62 | * @param id 63 | * @return a probability in range [0.0, 1.0] 64 | */ 65 | public abstract double cdf(long id); 66 | 67 | /** 68 | * Choose a random id in range [min, max) according to the probability 69 | * distribution. 70 | * @param rng a random number generator to use for random choice 71 | * @return the chosen id 72 | */ 73 | public abstract long choose(Random rng); 74 | 75 | 76 | /** 77 | * Quantile function for the distribution 78 | * @return x such that Pr(X <= x) = p 79 | */ 80 | public abstract long quantile(double p); 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/distributions/UniformDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.distributions; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | import com.facebook.LinkBench.Config; 22 | import com.facebook.LinkBench.ConfigUtil; 23 | 24 | /** 25 | * Uniform distribution over integers in range [minID, maxID), 26 | * where minID is included in range and maxID excluded 27 | * 28 | */ 29 | public class UniformDistribution implements ProbabilityDistribution { 30 | 31 | private long min = 0; 32 | private long max = 1; 33 | private double scale = 1.0; 34 | 35 | public void init(long min, long max, Properties props, String keyPrefix) { 36 | if (max <= min) { 37 | throw new IllegalArgumentException("max = " + max + " <= min = " + min + 38 | ": probability distribution cannot have zero or negative domain"); 39 | } 40 | this.min = min; 41 | this.max = max; 42 | if (props != null && props.containsKey(keyPrefix + Config.PROB_MEAN)) { 43 | scale = (max - min) * ConfigUtil.getDouble(props, 44 | keyPrefix + Config.PROB_MEAN); 45 | } else { 46 | scale = 1.0; 47 | } 48 | } 49 | 50 | public void init(long min, long max, double scale) { 51 | this.min = min; 52 | this.max = max; 53 | this.scale = scale; 54 | } 55 | 56 | @Override 57 | public double pdf(long id) { 58 | return scaledPDF(id, 1.0); 59 | } 60 | 61 | @Override 62 | public double expectedCount(long id) { 63 | return scaledPDF(id, scale); 64 | } 65 | 66 | private double scaledPDF(long id, double scale) { 67 | // Calculate this way to avoid losing precision by calculating very 68 | // small pdf number 69 | if (id < min || id >= max) return 0.0; 70 | return scale / (double) (max - min); 71 | } 72 | 73 | /** 74 | * Cumulative distribution function for distribution 75 | * @param id 76 | * @return 77 | */ 78 | public double cdf(long id) { 79 | if (id >= max) { 80 | return 1.0; 81 | } 82 | if (id < min) { 83 | return 0.0; 84 | } 85 | long n = max - min; 86 | long rank = id - min + 1; 87 | 88 | return rank / (double)n; 89 | } 90 | 91 | /** 92 | * Quantile function 93 | */ 94 | public long quantile(double p) { 95 | assert(p >= 0.0 && p <= 1.0); 96 | long n = max - min; 97 | long i = (long)Math.floor(p * n); 98 | if (i == n) return max - 1; 99 | return i + min; 100 | } 101 | 102 | // Total number of representable numbers by int 103 | private static final long UINT_RANGE = Integer.MAX_VALUE - (long) Integer.MIN_VALUE; 104 | 105 | /** Choose an id X uniformly in the range*/ 106 | public long choose(Random rng) { 107 | long n = max - min; 108 | // Java's random number generator has less randomness in lower bits 109 | // so just taking a mod doesn't give a good quality result. 110 | if (n <= Integer.MAX_VALUE) { 111 | return min + (long)rng.nextInt((int)n); 112 | } else if (n < UINT_RANGE) { 113 | return randint2(rng, n); 114 | } else { 115 | return UINT_RANGE * rng.nextInt((int)(n / UINT_RANGE)) + 116 | randint2(rng, n % UINT_RANGE); 117 | } 118 | } 119 | 120 | /** 121 | * Produce a random integer in range [0, n] 122 | * n must be in range [0, MAX_INT - MIN_INT] 123 | * @param rng 124 | * @param n 125 | * @return 126 | */ 127 | private long randint2(Random rng, long n) { 128 | assert(n < UINT_RANGE); 129 | double p = Integer.MAX_VALUE / (double)n; 130 | if (rng.nextDouble() < p) { 131 | return rng.nextInt(Integer.MAX_VALUE); 132 | } else { 133 | return Integer.MAX_VALUE + 134 | (long)(rng.nextInt((int)(n - Integer.MAX_VALUE))); 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/generators/DataGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.generators; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | public interface DataGenerator { 22 | 23 | public void init(Properties props, String keyPrefix); 24 | 25 | /** 26 | * Fill the provided array with randomly generated data 27 | * @param data 28 | * @return the argument, as a convenience so that an array can be 29 | * constructed and filled in a single statement 30 | */ 31 | public byte[] fill(Random rng, byte data[]); 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/generators/MotifDataGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.generators; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | import com.facebook.LinkBench.Config; 22 | import com.facebook.LinkBench.ConfigUtil; 23 | import com.facebook.LinkBench.LinkBenchConfigError; 24 | 25 | /** 26 | * A simple data generator where the same sequences of bytes, or "motifs" occur 27 | * multiple times. This is designed to emulate one particular property of real 28 | * data that is exploited by compression algorithms. Typically a short sequence 29 | * of data generated by this generator will not be very compressible on its own, 30 | * as no motifs will recur, but if multiple output strings are concatenated 31 | * together then the same motifs will recur repeatedly and the data will be 32 | * compressible. 33 | * 34 | * The motif data generator has a buffer of "shared" motifs, which reoccur 35 | * frequently in the output of the generator 36 | * 37 | * The data generator generates bytes from within the range of values [min, max). 38 | * There is an additional parameter, which is called uniqueness for lack of a 39 | * better name. The generator fills a buffer with data in chunks. A chunk 40 | * is either generated as random new bytes, or is drawn from the "motifs", 41 | * 42 | * The uniqueness parameter controls the proportion of new chunks versus duplicated 43 | * motifs. It is a probability between 0.0 and 1.0. It can also be seen as the expected 44 | * percentage of bytes are generated from scratch. 45 | * 46 | * Control how often motifs appear in data 47 | * uniqueness = 0.0: all data drawn from motifs 48 | * uniqueness 1.0: completely independent bytes 49 | */ 50 | public class MotifDataGenerator implements DataGenerator { 51 | private static final int MAX_CHUNK_SIZE = 128; 52 | 53 | public static final int DEFAULT_MOTIF_BUFFER_SIZE = 512; 54 | 55 | 56 | /** Lowest byte to appear in output */ 57 | private int start; 58 | /** Number of distinct bytes to appear in output */ 59 | private int range; 60 | /** percentage of data drawn from motifs */ 61 | private double uniqueness; 62 | 63 | 64 | /** 65 | * Buffer with a sequence of random bytes that are 66 | * pasted into output. Starts off null, initialized 67 | * on demand. 68 | */ 69 | private byte motifs[]; 70 | /** Size of motif buffer */ 71 | private int motifBytes; 72 | 73 | 74 | public MotifDataGenerator() { 75 | start = '\0'; 76 | range = 1; 77 | uniqueness = 0.0; 78 | } 79 | 80 | /** 81 | * Generate characters from start to end (inclusive both ends) 82 | * @param start 83 | * @param end 84 | */ 85 | public void init(int start, int end, double uniqueness) { 86 | init(start, end, uniqueness, DEFAULT_MOTIF_BUFFER_SIZE); 87 | } 88 | 89 | public void init(int start, int end, double uniqueness, int motifBytes) { 90 | if (start < 0 || start >= 256) { 91 | throw new LinkBenchConfigError("start " + start + 92 | " out of range [0,255]"); 93 | } 94 | if (end < 0 || end >= 256) { 95 | throw new LinkBenchConfigError("endbyte " + end + 96 | " out of range [0,255]"); 97 | } 98 | 99 | if (start >= end) { 100 | throw new LinkBenchConfigError("startByte " + start 101 | + " >= endByte " + end); 102 | } 103 | this.start = (byte)start; 104 | this.range = end - start + 1; 105 | this.uniqueness = uniqueness; 106 | this.motifBytes = motifBytes; 107 | this.motifs = null; 108 | } 109 | 110 | @Override 111 | public void init(Properties props, String keyPrefix) { 112 | int startByte = ConfigUtil.getInt(props, keyPrefix + 113 | Config.UNIFORM_GEN_STARTBYTE); 114 | int endByte = ConfigUtil.getInt(props, keyPrefix + 115 | Config.UNIFORM_GEN_ENDBYTE); 116 | double uniqueness = ConfigUtil.getDouble(props, keyPrefix + 117 | Config.MOTIF_GEN_UNIQUENESS); 118 | if (props.contains(keyPrefix + Config.MOTIF_GEN_LENGTH)) { 119 | int motifBytes = ConfigUtil.getInt(props, keyPrefix 120 | + Config.MOTIF_GEN_LENGTH); 121 | init(startByte, endByte, uniqueness, motifBytes); 122 | } else { 123 | init(startByte, endByte, uniqueness); 124 | } 125 | } 126 | 127 | /** 128 | * Give an upper bound for the compression ratio for the algorithm 129 | * @return number between 0.0 and 1.0 - 0.0 is perfectly compressible, 130 | * 1.0 is incompressible 131 | */ 132 | public double estMaxCompression() { 133 | // Avg bytes required to represent each character (uniformly distributed) 134 | double charCompression = range / (double) 255; 135 | // random data shouldn't have any inter-character correlations that can 136 | // be compressed. Upper bound derived by assuming motif is completely 137 | // compressible 138 | return charCompression * uniqueness; 139 | } 140 | 141 | @Override 142 | public byte[] fill(Random rng, byte[] data) { 143 | // Fill motifs now so that we can use rng 144 | if (motifs == null) { 145 | motifs = new byte[motifBytes]; 146 | for (int i = 0; i < motifs.length; i++) { 147 | motifs[i] = (byte) (start + rng.nextInt(range)); 148 | } 149 | } 150 | 151 | int n = data.length; 152 | int chunk = Math.min(MAX_CHUNK_SIZE, motifBytes); 153 | 154 | for (int i = 0; i < n; i += chunk) { 155 | if (rng.nextDouble() < uniqueness) { 156 | int chunkEnd = Math.min(n, i + chunk); 157 | // New sequence of unique bytes 158 | for (int j = i; j < chunkEnd; j++) { 159 | data[j] = (byte) (start + rng.nextInt(range)); 160 | } 161 | } else { 162 | int thisChunk = Math.min(chunk, n - i); 163 | int k = rng.nextInt(motifBytes - thisChunk + 1); 164 | // Copy previous sequence of bytes 165 | System.arraycopy(motifs, k, data, i, thisChunk); 166 | } 167 | } 168 | return data; 169 | } 170 | 171 | } 172 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/generators/UniformDataGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.generators; 17 | 18 | import java.util.Properties; 19 | import java.util.Random; 20 | 21 | import com.facebook.LinkBench.Config; 22 | import com.facebook.LinkBench.ConfigUtil; 23 | import com.facebook.LinkBench.LinkBenchConfigError; 24 | 25 | /** 26 | * A super simple data generator that generates a string of 27 | * characters chosen uniformly from a range. 28 | * 29 | * This probably isn't a good generator to use if you want something realistic, 30 | * especially if compressibility properties of the data will affect your 31 | * experiment. 32 | */ 33 | public class UniformDataGenerator implements DataGenerator { 34 | private int range; 35 | private int start; 36 | 37 | public UniformDataGenerator() { 38 | start = '\0'; 39 | range = 1; 40 | } 41 | 42 | /** 43 | * Generate characters from start to end (inclusive both ends) 44 | * @param start 45 | * @param end 46 | */ 47 | public void init(int start, int end) { 48 | if (start < 0 || start >= 256) { 49 | throw new LinkBenchConfigError("start " + start + 50 | " out of range [0,255]"); 51 | } 52 | if (end < 0 || end >= 256) { 53 | throw new LinkBenchConfigError("endbyte " + end + 54 | " out of range [0,255]"); 55 | } 56 | 57 | if (start >= end) { 58 | throw new LinkBenchConfigError("startByte " + start 59 | + " >= endByte " + end); 60 | } 61 | this.start = (byte)start; 62 | this.range = end - start + 1; 63 | } 64 | 65 | @Override 66 | public void init(Properties props, String keyPrefix) { 67 | int startByte = ConfigUtil.getInt(props, keyPrefix + 68 | Config.UNIFORM_GEN_STARTBYTE); 69 | int endByte = ConfigUtil.getInt(props, keyPrefix + 70 | Config.UNIFORM_GEN_ENDBYTE); 71 | init(startByte, endByte); 72 | } 73 | 74 | @Override 75 | public byte[] fill(Random rng, byte[] data) { 76 | return gen(rng, data, start, range); 77 | } 78 | 79 | public static byte[] gen(Random rng, byte[] data, 80 | int startByte, int range) { 81 | int n = data.length; 82 | for (int i = 0; i < n; i++) { 83 | data[i] = (byte) (startByte + rng.nextInt(range)); 84 | } 85 | return data; 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/stats/HdrLatencyHistogram.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-present MongoDB Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.stats; 17 | 18 | import java.io.PrintStream; 19 | import java.text.DecimalFormat; 20 | 21 | import com.facebook.LinkBench.LinkBenchOp; 22 | import com.facebook.LinkBench.LinkStore; 23 | import com.facebook.LinkBench.Logger; 24 | import org.HdrHistogram.Histogram; 25 | 26 | /** 27 | * Class used to track latency values using HDR Histogram. This provides 28 | * a more accurate estimate of the latency percentiles at a cost of a higher 29 | * memory footprint than the default histogram type 30 | * 31 | * See http://hdrhistogram.org/ for more details 32 | */ 33 | public class HdrLatencyHistogram implements LatencyHistogram { 34 | 35 | private static final int percentiles[] = new int[] {25, 50, 75, 95, 99}; 36 | private final Histogram histograms[]; 37 | private final int maxThreads; 38 | 39 | public HdrLatencyHistogram(int maxThreads, int histogramAccuracy, int maxHistogramValue) { 40 | this.maxThreads = maxThreads; 41 | histograms = new Histogram[LinkStore.MAX_OPTYPES]; 42 | for (LinkBenchOp op : LinkBenchOp.values()) { 43 | histograms[op.ordinal()] = new Histogram(maxHistogramValue, histogramAccuracy); 44 | } 45 | } 46 | 47 | /** 48 | * Get an estimate of the amount of memory that will be used by the histogram 49 | */ 50 | public long memoryUsageEstimate() { 51 | long usageInBytes = 0; 52 | for(Histogram histogram : histograms) 53 | { 54 | usageInBytes += histogram.getEstimatedFootprintInBytes(); 55 | } 56 | 57 | return usageInBytes; 58 | } 59 | 60 | /** 61 | * Track a latency value for a specific operation 62 | */ 63 | @Override 64 | public void recordLatency(int threadid, LinkBenchOp type, long microtimetaken) { 65 | histograms[type.ordinal()].recordValue(microtimetaken); 66 | } 67 | 68 | /** 69 | * Print the current latency stats tracked to the logs 70 | */ 71 | @Override 72 | public void displayLatencyStats() { 73 | Logger logger = Logger.getLogger(); 74 | // print percentiles 75 | for (LinkBenchOp type: LinkBenchOp.values()) { 76 | 77 | Histogram histogram = histograms[type.ordinal()]; 78 | if (histogram.getTotalCount() == 0) { // no samples of this type 79 | continue; 80 | } 81 | 82 | long sampleCounts = histogram.getTotalCount(); 83 | String logString = type.displayName() + String.format(" count = %d ", sampleCounts); 84 | for(int percentile: percentiles) { 85 | logString += String.format(" p%d = %dus ", percentile, histogram.getValueAtPercentile(percentile)); 86 | } 87 | 88 | double mean = histogram.getMean(); 89 | long max = histogram.getMaxValue(); 90 | logString += String.format(" max = %dus mean = %.3fus threads = %d", max, mean, maxThreads); 91 | logger.info(logString); 92 | } 93 | } 94 | 95 | /** 96 | * Print the current latency stats in csv format 97 | */ 98 | @Override 99 | public void printCSVStats(PrintStream out, boolean header) { 100 | printCSVStats(out, header, LinkBenchOp.values()); 101 | } 102 | 103 | private void printCSVStats(PrintStream out, boolean header, LinkBenchOp... ops) { 104 | 105 | // Write out the header 106 | if (header) { 107 | out.print("op,count"); 108 | for (int percentile: percentiles) { 109 | out.print(String.format(",p%d (us)", percentile)); 110 | } 111 | 112 | out.print(",max (us),mean (us),threads"); 113 | out.println(); 114 | } 115 | 116 | DecimalFormat df = new DecimalFormat("#.##"); 117 | for (LinkBenchOp op: ops) { 118 | Histogram histogram = histograms[op.ordinal()]; 119 | long samples = histogram.getTotalCount(); 120 | if (samples == 0) { 121 | continue; 122 | } 123 | 124 | out.print(op.name()); 125 | out.print(","); 126 | out.print(samples); 127 | 128 | for (int percentile: percentiles) { 129 | double percentileValue = histogram.getValueAtPercentile(percentile); 130 | out.print(","); 131 | out.print(df.format(percentileValue)); 132 | } 133 | 134 | String max = df.format(histogram.getMaxValueAsDouble()); 135 | String mean = df.format(histogram.getMean()); 136 | 137 | out.print(","); 138 | out.print(max); 139 | out.print(","); 140 | out.print(mean); 141 | out.print(","); 142 | out.print(maxThreads); 143 | out.println(); 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/stats/LatencyHistogram.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-present MongoDB Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.facebook.LinkBench.stats; 18 | 19 | import java.io.PrintStream; 20 | 21 | import com.facebook.LinkBench.LinkBenchOp; 22 | 23 | public interface LatencyHistogram { 24 | 25 | /** 26 | * Record a latency value from a test 27 | * @param threadid the thread that is reporting the latency 28 | * @param type the type of operation that the latency is from 29 | * @param microtimetaken the time taken, in microseconds, for the operation to complete 30 | */ 31 | public void recordLatency(int threadid, LinkBenchOp type, 32 | long microtimetaken); 33 | 34 | /** 35 | * Print statistics about latency to the logs 36 | */ 37 | public void displayLatencyStats(); 38 | 39 | /** 40 | * Save the latency results in csv format 41 | * @param out the stream to write the csv data to 42 | * @param header true if the header values of the csv should be printed 43 | */ 44 | public void printCSVStats(PrintStream out, boolean header); 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/stats/LatencyHistogramFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-present MongoDB Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.stats; 17 | 18 | import java.util.Properties; 19 | 20 | import com.facebook.LinkBench.ConfigUtil; 21 | 22 | import com.facebook.LinkBench.Logger; 23 | 24 | public class LatencyHistogramFactory { 25 | 26 | private final Logger logger; 27 | 28 | public LatencyHistogramFactory(Logger logger) { 29 | this.logger = logger; 30 | } 31 | 32 | /** 33 | * Create a latency histogram for a test given the config options 34 | * @param maxThreads the number of threads used by the test 35 | * @param props the properties from the linkbench config file 36 | * @return the histogram for recording latency results 37 | */ 38 | public LatencyHistogram create(int maxThreads, Properties props) 39 | { 40 | String useHdr = props.getProperty("use_hdr_histogram"); 41 | if (Boolean.valueOf(useHdr)) 42 | { 43 | int histogramAccuracy = ConfigUtil.getInt(props, "hdr_histogram_accuracy"); 44 | int maxHistogramValue = ConfigUtil.getInt(props, "hdr_histogram_max_latency"); 45 | 46 | HdrLatencyHistogram histogram = new HdrLatencyHistogram(maxThreads, histogramAccuracy, maxHistogramValue); 47 | logger.info(String.format( 48 | "Creating HDR histogram with %d significant digits, %d max latency, and memory footprint of %d bytes", 49 | histogramAccuracy, 50 | maxHistogramValue, 51 | histogram.memoryUsageEstimate())); 52 | 53 | return histogram; 54 | } 55 | 56 | // Using default latency tracking (lower memory footprint with reduced accuracy) 57 | return new LatencyStats(maxThreads); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/stats/RunningMean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.stats; 17 | 18 | /** 19 | * Keep track of mean in numerically stable way 20 | * See "Comparison of Several Algorithms for Computing Sample Means and 21 | * Variances", Ling, 1974, J. American Stat. Assoc. 22 | */ 23 | public class RunningMean { 24 | /** Number of samples */ 25 | private long n; 26 | 27 | /** First sample */ 28 | private final double v1; 29 | 30 | /** sum of difference */ 31 | private double running; 32 | 33 | /** initialize with first sample */ 34 | public RunningMean(double v1) { 35 | super(); 36 | this.v1 = v1; 37 | this.n = 1; 38 | this.running = 0.0; 39 | } 40 | 41 | public void addSample(double vi) { 42 | n++; 43 | running += (vi - v1); 44 | } 45 | 46 | public double mean() { 47 | return v1 + running / n; 48 | } 49 | 50 | public long samples() { 51 | return n; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/facebook/LinkBench/util/ClassLoadUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench.util; 17 | 18 | import java.lang.reflect.Constructor; 19 | 20 | /** 21 | * Utility methods for dynamic loading of classes 22 | */ 23 | public class ClassLoadUtil { 24 | 25 | private static final Class[] EMPTY_ARRAY = new Class[]{}; 26 | 27 | /** 28 | * Load a class by name. 29 | * @param name the class name. 30 | * @return the class object. 31 | * @throws ClassNotFoundException if the class is not found. 32 | */ 33 | public static Class getClassByName(String name) 34 | throws ClassNotFoundException { 35 | ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); 36 | return Class.forName(name, true, classLoader); 37 | } 38 | 39 | /** Create an object for the given class and initialize it from conf 40 | * 41 | * @param theClass class of which an object is created 42 | * @param expected the expected parent class or interface 43 | * @return a new object 44 | */ 45 | public static T newInstance(Class theClass, Class expected) { 46 | T result; 47 | try { 48 | if (!expected.isAssignableFrom(theClass)) { 49 | throw new Exception("Specified class " + theClass.getName() + "" + 50 | "does not extend/implement " + expected.getName()); 51 | } 52 | Class clazz = (Class)theClass; 53 | Constructor meth = clazz.getDeclaredConstructor(EMPTY_ARRAY); 54 | meth.setAccessible(true); 55 | result = meth.newInstance(); 56 | } catch (Exception e) { 57 | throw new RuntimeException(e); 58 | } 59 | return result; 60 | } 61 | 62 | public static T newInstance(String className, Class expected) 63 | throws ClassNotFoundException { 64 | return newInstance(getClassByName(className), expected); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/DummyLinkStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.util.Properties; 20 | 21 | public class DummyLinkStoreTest extends LinkStoreTestBase { 22 | 23 | private Properties props; 24 | 25 | @Override 26 | protected void initStore(Properties props) { 27 | // Do nothing 28 | this.props = props; 29 | } 30 | 31 | @Override 32 | protected DummyLinkStore getStoreHandle(boolean initialized) 33 | throws IOException, Exception { 34 | DummyLinkStore store = new DummyLinkStore(); 35 | if (initialized) { 36 | store.initialize(props, Phase.REQUEST, 0); 37 | } 38 | return store; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/GeneratedDataDump.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.BufferedOutputStream; 19 | import java.io.FileNotFoundException; 20 | import java.io.FileOutputStream; 21 | import java.io.IOException; 22 | import java.io.OutputStream; 23 | import java.util.Random; 24 | 25 | import com.facebook.LinkBench.generators.DataGenerator; 26 | import com.facebook.LinkBench.generators.MotifDataGenerator; 27 | import com.facebook.LinkBench.generators.UniformDataGenerator; 28 | /** 29 | * Generate some sample data using data generators, in order to test out compressibility 30 | * of randomly generated data. 31 | * 32 | * This generates several output files consistent of many generated payload data fields 33 | * separated by newlines 34 | */ 35 | public class GeneratedDataDump { 36 | private static final Random rng = new Random(); 37 | 38 | public static void main(String args[]) { 39 | String outputDir = ""; 40 | if (args.length == 0) { 41 | outputDir = "."; 42 | } else if (args.length == 1) { 43 | outputDir = args[0]; 44 | } else { 45 | System.err.println("GeneratedDataDump "); 46 | System.exit(1); 47 | } 48 | 49 | 50 | // Number of bytes per row 51 | final int objBytes = 256; 52 | final int assocBytes = 6; 53 | // Number of rows to generate 54 | final int objRows = 250000; 55 | final int assocRows = 10000000; 56 | writeGeneratedDataFile(outputDir + "/gen-data-motif.txt", makeMotifObj(), objRows, objBytes); 57 | writeGeneratedDataFile(outputDir + "/gen-data-uniform.txt", makeUniformObj(), objRows, objBytes); 58 | writeGeneratedDataFile(outputDir + "/gen-data-assoc-motif.txt", makeMotifAssoc(), assocRows, assocBytes); 59 | writeGeneratedDataFile(outputDir + "/gen-data-assoc-uniform.txt", makeUniformAssoc(), assocRows, assocBytes); 60 | } 61 | 62 | private static void writeGeneratedDataFile(String outFileName, 63 | DataGenerator gen, int rows, int bytes) { 64 | OutputStream out = null; 65 | try { 66 | out = new BufferedOutputStream(new FileOutputStream(outFileName)); 67 | } catch (FileNotFoundException e) { 68 | System.err.println("file " + outFileName + " could not be opened"); 69 | System.exit(1); 70 | } 71 | 72 | 73 | byte buf[] = new byte[bytes]; 74 | 75 | try { 76 | for (int i = 0; i < rows; i++) { 77 | gen.fill(rng, buf); 78 | out.write(buf); 79 | out.write('\n'); 80 | } 81 | out.close(); 82 | } catch (IOException e) { 83 | e.printStackTrace(); 84 | System.exit(1); 85 | } 86 | } 87 | 88 | private static DataGenerator makeUniformObj() { 89 | UniformDataGenerator gen = new UniformDataGenerator(); 90 | gen.init(50, 75); 91 | return gen; 92 | } 93 | 94 | private static MotifDataGenerator makeMotifObj() { 95 | MotifDataGenerator gen = new MotifDataGenerator(); 96 | int start = 50; 97 | int end = 220; 98 | double uniqueness = 0.63; 99 | gen.init(start, end, uniqueness); 100 | return gen; 101 | } 102 | 103 | private static DataGenerator makeUniformAssoc() { 104 | UniformDataGenerator gen = new UniformDataGenerator(); 105 | gen.init(50, 75); 106 | return gen; 107 | } 108 | 109 | private static MotifDataGenerator makeMotifAssoc() { 110 | MotifDataGenerator gen = new MotifDataGenerator(); 111 | int start = 32; 112 | int end = 100; 113 | double uniqueness = 0.225; 114 | int motifSize = 128; 115 | gen.init(start, end, uniqueness, motifSize); 116 | return gen; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/GeomDistTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Properties; 19 | 20 | import org.junit.Test; 21 | 22 | import com.facebook.LinkBench.distributions.GeometricDistribution; 23 | import com.facebook.LinkBench.distributions.ProbabilityDistribution; 24 | 25 | public class GeomDistTest extends DistributionTestBase { 26 | 27 | @Override 28 | protected ProbabilityDistribution getDist() { 29 | return new GeometricDistribution(); 30 | } 31 | 32 | @Override 33 | protected Properties getDistParams() { 34 | Properties props = new Properties(); 35 | props.setProperty(GeometricDistribution.PROB_PARAM_KEY, "0.2"); 36 | return props; 37 | } 38 | 39 | /** 40 | * Test cdf and pdf against precalculated values 41 | */ 42 | @Test 43 | public void testGeom() { 44 | GeometricDistribution d = new GeometricDistribution(); 45 | d.init(1, Long.MAX_VALUE, 0.3, 1.0); 46 | assertEquals(0.3, d.cdf(1), 0.001); 47 | assertEquals(0.51, d.cdf(2), 0.001); 48 | assertEquals(0.657, d.cdf(3), 0.001); 49 | assertEquals(0.917646, d.cdf(7), 0.001); 50 | assertEquals(0.971752, d.cdf(10), 0.001); 51 | assertEquals(0.995252, d.cdf(15), 0.001); 52 | 53 | assertEquals(0.3, d.pdf(1), 0.001); 54 | assertEquals(0.21, d.pdf(2), 0.001); 55 | assertEquals(0.147, d.pdf(3), 0.001); 56 | assertEquals(0.035, d.pdf(7), 0.001); 57 | assertEquals(0.012106, d.pdf(10), 0.001); 58 | assertEquals(0.002035, d.pdf(15), 0.001); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/HarmonicTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import junit.framework.TestCase; 19 | 20 | import org.junit.Test; 21 | import org.junit.experimental.categories.Category; 22 | 23 | import com.facebook.LinkBench.distributions.ApproxHarmonic; 24 | import com.facebook.LinkBench.distributions.Harmonic; 25 | import com.facebook.LinkBench.testtypes.SlowTest; 26 | 27 | 28 | @Category(SlowTest.class) 29 | public class HarmonicTest extends TestCase { 30 | 31 | @Test 32 | @Category(SlowTest.class) 33 | public void testHarmonic() { 34 | assertEquals(1, Harmonic.generalizedHarmonic(1, 0.8), 0.001); 35 | assertEquals(1.99534, Harmonic.generalizedHarmonic(10, 1.5), 0.00001); 36 | assertEquals(61.8010, Harmonic.generalizedHarmonic(1000, 0.5), 0.001); 37 | assertEquals(207.541, Harmonic.generalizedHarmonic(1000000, 0.7), 0.002); 38 | assertEquals(2679914.0, Harmonic.generalizedHarmonic(12345678, 0.1), 39 | 1); 40 | } 41 | 42 | static final double SHAPES[] = {0.01, 0.1, 0.5, 0.9, 0.99}; 43 | 44 | 45 | @Test 46 | public void testApproxFast() { 47 | for (long i = 0; i < 16; i+=4) { 48 | testApproxHelper(i); 49 | } 50 | } 51 | 52 | @Test 53 | @Category(SlowTest.class) 54 | public void testApproxSlow() { 55 | for (long i = 16; i < 30; i+=4) { 56 | testApproxHelper(i); 57 | } 58 | } 59 | 60 | /** Test that approximation is close to actual for a range of shapes and ns */ 61 | private void testApproxHelper(long i) { 62 | long n = (long)Math.pow(2, i); 63 | for (double shape: SHAPES) { 64 | double exact = Harmonic.generalizedHarmonic(n, shape); 65 | long start = System.currentTimeMillis(); 66 | double approx = ApproxHarmonic.generalizedHarmonic(n, shape); 67 | long end = System.currentTimeMillis(); 68 | System.err.format("ApproxHarmonic.generalizedHarmonic(%d, %f) " + 69 | "took %.3fs\n", n, shape, (end - start) / 1000.0); 70 | double err = approx - exact; 71 | double errPc = (err / exact) * 100.0; 72 | System.err.format("ApproxHarmonic.generalizedHarmonic(%d, %f) = %f. " + 73 | "exact=%f err=%f err%%=%.2f\n", n, shape, approx, 74 | exact, err, errPc); 75 | double errThresh = 0.05; 76 | assertTrue(String.format("Err%%=%.3f must be < 0.05%%", Math.abs(errPc)), 77 | Math.abs(errPc) < errThresh); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/InvertibleShufflerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Arrays; 19 | 20 | import org.junit.Test; 21 | 22 | import junit.framework.TestCase; 23 | /** 24 | * Check that the shuffler obeys expected invariants 25 | */ 26 | public class InvertibleShufflerTest extends TestCase { 27 | 28 | 29 | @Test 30 | public void testShuffleSmallRange() { 31 | long seed = 1234; 32 | testShuffle(0, 10, true, seed, 1); 33 | testShuffle(0, 4, true, seed, 2); 34 | testShuffle(0, 10, true, seed, 5); 35 | testShuffle(0, 7, true, seed, 3); 36 | testShuffle(0, 10, true, seed, 7); 37 | testShuffle(0, 10, true, seed, 11); 38 | testShuffle(0, 10, true, seed, 15); 39 | testShuffle(0, 100, true, seed, 11); 40 | } 41 | 42 | @Test 43 | public void testShuffleMedRange() { 44 | testShuffle(512, 10543, false, 13, 7); 45 | testShuffle(512, 10543, false, 13, 7, 27, 140); 46 | } 47 | 48 | @Test 49 | public void testShuffleLargeRange() { 50 | testShuffle(12345, 123456, false, 13, 7, 27, 140); 51 | } 52 | 53 | /** 54 | * Check that result is a valid permutation (i.e. a 1->1 mapping) 55 | * @param minId 56 | * @param maxId 57 | * @param params 58 | */ 59 | public static void testShuffle(int minId, int maxId, boolean print, 60 | long... params) { 61 | String shuffleDesc = String.format( 62 | "Permuting range [%d,%d) with params %s", minId, maxId, 63 | Arrays.toString(params)); 64 | 65 | if (print) { 66 | System.err.println(shuffleDesc); 67 | } 68 | 69 | 70 | int n = maxId - minId; 71 | 72 | 73 | //ProbDistShuffler shuf = new ProbDistShuffler(params[0], (int)params[1], n); 74 | InvertibleShuffler shuf = new InvertibleShuffler(params[0], (int)params[1], n); 75 | 76 | long reverse[] = new long[n]; // Store the reverse permutation 77 | // Store if ID has appeared (inited to false) 78 | boolean exists[] = new boolean[n]; 79 | 80 | for (int i = minId; i < maxId; i++) { 81 | //long lj = Shuffler.getPermutationValue(i, minId, maxId, params); 82 | long lj = minId + shuf.permute(i - minId); 83 | assertEquals(i, minId + shuf.invertPermute(lj - minId)); 84 | if (lj < minId || lj >= maxId) { 85 | fail(String.format("Error with test %s, permutation result p(%d) = %d" 86 | + " out of range [%d, %d)", shuffleDesc, i, lj, minId, maxId)); 87 | } 88 | assertTrue(lj >= minId); 89 | assertTrue(lj < maxId); 90 | int j = (int)lj; // Must be in integer range 91 | if (exists[j - minId]) { 92 | fail(String.format( 93 | "Error with test %s: collision. p(%d) = p(%d) = %d", 94 | shuffleDesc, i, reverse[j - minId], j)); 95 | } 96 | reverse[j - minId] = i; 97 | exists[j - minId] = true; 98 | if (print) { 99 | System.err.print(" " + j); 100 | } 101 | } 102 | if (print) { 103 | System.err.println(); 104 | } 105 | /* If we made it to here there were no collisions and we know all 106 | * n ids appeared in the permutation */ 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/LogNormalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Properties; 19 | 20 | import com.facebook.LinkBench.distributions.LogNormalDistribution; 21 | import com.facebook.LinkBench.distributions.ProbabilityDistribution; 22 | 23 | public class LogNormalTest extends DistributionTestBase { 24 | 25 | @Override 26 | protected ProbabilityDistribution getDist() { 27 | return new LogNormalDistribution(); 28 | } 29 | 30 | @Override 31 | protected Properties getDistParams() { 32 | Properties props = new Properties(); 33 | props.setProperty(LogNormalDistribution.CONFIG_SIGMA, "1"); 34 | props.setProperty(LogNormalDistribution.CONFIG_MEDIAN, "5000"); 35 | return props; 36 | } 37 | 38 | @Override 39 | protected double tolerance() { 40 | return 0.05; 41 | } 42 | 43 | /** 44 | * Sanity check values 45 | */ 46 | public void testLogNormal() { 47 | LogNormalDistribution d = new LogNormalDistribution(); 48 | int median = 10; 49 | d.init(0, 100, median, 1); 50 | // CDF of median should be 0.5 by def. 51 | assertEquals(0.5, d.cdf(median), 0.01); 52 | 53 | 54 | // Precomputed points 55 | d.init(0, 1000, 100, 1); 56 | assertEquals(0.033434, d.cdf(16), 0.0001); 57 | assertEquals(0.327695, d.cdf(64), 0.0001); 58 | assertEquals(0.597491, d.cdf(128), 0.0001); 59 | assertEquals(0.94878, d.cdf(512), 0.0001); 60 | } 61 | 62 | @Override 63 | public void testPDFSanity() { 64 | System.err.println("test not implemented"); 65 | } 66 | 67 | @Override 68 | public void testPDFSum() { 69 | System.err.println("test not implemented"); 70 | } 71 | 72 | @Override 73 | public void testCDFPDFConsistency() { 74 | System.err.println("test not implemented"); 75 | } 76 | 77 | @Override 78 | public void testQuantileSanity() { 79 | System.err.println("test not implemented"); 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MemoryGraphStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.util.Properties; 20 | 21 | public class MemoryGraphStoreTest extends GraphStoreTestBase { 22 | 23 | MemoryLinkStore store; 24 | @Override 25 | protected void initStore(Properties props) throws IOException, Exception { 26 | store = new MemoryLinkStore(); 27 | } 28 | 29 | @Override 30 | protected DummyLinkStore getStoreHandle(boolean initialized) throws IOException, Exception { 31 | return new DummyLinkStore(store.newHandle(), initialized); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MemoryLinkStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.util.Properties; 20 | 21 | public class MemoryLinkStoreTest extends LinkStoreTestBase { 22 | 23 | MemoryLinkStore store; 24 | 25 | @Override 26 | public void setUp() throws Exception { 27 | super.setUp(); 28 | } 29 | 30 | @Override 31 | protected Properties basicProps() { 32 | Properties props = super.basicProps(); 33 | props.setProperty(Config.LINKSTORE_CLASS, MemoryLinkStore.class.getName()); 34 | return props; 35 | } 36 | 37 | @Override 38 | protected void initStore(Properties props) throws IOException, 39 | Exception { 40 | store = new MemoryLinkStore(); 41 | } 42 | 43 | @Override 44 | protected DummyLinkStore getStoreHandle(boolean initialized) { 45 | // Return a new memory link store handle. The underlying link store doesn't 46 | // need to be initialized, so just set wrapper to correct init status 47 | return new DummyLinkStore(store.newHandle(), initialized); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MemoryNodeStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.util.Properties; 20 | 21 | public class MemoryNodeStoreTest extends NodeStoreTestBase { 22 | MemoryLinkStore store; 23 | @Override 24 | protected void initNodeStore(Properties props) throws Exception, IOException { 25 | store = new MemoryLinkStore(); 26 | store.initialize(props, Phase.REQUEST, 0); 27 | } 28 | 29 | @Override 30 | protected NodeStore getNodeStoreHandle(boolean initialized) throws Exception, IOException { 31 | return new DummyLinkStore(store.newHandle(), initialized); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbGraphStoreTest2.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.util.Properties; 20 | 21 | import org.junit.experimental.categories.Category; 22 | 23 | import com.mongodb.client.MongoDatabase; 24 | import com.facebook.LinkBench.testtypes.MongoDbTest; 25 | 26 | public class MongoDbGraphStoreTest2 extends GraphStoreTestBase { 27 | 28 | private Properties props; 29 | private MongoDatabase conn; 30 | 31 | @Override 32 | protected void initStore(Properties props) throws IOException, Exception { 33 | this.props = props; 34 | this.conn = MongoDbTestConfig2.createConnection(testDB); 35 | MongoDbTestConfig2.dropTestTables(conn, testDB); 36 | MongoDbTestConfig2.createTestTables(conn, testDB); 37 | } 38 | 39 | @Override 40 | protected long getIDCount() { 41 | // Make quicker 42 | return 1000; 43 | } 44 | 45 | @Override 46 | protected int getRequestCount() { 47 | // Make quicker, enough requests that we can reasonably check 48 | // that operation percentages are about about right 49 | return 20000; 50 | } 51 | 52 | @Override 53 | protected void tearDown() throws Exception { 54 | super.tearDown(); 55 | MongoDbTestConfig2.dropTestTables(conn, testDB); 56 | } 57 | 58 | @Override 59 | protected Properties basicProps() { 60 | Properties props = super.basicProps(); 61 | MongoDbTestConfig2.fillMongoDbTestServerProps(props); 62 | return props; 63 | } 64 | 65 | 66 | @Override 67 | protected DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 68 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMongoDb2()); 69 | if (initialize) { 70 | result.initialize(props, Phase.REQUEST, 0); 71 | } 72 | return result; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbGraphStoreTestBasic.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.util.Properties; 20 | 21 | import org.junit.experimental.categories.Category; 22 | 23 | import com.mongodb.client.MongoDatabase; 24 | import com.facebook.LinkBench.testtypes.MongoDbTest; 25 | 26 | public class MongoDbGraphStoreTestBasic extends GraphStoreTestBase { 27 | 28 | private Properties props; 29 | private MongoDatabase conn; 30 | 31 | @Override 32 | protected void initStore(Properties props) throws IOException, Exception { 33 | this.props = props; 34 | this.conn = MongoDbTestConfigBasic.createConnection(testDB); 35 | MongoDbTestConfigBasic.dropTestTables(conn, testDB); 36 | MongoDbTestConfigBasic.createTestTables(conn, testDB); 37 | } 38 | 39 | @Override 40 | protected long getIDCount() { 41 | // Make quicker 42 | return 1000; 43 | } 44 | 45 | @Override 46 | protected int getRequestCount() { 47 | // Make quicker, enough requests that we can reasonably check 48 | // that operation percentages are about about right 49 | return 20000; 50 | } 51 | 52 | @Override 53 | protected void tearDown() throws Exception { 54 | super.tearDown(); 55 | MongoDbTestConfigBasic.dropTestTables(conn, testDB); 56 | } 57 | 58 | @Override 59 | protected Properties basicProps() { 60 | Properties props = super.basicProps(); 61 | MongoDbTestConfigBasic.fillMongoDbTestServerProps(props); 62 | return props; 63 | } 64 | 65 | 66 | @Override 67 | protected DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 68 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMongoDbBasic()); 69 | if (initialize) { 70 | result.initialize(props, Phase.REQUEST, 0); 71 | } 72 | return result; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbLinkStoreTest2.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import com.mongodb.client.MongoDatabase; 4 | 5 | import java.io.IOException; 6 | import java.util.Properties; 7 | 8 | 9 | /** 10 | * Test the MongoDB LinkStore implementation. 11 | * 12 | * Assumes that the database specified by the testDB field has been created 13 | * with permissions for a user/pass linkbench/linkbench to create tables, select, 14 | * insert, delete, etc. 15 | */ 16 | public class MongoDbLinkStoreTest2 extends LinkStoreTestBase { 17 | 18 | private MongoDatabase conn; 19 | 20 | /** Properties for last initStore call */ 21 | private Properties currProps; 22 | 23 | @Override 24 | protected long getIDCount() { 25 | // Make test smaller so that it doesn't take too long 26 | return 2500; 27 | } 28 | 29 | @Override 30 | protected int getRequestCount() { 31 | // Fewer requests to keep test quick 32 | return 10000; 33 | } 34 | 35 | protected Properties basicProps() { 36 | Properties props = super.basicProps(); 37 | MongoDbTestConfig2.fillMongoDbTestServerProps(props); 38 | return props; 39 | } 40 | 41 | 42 | @Override 43 | protected void initStore(Properties props) throws IOException, Exception { 44 | this.currProps = (Properties)props.clone(); 45 | conn = MongoDbTestConfig2.createConnection(testDB); 46 | MongoDbTestConfig2.dropTestTables(conn, testDB); 47 | MongoDbTestConfig2.createTestTables(conn, testDB); 48 | } 49 | 50 | 51 | 52 | @Override 53 | public DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 54 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMongoDb2()); 55 | if (initialize) { 56 | result.initialize(currProps, Phase.REQUEST, 0); 57 | } 58 | return result; 59 | } 60 | 61 | @Override protected void tearDown() throws Exception { 62 | super.tearDown(); 63 | MongoDbTestConfig2.dropTestTables(conn, testDB); 64 | conn = null; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbLinkStoreTestBasic.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import com.mongodb.client.MongoDatabase; 4 | 5 | import java.io.IOException; 6 | import java.util.Properties; 7 | 8 | 9 | /** 10 | * Test the MongoDB LinkStore implementation. 11 | * 12 | * Assumes that the database specified by the testDB field has been created 13 | * with permissions for a user/pass linkbench/linkbench to create tables, select, 14 | * insert, delete, etc. 15 | */ 16 | public class MongoDbLinkStoreTestBasic extends LinkStoreTestBase { 17 | 18 | private MongoDatabase conn; 19 | 20 | /** Properties for last initStore call */ 21 | private Properties currProps; 22 | 23 | @Override 24 | protected long getIDCount() { 25 | // Make test smaller so that it doesn't take too long 26 | return 2500; 27 | } 28 | 29 | @Override 30 | protected int getRequestCount() { 31 | // Fewer requests to keep test quick 32 | return 10000; 33 | } 34 | 35 | protected Properties basicProps() { 36 | Properties props = super.basicProps(); 37 | MongoDbTestConfigBasic.fillMongoDbTestServerProps(props); 38 | return props; 39 | } 40 | 41 | 42 | @Override 43 | protected void initStore(Properties props) throws IOException, Exception { 44 | this.currProps = (Properties)props.clone(); 45 | conn = MongoDbTestConfigBasic.createConnection(testDB); 46 | MongoDbTestConfigBasic.dropTestTables(conn, testDB); 47 | MongoDbTestConfigBasic.createTestTables(conn, testDB); 48 | } 49 | 50 | 51 | 52 | @Override 53 | public DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 54 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMongoDbBasic()); 55 | if (initialize) { 56 | result.initialize(currProps, Phase.REQUEST, 0); 57 | } 58 | return result; 59 | } 60 | 61 | @Override protected void tearDown() throws Exception { 62 | super.tearDown(); 63 | MongoDbTestConfigBasic.dropTestTables(conn, testDB); 64 | conn = null; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbNodeStoreTest2.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import com.mongodb.client.MongoDatabase; 4 | 5 | import java.io.IOException; 6 | import java.util.Properties; 7 | 8 | 9 | public class MongoDbNodeStoreTest2 extends NodeStoreTestBase { 10 | 11 | MongoDatabase conn; 12 | Properties currProps; 13 | 14 | @Override 15 | protected Properties basicProps() { 16 | Properties props = super.basicProps(); 17 | MongoDbTestConfig2.fillMongoDbTestServerProps(props); 18 | return props; 19 | } 20 | 21 | @Override 22 | protected void initNodeStore(Properties props) throws Exception, IOException { 23 | currProps = props; 24 | conn = MongoDbTestConfig2.createConnection(testDB); 25 | MongoDbTestConfig2.dropTestTables(conn, testDB); 26 | MongoDbTestConfig2.createTestTables(conn, testDB); 27 | } 28 | 29 | @Override 30 | protected NodeStore getNodeStoreHandle(boolean initialize) throws Exception, IOException { 31 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMongoDb2()); 32 | if (initialize) { 33 | result.initialize(currProps, Phase.REQUEST, 0); 34 | } 35 | return result; 36 | } 37 | 38 | } 39 | 40 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbNodeStoreTestBasic.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import com.mongodb.client.MongoDatabase; 4 | 5 | import java.io.IOException; 6 | import java.util.Properties; 7 | 8 | 9 | public class MongoDbNodeStoreTestBasic extends NodeStoreTestBase { 10 | 11 | MongoDatabase conn; 12 | Properties currProps; 13 | 14 | @Override 15 | protected Properties basicProps() { 16 | Properties props = super.basicProps(); 17 | MongoDbTestConfigBasic.fillMongoDbTestServerProps(props); 18 | return props; 19 | } 20 | 21 | @Override 22 | protected void initNodeStore(Properties props) throws Exception, IOException { 23 | currProps = props; 24 | conn = MongoDbTestConfigBasic.createConnection(testDB); 25 | MongoDbTestConfigBasic.dropTestTables(conn, testDB); 26 | MongoDbTestConfigBasic.createTestTables(conn, testDB); 27 | } 28 | 29 | @Override 30 | protected NodeStore getNodeStoreHandle(boolean initialize) throws Exception, IOException { 31 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMongoDbBasic()); 32 | if (initialize) { 33 | result.initialize(currProps, Phase.REQUEST, 0); 34 | } 35 | return result; 36 | } 37 | 38 | } 39 | 40 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbTestConfig2.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import com.mongodb.MongoClient; 4 | import com.mongodb.MongoClientURI; 5 | import com.mongodb.client.MongoCollection; 6 | import com.mongodb.client.MongoDatabase; 7 | import com.mongodb.client.model.IndexOptions; 8 | import com.mongodb.client.model.Indexes; 9 | import org.bson.Document; 10 | import org.bson.conversions.Bson; 11 | 12 | import java.util.Map; 13 | import java.util.Properties; 14 | 15 | import static com.facebook.LinkBench.Config.*; 16 | import static com.facebook.LinkBench.GraphStore.*; 17 | import static com.facebook.LinkBench.LinkStoreMongoDb2.*; 18 | import static com.mongodb.client.model.Updates.combine; 19 | 20 | /** 21 | * Class containing hardcoded parameters and helper functions used to create 22 | * and connect to the unit test database for MongoDB. 23 | */ 24 | public class MongoDbTestConfig2 { 25 | 26 | // Hardcoded parameters for now 27 | @SuppressWarnings("FieldCanBeLocal") 28 | private static String host = "localhost"; 29 | 30 | @SuppressWarnings("FieldCanBeLocal") 31 | private static int port = 27017; 32 | 33 | @SuppressWarnings("FieldCanBeLocal") 34 | private static String user = "linkbench"; 35 | 36 | @SuppressWarnings("FieldCanBeLocal") 37 | private static String pass = "linkbench"; 38 | 39 | private static String url = "mongodb://localhost:27017/replset"; 40 | private static String linktable = "test_linktable"; 41 | private static String counttable = "test_counttable"; 42 | private static String nodetable = "test_nodetable"; 43 | 44 | static boolean getBooleanOrDefault(String name, boolean missing) { 45 | try { 46 | Map env = System.getenv(); 47 | if(env.containsKey(name)) { 48 | return Boolean.valueOf(env.get(name)); 49 | } else { 50 | return missing; 51 | } 52 | } catch (Exception e ) { 53 | } 54 | return missing; 55 | } 56 | static int getIntegerOrDefault(String name, int missing) { 57 | try { 58 | Map env = System.getenv(); 59 | if(env.containsKey(name)) { 60 | return Integer.valueOf(env.get(name)); 61 | } else { 62 | return missing; 63 | } 64 | } catch (Exception e ) { 65 | } 66 | return missing; 67 | } 68 | static void fillMongoDbTestServerProps(Properties props) { 69 | props.setProperty(LINKSTORE_CLASS, LinkStoreMongoDb2.class.getName()); 70 | props.setProperty(NODESTORE_CLASS, LinkStoreMongoDb2.class.getName()); 71 | props.setProperty(CONFIG_HOST, host); 72 | props.setProperty(CONFIG_PORT, Integer.toString(port)); 73 | props.setProperty(CONFIG_USER, user); 74 | props.setProperty(CONFIG_PASSWORD, pass); 75 | props.setProperty(CONFIG_URL, url); 76 | props.setProperty(CHECK_COUNT, Boolean.toString(getBooleanOrDefault(CHECK_COUNT, true))); 77 | 78 | props.setProperty(REQUEST_RANDOM_SEED, "12020569"); 79 | props.setProperty(LOAD_RANDOM_SEED, "26854520010"); 80 | 81 | props.setProperty(Config.LINK_TABLE, linktable); 82 | props.setProperty(Config.COUNT_TABLE, counttable); 83 | props.setProperty(Config.NODE_TABLE, nodetable); 84 | } 85 | 86 | static MongoDatabase createConnection(String testDB) { 87 | return new MongoClient(new MongoClientURI(url)).getDatabase(testDB); 88 | } 89 | 90 | 91 | private static MongoCollection createTestCollection(MongoDatabase database, String name) { 92 | database.createCollection(name); 93 | return database.getCollection(name); 94 | } 95 | 96 | private static void createIndex(MongoCollection collection, Bson index, boolean unqiue) { 97 | collection.createIndex(index, new IndexOptions().unique(unqiue)); 98 | } 99 | 100 | static void createTestTables(MongoDatabase database, String testDB) { 101 | MongoCollection collection; 102 | 103 | collection = createTestCollection(database,MongoDbTestConfig2.linktable); 104 | createIndex(collection, 105 | combine(Indexes.ascending("id1"), 106 | Indexes.ascending("link_type"), 107 | Indexes.ascending("visibility"), 108 | Indexes.ascending("time"), 109 | Indexes.ascending("id2"), 110 | Indexes.ascending("version"), 111 | Indexes.ascending("data")), 112 | false); 113 | 114 | collection = createTestCollection(database, MongoDbTestConfig2.nodetable); 115 | 116 | collection = createTestCollection(database, MongoDbTestConfig2.counttable); 117 | 118 | } 119 | 120 | static void dropTestTables(MongoDatabase conn, String testDB) { 121 | conn.getCollection(MongoDbTestConfig2.linktable).drop(); 122 | conn.getCollection(MongoDbTestConfig2.nodetable).drop(); 123 | conn.getCollection(MongoDbTestConfig2.counttable).drop(); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MongoDbTestConfigBasic.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import com.mongodb.MongoClient; 4 | import com.mongodb.MongoClientURI; 5 | import com.mongodb.client.MongoCollection; 6 | import com.mongodb.client.MongoDatabase; 7 | import com.mongodb.client.model.IndexOptions; 8 | import com.mongodb.client.model.Indexes; 9 | import org.bson.Document; 10 | import org.bson.conversions.Bson; 11 | 12 | import java.util.Map; 13 | import java.util.Properties; 14 | 15 | import static com.facebook.LinkBench.Config.*; 16 | import static com.facebook.LinkBench.LinkStoreMongoDbBasic.*; 17 | import static com.mongodb.client.model.Updates.combine; 18 | 19 | /** 20 | * Class containing hardcoded parameters and helper functions used to create 21 | * and connect to the unit test database for MongoDB. 22 | */ 23 | public class MongoDbTestConfigBasic { 24 | 25 | // Hardcoded parameters for now 26 | @SuppressWarnings("FieldCanBeLocal") 27 | private static String host = "localhost"; 28 | 29 | @SuppressWarnings("FieldCanBeLocal") 30 | private static int port = 27017; 31 | 32 | @SuppressWarnings("FieldCanBeLocal") 33 | private static String user = "linkbench"; 34 | 35 | @SuppressWarnings("FieldCanBeLocal") 36 | private static String pass = "linkbench"; 37 | 38 | private static String url = "mongodb://localhost:27017/replset"; 39 | private static String linktable = "test_linktable"; 40 | private static String counttable = "test_counttable"; 41 | private static String nodetable = "test_nodetable"; 42 | 43 | static boolean getBooleanOrDefault(String name, boolean missing) { 44 | try { 45 | Map env = System.getenv(); 46 | if(env.containsKey(name)) { 47 | return Boolean.valueOf(env.get(name)); 48 | } else { 49 | return missing; 50 | } 51 | } catch (Exception e ) { 52 | } 53 | return missing; 54 | } 55 | static int getIntegerOrDefault(String name, int missing) { 56 | try { 57 | Map env = System.getenv(); 58 | if(env.containsKey(name)) { 59 | return Integer.valueOf(env.get(name)); 60 | } else { 61 | return missing; 62 | } 63 | } catch (Exception e ) { 64 | } 65 | return missing; 66 | } 67 | static void fillMongoDbTestServerProps(Properties props) { 68 | props.setProperty(LINKSTORE_CLASS, LinkStoreMongoDbBasic.class.getName()); 69 | props.setProperty(NODESTORE_CLASS, LinkStoreMongoDbBasic.class.getName()); 70 | props.setProperty(CONFIG_HOST, host); 71 | props.setProperty(CONFIG_PORT, Integer.toString(port)); 72 | props.setProperty(CONFIG_USER, user); 73 | props.setProperty(CONFIG_PASSWORD, pass); 74 | props.setProperty(CONFIG_URL, url); 75 | props.setProperty(CHECK_COUNT, Boolean.toString(getBooleanOrDefault(CHECK_COUNT, true))); 76 | props.setProperty(SKIP_TRANSACTIONS, Boolean.toString(getBooleanOrDefault(SKIP_TRANSACTIONS, DEFAULT_SKIP_TRANSACTIONS))); 77 | props.setProperty(MAX_RETRIES, Integer.toString(getIntegerOrDefault(MAX_RETRIES, DEFAULT_MAX_RETRIES))); 78 | props.setProperty(BULKINSERT_SIZE, Integer.toString(getIntegerOrDefault(BULKINSERT_SIZE, DEFAULT_BULKINSERT_SIZE))); 79 | 80 | props.setProperty(REQUEST_RANDOM_SEED, "12020569"); 81 | props.setProperty(LOAD_RANDOM_SEED, "26854520010"); 82 | 83 | props.setProperty(Config.LINK_TABLE, linktable); 84 | props.setProperty(Config.COUNT_TABLE, counttable); 85 | props.setProperty(Config.NODE_TABLE, nodetable); 86 | } 87 | 88 | static MongoDatabase createConnection(String testDB) { 89 | return new MongoClient(new MongoClientURI(url)).getDatabase(testDB); 90 | } 91 | 92 | 93 | private static MongoCollection createTestCollection(MongoDatabase database, String name) { 94 | database.createCollection(name); 95 | return database.getCollection(name); 96 | } 97 | 98 | private static void createIndex(MongoCollection collection, Bson index, boolean unqiue) { 99 | collection.createIndex(index, new IndexOptions().unique(unqiue)); 100 | } 101 | 102 | static void createTestTables(MongoDatabase database, String testDB) { 103 | MongoCollection collection; 104 | 105 | collection = createTestCollection(database,MongoDbTestConfigBasic.linktable); 106 | createIndex(collection, 107 | combine(Indexes.ascending("id1"), 108 | Indexes.ascending("link_type"), 109 | Indexes.ascending("id2")), 110 | true); 111 | createIndex(collection, 112 | combine(Indexes.ascending("id1"), 113 | Indexes.ascending("link_type"), 114 | Indexes.ascending("visibility"), 115 | Indexes.ascending("time")), 116 | false); 117 | 118 | collection = createTestCollection(database, MongoDbTestConfigBasic.nodetable); 119 | createIndex(collection, combine(Indexes.ascending("id")), true); 120 | 121 | 122 | collection = createTestCollection(database, MongoDbTestConfigBasic.counttable); 123 | createIndex(collection, 124 | combine(Indexes.ascending("id"), 125 | Indexes.ascending("link_type")), true); 126 | 127 | } 128 | 129 | static void dropTestTables(MongoDatabase conn, String testDB) { 130 | conn.getCollection(MongoDbTestConfigBasic.linktable).drop(); 131 | conn.getCollection(MongoDbTestConfigBasic.nodetable).drop(); 132 | conn.getCollection(MongoDbTestConfigBasic.counttable).drop(); 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MySqlGraphStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.util.Properties; 21 | 22 | import org.junit.experimental.categories.Category; 23 | 24 | import com.facebook.LinkBench.testtypes.MySqlTest; 25 | 26 | public class MySqlGraphStoreTest extends GraphStoreTestBase { 27 | 28 | private Properties props; 29 | private Connection conn; 30 | 31 | @Override 32 | protected void initStore(Properties props) throws IOException, Exception { 33 | this.props = props; 34 | this.conn = MySqlTestConfig.createConnection(testDB); 35 | MySqlTestConfig.dropTestTables(conn, testDB); 36 | MySqlTestConfig.createTestTables(conn, testDB); 37 | } 38 | 39 | @Override 40 | protected long getIDCount() { 41 | // Make quicker 42 | return 1000; 43 | } 44 | 45 | @Override 46 | protected int getRequestCount() { 47 | // Make quicker, enough requests that we can reasonably check 48 | // that operation percentages are about about right 49 | return 20000; 50 | } 51 | 52 | @Override 53 | protected void tearDown() throws Exception { 54 | super.tearDown(); 55 | MySqlTestConfig.dropTestTables(conn, testDB); 56 | } 57 | 58 | @Override 59 | protected Properties basicProps() { 60 | Properties props = super.basicProps(); 61 | MySqlTestConfig.fillMySqlTestServerProps(props); 62 | return props; 63 | } 64 | 65 | 66 | @Override 67 | protected DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 68 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMysql()); 69 | if (initialize) { 70 | result.initialize(props, Phase.REQUEST, 0); 71 | } 72 | return result; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MySqlLinkStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.util.Properties; 21 | 22 | import org.junit.experimental.categories.Category; 23 | 24 | import com.facebook.LinkBench.testtypes.MySqlTest; 25 | 26 | /** 27 | * Test the MySQL LinkStore implementation. 28 | * 29 | * Assumes that the database specified by the testDB field has been created 30 | * with permissions for a user/pass linkbench/linkbench to create tables, select, 31 | * insert, delete, etc. 32 | */ 33 | public class MySqlLinkStoreTest extends LinkStoreTestBase { 34 | 35 | private Connection conn; 36 | 37 | /** Properties for last initStore call */ 38 | private Properties currProps; 39 | 40 | @Override 41 | protected long getIDCount() { 42 | // Make test smaller so that it doesn't take too long 43 | return 5000; 44 | } 45 | 46 | @Override 47 | protected int getRequestCount() { 48 | // Fewer requests to keep test quick 49 | return 20000; 50 | } 51 | 52 | protected Properties basicProps() { 53 | Properties props = super.basicProps(); 54 | MySqlTestConfig.fillMySqlTestServerProps(props); 55 | return props; 56 | } 57 | 58 | 59 | @Override 60 | protected void initStore(Properties props) throws IOException, Exception { 61 | this.currProps = (Properties)props.clone(); 62 | if (conn != null) { 63 | conn.close(); 64 | } 65 | conn = MySqlTestConfig.createConnection(testDB); 66 | MySqlTestConfig.dropTestTables(conn, testDB); 67 | MySqlTestConfig.createTestTables(conn, testDB); 68 | } 69 | 70 | 71 | 72 | @Override 73 | public DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 74 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMysql()); 75 | if (initialize) { 76 | result.initialize(currProps, Phase.REQUEST, 0); 77 | } 78 | return result; 79 | } 80 | 81 | @Override protected void tearDown() throws Exception { 82 | super.tearDown(); 83 | MySqlTestConfig.dropTestTables(conn, testDB); 84 | conn.close(); 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MySqlNodeStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.util.Properties; 21 | 22 | import org.junit.experimental.categories.Category; 23 | 24 | import com.facebook.LinkBench.testtypes.MySqlTest; 25 | 26 | public class MySqlNodeStoreTest extends NodeStoreTestBase { 27 | 28 | Connection conn; 29 | Properties currProps; 30 | 31 | @Override 32 | protected Properties basicProps() { 33 | Properties props = super.basicProps(); 34 | MySqlTestConfig.fillMySqlTestServerProps(props); 35 | return props; 36 | } 37 | 38 | @Override 39 | protected void initNodeStore(Properties props) throws Exception, IOException { 40 | currProps = props; 41 | conn = MySqlTestConfig.createConnection(testDB); 42 | MySqlTestConfig.dropTestTables(conn, testDB); 43 | MySqlTestConfig.createTestTables(conn, testDB); 44 | } 45 | 46 | @Override 47 | protected NodeStore getNodeStoreHandle(boolean initialize) throws Exception, IOException { 48 | DummyLinkStore result = new DummyLinkStore(new LinkStoreMysql()); 49 | if (initialize) { 50 | result.initialize(currProps, Phase.REQUEST, 0); 51 | } 52 | return result; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/MySqlTestConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.sql.Connection; 19 | import java.sql.DriverManager; 20 | import java.sql.SQLException; 21 | import java.sql.Statement; 22 | import java.util.Properties; 23 | 24 | /** 25 | * Class containing hardcoded parameters and helper functions used to create 26 | * and connect to the unit test database for MySql 27 | * @author tarmstrong 28 | */ 29 | public class MySqlTestConfig { 30 | 31 | // Hardcoded parameters for now 32 | static String host = "localhost"; 33 | static int port = 3306; 34 | static String user = "root"; 35 | static String pass = "pw"; 36 | static String linktable = "test_linktable"; 37 | static String counttable = "test_counttable"; 38 | static String nodetable = "test_nodetable"; 39 | 40 | public static void fillMySqlTestServerProps(Properties props) { 41 | props.setProperty(Config.LINKSTORE_CLASS, LinkStoreMysql.class.getName()); 42 | props.setProperty(Config.NODESTORE_CLASS, LinkStoreMysql.class.getName()); 43 | props.setProperty(LinkStoreMysql.CONFIG_HOST, host); 44 | props.setProperty(LinkStoreMysql.CONFIG_PORT, Integer.toString(port)); 45 | props.setProperty(LinkStoreMysql.CONFIG_USER, user); 46 | props.setProperty(LinkStoreMysql.CONFIG_PASSWORD, pass); 47 | props.setProperty(Config.LINK_TABLE, linktable); 48 | props.setProperty(Config.COUNT_TABLE, counttable); 49 | props.setProperty(Config.NODE_TABLE, nodetable); 50 | } 51 | 52 | static Connection createConnection(String testDB) 53 | throws InstantiationException, 54 | IllegalAccessException, ClassNotFoundException, SQLException { 55 | Class.forName("com.mysql.jdbc.Driver").newInstance(); 56 | String defDb = "test"; 57 | return DriverManager.getConnection( 58 | "jdbc:mysql://"+ MySqlTestConfig.host + ":" + 59 | MySqlTestConfig.port + "/" + defDb + 60 | "?elideSetAutoCommits=true" + 61 | "&useLocalTransactionState=true" + 62 | "&allowMultiQueries=true" + 63 | "&useLocalSessionState=true", 64 | MySqlTestConfig.user, MySqlTestConfig.pass); 65 | } 66 | 67 | 68 | static void createTestTables(Connection conn, String testDB) 69 | throws SQLException { 70 | Statement stmt = conn.createStatement(); 71 | stmt.executeUpdate("DROP DATABASE IF EXISTS " + testDB); 72 | stmt.executeUpdate("CREATE DATABASE " + testDB); 73 | stmt.executeUpdate(String.format( 74 | "CREATE TABLE `%s`.`%s` (" + 75 | "`id1` bigint(20) unsigned NOT NULL DEFAULT '0'," + 76 | "`id2` bigint(20) unsigned NOT NULL DEFAULT '0'," + 77 | "`link_type` bigint(20) unsigned NOT NULL DEFAULT '0'," + 78 | "`visibility` tinyint(3) NOT NULL DEFAULT '0'," + 79 | "`data` varchar(255) NOT NULL DEFAULT ''," + 80 | "`time` bigint(20) unsigned NOT NULL DEFAULT '0'," + 81 | "`version` int(11) unsigned NOT NULL DEFAULT '0'," + 82 | "PRIMARY KEY (`id1`,`id2`,`link_type`)," + 83 | "KEY `id1_type` (`id1`,`link_type`,`visibility`,`time`,`version`,`data`)" + 84 | ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", 85 | testDB, MySqlTestConfig.linktable)); 86 | stmt.executeUpdate(String.format("CREATE TABLE `%s`.`%s` (" + 87 | "`id` bigint(20) unsigned NOT NULL DEFAULT '0'," + 88 | "`link_type` bigint(20) unsigned NOT NULL DEFAULT '0'," + 89 | "`count` int(10) unsigned NOT NULL DEFAULT '0'," + 90 | "`time` bigint(20) unsigned NOT NULL DEFAULT '0'," + 91 | "`version` bigint(20) unsigned NOT NULL DEFAULT '0'," + 92 | "PRIMARY KEY (`id`,`link_type`)" + 93 | ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", 94 | testDB, MySqlTestConfig.counttable)); 95 | stmt.executeUpdate(String.format( 96 | "CREATE TABLE `%s`.`%s` (" + 97 | "`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT," + 98 | "`type` int(10) unsigned NOT NULL," + 99 | "`version` bigint(20) unsigned NOT NULL," + 100 | "`time` int(10) unsigned NOT NULL," + 101 | "`data` mediumtext NOT NULL," + 102 | "primary key(`id`)" + 103 | ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", 104 | testDB, MySqlTestConfig.nodetable)); 105 | } 106 | 107 | static void dropTestTables(Connection conn, String testDB) 108 | throws SQLException { 109 | Statement stmt = conn.createStatement(); 110 | stmt.executeUpdate(String.format("DROP TABLE IF EXISTS `%s`.`%s`;", 111 | testDB, MySqlTestConfig.linktable)); 112 | stmt.executeUpdate(String.format("DROP TABLE IF EXISTS `%s`.`%s`;", 113 | testDB, MySqlTestConfig.counttable)); 114 | stmt.executeUpdate(String.format("DROP TABLE IF EXISTS `%s`.`%s`;", 115 | testDB, MySqlTestConfig.nodetable)); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/PgsqlGraphStoreTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import java.io.IOException; 4 | import java.sql.Connection; 5 | import java.util.Properties; 6 | 7 | import com.facebook.LinkBench.testtypes.PgsqlTest; 8 | 9 | public class PgsqlGraphStoreTest extends GraphStoreTestBase { 10 | 11 | private Properties props; 12 | private Connection conn; 13 | 14 | protected void initStore(Properties props) throws IOException, Exception { 15 | this.props = props; 16 | this.conn = PgsqlTestConfig.createConnection(testDB); 17 | PgsqlTestConfig.dropTestTables(conn, testDB); 18 | PgsqlTestConfig.createTestTables(conn, testDB); 19 | } 20 | 21 | protected long getIDCount() { 22 | // Make quicker 23 | return 1000; 24 | } 25 | 26 | protected int getRequestCount() { 27 | // Make quicker, enough requests that we can reasonably check 28 | // that operation percentages are about about right 29 | return 20000; 30 | } 31 | 32 | protected void tearDown() throws Exception { 33 | super.tearDown(); 34 | PgsqlTestConfig.dropTestTables(conn, testDB); 35 | } 36 | 37 | protected Properties basicProps() { 38 | Properties props = super.basicProps(); 39 | PgsqlTestConfig.fillPgsqlTestServerProps(props); 40 | return props; 41 | } 42 | 43 | 44 | protected DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 45 | DummyLinkStore result = new DummyLinkStore(new LinkStorePgsql()); 46 | if (initialize) { 47 | result.initialize(props, Phase.REQUEST, 0); 48 | } 49 | return result; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/PgsqlLinkStoreTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import java.io.IOException; 4 | import java.sql.Connection; 5 | import java.util.Properties; 6 | 7 | import com.facebook.LinkBench.testtypes.PgsqlTest; 8 | 9 | /** 10 | * Test the Postgres LinkStore implementation. 11 | * 12 | * Assumes that the database specified by the testDB field has been created 13 | * with permissions for a user/pass linkbench/linkbench to create tables, select, 14 | * insert, delete, etc. 15 | */ 16 | 17 | public class PgsqlLinkStoreTest extends LinkStoreTestBase { 18 | 19 | private Connection conn; 20 | 21 | /** Properties for last initStore call */ 22 | private Properties currProps; 23 | 24 | protected long getIDCount() { 25 | // Make test smaller so that it doesn't take too long 26 | return 5000; 27 | } 28 | 29 | protected int getRequestCount() { 30 | // Fewer requests to keep test quick 31 | return 20000; 32 | } 33 | 34 | protected Properties basicProps() { 35 | Properties props = super.basicProps(); 36 | PgsqlTestConfig.fillPgsqlTestServerProps(props); 37 | return props; 38 | } 39 | 40 | 41 | protected void initStore(Properties props) throws IOException, Exception { 42 | this.currProps = (Properties)props.clone(); 43 | if (conn != null) { 44 | conn.close(); 45 | } 46 | conn = PgsqlTestConfig.createConnection(testDB); 47 | PgsqlTestConfig.dropTestTables(conn, testDB); 48 | PgsqlTestConfig.createTestTables(conn, testDB); 49 | } 50 | 51 | 52 | public DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { 53 | DummyLinkStore result = new DummyLinkStore(new LinkStorePgsql()); 54 | if (initialize) { 55 | result.initialize(currProps, Phase.REQUEST, 0); 56 | } 57 | return result; 58 | } 59 | 60 | protected void tearDown() throws Exception { 61 | super.tearDown(); 62 | PgsqlTestConfig.dropTestTables(conn, testDB); 63 | conn.close(); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/PgsqlNodeStoreTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import java.io.IOException; 4 | import java.sql.Connection; 5 | import java.util.Properties; 6 | 7 | import com.facebook.LinkBench.testtypes.PgsqlTest; 8 | 9 | public class PgsqlNodeStoreTest extends NodeStoreTestBase { 10 | 11 | Connection conn; 12 | Properties currProps; 13 | 14 | protected Properties basicProps() { 15 | Properties props = super.basicProps(); 16 | PgsqlTestConfig.fillPgsqlTestServerProps(props); 17 | return props; 18 | } 19 | 20 | protected void initNodeStore(Properties props) throws Exception, IOException { 21 | currProps = props; 22 | conn = PgsqlTestConfig.createConnection(testDB); 23 | PgsqlTestConfig.dropTestTables(conn, testDB); 24 | PgsqlTestConfig.createTestTables(conn, testDB); 25 | } 26 | 27 | protected NodeStore getNodeStoreHandle(boolean initialize) throws Exception, IOException { 28 | DummyLinkStore result = new DummyLinkStore(new LinkStorePgsql()); 29 | if (initialize) { 30 | result.initialize(currProps, Phase.REQUEST, 0); 31 | } 32 | return result; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/PgsqlTestConfig.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | import java.sql.Statement; 7 | import java.util.Properties; 8 | 9 | public class PgsqlTestConfig { 10 | 11 | // Hardcoded parameters for now 12 | static String host = "localhost"; 13 | static int port = 5432; 14 | static String user = "linkbench"; 15 | static String pass = "pw"; 16 | static String linktable = "test_linktable"; 17 | static String counttable = "test_counttable"; 18 | static String nodetable = "test_nodetable"; 19 | 20 | public static void fillPgsqlTestServerProps(Properties props) { 21 | props.setProperty(Config.LINKSTORE_CLASS, LinkStorePgsql.class.getName()); 22 | props.setProperty(Config.NODESTORE_CLASS, LinkStorePgsql.class.getName()); 23 | props.setProperty(LinkStorePgsql.CONFIG_HOST, host); 24 | props.setProperty(LinkStorePgsql.CONFIG_PORT, Integer.toString(port)); 25 | props.setProperty(LinkStorePgsql.CONFIG_USER, user); 26 | props.setProperty(LinkStorePgsql.CONFIG_PASSWORD, pass); 27 | props.setProperty(Config.LINK_TABLE, linktable); 28 | props.setProperty(Config.COUNT_TABLE, counttable); 29 | props.setProperty(Config.NODE_TABLE, nodetable); 30 | } 31 | 32 | static Connection createConnection(String testDB) 33 | throws InstantiationException, 34 | IllegalAccessException, ClassNotFoundException, SQLException, ReflectiveOperationException { 35 | Logger.getLogger().info("create connection"); 36 | Class.forName("org.postgresql.Driver").getConstructor().newInstance(); 37 | String jdbcUrl = "jdbc:postgresql://"+ host + ":" + port + "/linkbench"; 38 | jdbcUrl += "?elideSetAutoCommits=true" + 39 | "&useLocalTransactionState=true" + 40 | "&allowMultiQueries=true" + 41 | "&useLocalSessionState=true" + 42 | "&useAffectedRows=true"; 43 | 44 | return DriverManager.getConnection(jdbcUrl, PgsqlTestConfig.user, PgsqlTestConfig.pass); 45 | } 46 | 47 | static void createTestTables(Connection conn, String testDB) throws SQLException { 48 | Logger.getLogger().info("createTestTables"); 49 | Statement stmt = conn.createStatement(); 50 | stmt.executeUpdate("DROP SCHEMA IF EXISTS " + testDB + " CASCADE"); 51 | stmt.executeUpdate("CREATE SCHEMA " + testDB); 52 | 53 | stmt.executeUpdate(String.format( 54 | "CREATE TABLE %s.%s ( " + 55 | " id1 bigint NOT NULL DEFAULT '0', " + 56 | " id2 bigint NOT NULL DEFAULT '0', " + 57 | " link_type bigint NOT NULL DEFAULT '0', " + 58 | " visibility smallint NOT NULL DEFAULT '0', " + 59 | " data bytea NOT NULL , " + 60 | " time bigint NOT NULL DEFAULT '0', " + 61 | " version int NOT NULL DEFAULT '0', " + 62 | " PRIMARY KEY (link_type, id1,id2))", 63 | testDB, PgsqlTestConfig.linktable)); 64 | 65 | stmt.executeUpdate(String.format( 66 | "CREATE INDEX id1_type on %s.%s ( " + 67 | " id1,link_type,visibility,time,id2,version,data)", 68 | testDB, PgsqlTestConfig.linktable)); 69 | 70 | stmt.executeUpdate(String.format( 71 | "CREATE TABLE %s.%s ( " + 72 | " id bigint NOT NULL DEFAULT '0', " + 73 | " link_type bigint NOT NULL DEFAULT '0', " + 74 | " count bigint NOT NULL DEFAULT '0', " + 75 | " time bigint NOT NULL DEFAULT '0', " + 76 | " version bigint NOT NULL DEFAULT '0', " + 77 | " PRIMARY KEY (id,link_type))", 78 | testDB, PgsqlTestConfig.counttable)); 79 | 80 | stmt.executeUpdate(String.format( 81 | "CREATE TABLE %s.%s ( " + 82 | " id BIGSERIAL NOT NULL, " + 83 | " type int NOT NULL, " + 84 | " version bigint NOT NULL, " + 85 | " time int NOT NULL, " + 86 | " data bytea NOT NULL, " + 87 | " PRIMARY KEY(id))", 88 | testDB, PgsqlTestConfig.nodetable)); 89 | 90 | stmt.close(); 91 | } 92 | 93 | static void dropTestTables(Connection conn, String testDB) throws SQLException { 94 | Logger.getLogger().info("dropTestTables"); 95 | Statement stmt = conn.createStatement(); 96 | int rlink = stmt.executeUpdate(String.format("DROP TABLE IF EXISTS %s.%s", 97 | testDB, PgsqlTestConfig.linktable)); 98 | int rcount = stmt.executeUpdate(String.format("DROP TABLE IF EXISTS %s.%s", 99 | testDB, PgsqlTestConfig.counttable)); 100 | int rnode = stmt.executeUpdate(String.format("DROP TABLE IF EXISTS %s.%s", 101 | testDB, PgsqlTestConfig.nodetable)); 102 | if (rlink != 0 || rcount != 0 || rnode != 0) { 103 | throw new IllegalStateException("dropTestTables failed with (link,count,node)=(" + 104 | rlink + "," + rcount + "," + rnode + ")"); 105 | } 106 | stmt.close(); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/PiecewiseDistTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.ArrayList; 19 | import java.util.Properties; 20 | 21 | import com.facebook.LinkBench.distributions.PiecewiseLinearDistribution; 22 | import com.facebook.LinkBench.distributions.PiecewiseLinearDistribution.Point; 23 | import com.facebook.LinkBench.distributions.ProbabilityDistribution; 24 | 25 | public class PiecewiseDistTest extends DistributionTestBase { 26 | 27 | ArrayList testDistribution = null; 28 | @Override 29 | public void setUp() throws Exception { 30 | super.setUp(); 31 | // Make up an arbitrary distribution 32 | testDistribution = new ArrayList(); 33 | testDistribution.add(new Point(0, 0.1)); 34 | testDistribution.add(new Point(1, 0.15)); 35 | testDistribution.add(new Point(2, 0.17)); 36 | testDistribution.add(new Point(3, 0.20)); 37 | testDistribution.add(new Point(4, 0.23)); 38 | testDistribution.add(new Point(10, 0.26)); 39 | testDistribution.add(new Point(20, 0.4)); 40 | testDistribution.add(new Point(30, 0.45)); 41 | testDistribution.add(new Point(40, 0.6)); 42 | testDistribution.add(new Point(55, 0.64)); 43 | testDistribution.add(new Point(70, 0.70)); 44 | testDistribution.add(new Point(90, 0.75)); 45 | testDistribution.add(new Point(100, 0.82)); 46 | testDistribution.add(new Point(110, 0.92)); 47 | testDistribution.add(new Point(120, 1.0)); 48 | } 49 | 50 | @Override 51 | protected int cdfChecks() { 52 | return 50; 53 | } 54 | 55 | @Override 56 | protected ProbabilityDistribution getDist() { 57 | return new PiecewiseLinearDistribution() { 58 | 59 | @Override 60 | public void init(long min, long max, Properties props, String keyPrefix) { 61 | init(min, max, testDistribution); 62 | } 63 | }; 64 | } 65 | 66 | @Override 67 | public void testCDFSanity() { 68 | System.err.println("CDF not implemented"); 69 | } 70 | 71 | @Override 72 | public void testCDFChooseConsistency() { 73 | System.err.println("CDF not implemented"); 74 | } 75 | 76 | @Override 77 | public void testCDFPDFConsistency() { 78 | System.err.println("CDF not implemented"); 79 | } 80 | 81 | @Override 82 | public void testQuantileSanity() { 83 | System.err.println("Quantile not implemented"); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/TestAccessDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.File; 19 | import java.util.Properties; 20 | import java.util.Random; 21 | 22 | import org.junit.Test; 23 | 24 | import com.facebook.LinkBench.RealDistribution.DistributionType; 25 | import com.facebook.LinkBench.distributions.AccessDistributions.AccessDistMode; 26 | import com.facebook.LinkBench.distributions.AccessDistributions.AccessDistribution; 27 | import com.facebook.LinkBench.distributions.AccessDistributions.BuiltinAccessDistribution; 28 | import com.facebook.LinkBench.distributions.AccessDistributions.ProbAccessDistribution; 29 | import com.facebook.LinkBench.distributions.UniformDistribution; 30 | import com.facebook.LinkBench.distributions.ZipfDistribution; 31 | 32 | import junit.framework.AssertionFailedError; 33 | import junit.framework.TestCase; 34 | 35 | public class TestAccessDistribution extends TestCase { 36 | 37 | @Test 38 | public void testMultiple() { 39 | testSanityBuiltinDist(AccessDistMode.MULTIPLE, 3); 40 | } 41 | 42 | @Test 43 | public void testPerfectPower() { 44 | testSanityBuiltinDist(AccessDistMode.PERFECT_POWER, 3); 45 | } 46 | 47 | @Test 48 | public void testPower() { 49 | testSanityBuiltinDist(AccessDistMode.POWER, 3); 50 | } 51 | 52 | @Test 53 | public void testReciprocal() { 54 | testSanityBuiltinDist(AccessDistMode.RECIPROCAL, 3); 55 | } 56 | 57 | @Test 58 | public void testRoundRobin() { 59 | testSanityBuiltinDist(AccessDistMode.ROUND_ROBIN, 0); 60 | } 61 | 62 | @Test 63 | public void testUniform() { 64 | UniformDistribution u = new UniformDistribution(); 65 | Properties props = new Properties(); 66 | int min = 100, max = 200; 67 | u.init(min, max, props, ""); 68 | ProbAccessDistribution unshuffled = new ProbAccessDistribution(u, null); 69 | testSanityAccessDist(unshuffled, min, max); 70 | 71 | ProbAccessDistribution shuffled = new ProbAccessDistribution(u, 72 | new InvertibleShuffler(13, 25, max - min)); 73 | testSanityAccessDist(shuffled, min, max); 74 | } 75 | 76 | @Test 77 | public void testZipf() { 78 | ZipfDistribution z = new ZipfDistribution(); 79 | Properties props = new Properties(); 80 | props.setProperty("shape", "0.5"); 81 | int min = 100, max = 200; 82 | z.init(min, max, props, ""); 83 | ProbAccessDistribution unshuffled = new ProbAccessDistribution(z, null); 84 | testSanityAccessDist(unshuffled, min, max); 85 | 86 | ProbAccessDistribution shuffled = new ProbAccessDistribution(z, 87 | new InvertibleShuffler(13, 25, max - min)); 88 | testSanityAccessDist(shuffled, min, max); 89 | } 90 | 91 | 92 | @Test 93 | public void testReal() { 94 | RealDistribution r = new RealDistribution(); 95 | Properties props = new Properties(); 96 | props.setProperty(Config.DISTRIBUTION_DATA_FILE, 97 | new File("config/Distribution.dat").getAbsolutePath()); 98 | int min = 100, max = 200; 99 | r.init(props, min, max, DistributionType.LINK_READS); 100 | ProbAccessDistribution unshuffled = new ProbAccessDistribution(r, null); 101 | testSanityAccessDist(unshuffled, min, max); 102 | 103 | ProbAccessDistribution shuffled = new ProbAccessDistribution(r, 104 | new InvertibleShuffler(13, 25, max - min)); 105 | testSanityAccessDist(shuffled, min, max); 106 | } 107 | 108 | 109 | 110 | public static void testSanityBuiltinDist(AccessDistMode mode, long config) { 111 | long minid = 123; 112 | long maxid = 12345; 113 | BuiltinAccessDistribution dist = new BuiltinAccessDistribution(mode, minid, 114 | maxid, config); 115 | testSanityAccessDist(dist, minid, maxid); 116 | } 117 | 118 | /** 119 | * Check that results are in range, etc. 120 | */ 121 | public static void testSanityAccessDist(AccessDistribution dist, long minid, 122 | long maxid) { 123 | long seed = System.currentTimeMillis(); 124 | System.err.println("Using seed " + seed); 125 | Random rng = new Random(seed); 126 | long id = 1; 127 | int trials = 10000; 128 | long start = System.currentTimeMillis(); 129 | for (int i = 0; i < trials; i++) { 130 | id = dist.nextID(rng, id); 131 | try { 132 | assertTrue(id >= minid); 133 | assertTrue(id < maxid); 134 | } catch (AssertionFailedError e) { 135 | System.err.println("Error: on trial " + i + " id returned: " 136 | + id + " not in range [" + minid + "," + maxid + ")"); 137 | throw e; 138 | } 139 | } 140 | long end = System.currentTimeMillis(); 141 | System.err.println("Took " + (end - start) + " ms for " + trials 142 | + " trials"); 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/TestHdrLatencyHistogram.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-present MongoDB Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.io.PrintStream; 19 | import java.io.ByteArrayOutputStream; 20 | import java.lang.ArrayIndexOutOfBoundsException; 21 | 22 | import org.junit.Test; 23 | import static org.junit.Assert.*; 24 | import com.facebook.LinkBench.stats.HdrLatencyHistogram; 25 | import com.facebook.LinkBench.stats.LatencyHistogram; 26 | 27 | public class TestHdrLatencyHistogram { 28 | 29 | private static final String header = 30 | "op,count,p25 (us),p50 (us),p75 (us),p95 (us),p99 (us),max (us),mean (us),threads\n"; 31 | 32 | @Test 33 | public void testSimpleDistribution() { 34 | LatencyHistogram histogram = new HdrLatencyHistogram(8, 3, 100); 35 | 36 | for(int i = 1; i <= 100; i++) 37 | { 38 | histogram.recordLatency(0, LinkBenchOp.ADD_LINK, i); 39 | } 40 | 41 | String result = getCsvResults(histogram); 42 | String metrics = "ADD_LINK,100,25,50,75,95,99,100,50.5,8\n"; 43 | 44 | assertEquals(header + metrics, result); 45 | } 46 | 47 | @Test 48 | public void testNoDataPoints() { 49 | LatencyHistogram histogram = new HdrLatencyHistogram(8, 3, 100); 50 | String results = getCsvResults(histogram); 51 | assertEquals(header, results); 52 | } 53 | 54 | @Test 55 | public void testNoHeaderNoData() { 56 | LatencyHistogram histogram = new HdrLatencyHistogram(8, 3, 100); 57 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 58 | PrintStream ps = new PrintStream(os); 59 | 60 | histogram.printCSVStats(ps, false); 61 | String result = os.toString(); 62 | 63 | assertEquals("", result); 64 | } 65 | 66 | @Test 67 | public void testLogsLatencyValues() { 68 | HdrLatencyHistogram histogram = new HdrLatencyHistogram(8, 3, 5); 69 | histogram.recordLatency(0, LinkBenchOp.ADD_LINK, 1); 70 | histogram.displayLatencyStats(); 71 | } 72 | 73 | @Test 74 | public void testLogNoStats() { 75 | HdrLatencyHistogram histogram = new HdrLatencyHistogram(8, 3, 5); 76 | histogram.displayLatencyStats(); 77 | } 78 | 79 | @Test(expected = ArrayIndexOutOfBoundsException.class) 80 | public void testOverMax() throws ArrayIndexOutOfBoundsException { 81 | HdrLatencyHistogram histogram = new HdrLatencyHistogram(8, 3, 5); 82 | histogram.recordLatency(0, LinkBenchOp.ADD_LINK, 1000000); 83 | } 84 | 85 | String getCsvResults(LatencyHistogram histogram) { 86 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 87 | PrintStream ps = new PrintStream(os); 88 | histogram.printCSVStats(ps, true); 89 | return os.toString(); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/TestLatencyHistogramFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-present MongoDB Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Properties; 19 | 20 | import org.junit.Test; 21 | import static org.junit.Assert.*; 22 | 23 | import com.facebook.LinkBench.stats.HdrLatencyHistogram; 24 | import com.facebook.LinkBench.stats.LatencyHistogram; 25 | import com.facebook.LinkBench.stats.LatencyHistogramFactory; 26 | import com.facebook.LinkBench.stats.LatencyStats; 27 | 28 | public class TestLatencyHistogramFactory { 29 | 30 | private static final Logger logger = Logger.getLogger(); 31 | 32 | @Test(expected = LinkBenchConfigError.class) 33 | public void testMissingBothConfigs() { 34 | Properties props = new Properties(); 35 | props.setProperty("use_hdr_histogram", "true"); 36 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 37 | factory.create(8, props); 38 | } 39 | 40 | @Test(expected = LinkBenchConfigError.class) 41 | public void testMissingAccuracyConfig() { 42 | Properties props = new Properties(); 43 | props.setProperty("use_hdr_histogram", "true"); 44 | props.setProperty("hdr_histogram_max_latency", "8"); 45 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 46 | factory.create(8, props); 47 | } 48 | 49 | @Test(expected = LinkBenchConfigError.class) 50 | public void testMissingMaxLatencyConfig() { 51 | Properties props = new Properties(); 52 | props.setProperty("use_hdr_histogram", "true"); 53 | props.setProperty("hdr_histogram_accuracy", "8"); 54 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 55 | factory.create(8, props); 56 | } 57 | 58 | @Test(expected = LinkBenchConfigError.class) 59 | public void testConfigsNotNumbers() { 60 | Properties props = new Properties(); 61 | props.setProperty("use_hdr_histogram", "true"); 62 | props.setProperty("hdr_histogram_accuracy", "test"); 63 | props.setProperty("hdr_histogram_max_latency", "test"); 64 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 65 | factory.create(8, props); 66 | } 67 | 68 | @Test 69 | public void testUseHistogramNotBool() { 70 | Properties props = new Properties(); 71 | props.setProperty("use_hdr_histogram", "test"); 72 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 73 | 74 | LatencyHistogram histogram = factory.create(8, props); 75 | 76 | assertTrue(histogram instanceof LatencyStats); 77 | } 78 | 79 | @Test 80 | public void testUseHistogramFalse() { 81 | Properties props = new Properties(); 82 | props.setProperty("use_hdr_histogram", "false"); 83 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 84 | 85 | LatencyHistogram histogram = factory.create(8, props); 86 | 87 | assertTrue(histogram instanceof LatencyStats); 88 | } 89 | 90 | @Test 91 | public void testUseHistogramNothingSpecified() { 92 | Properties props = new Properties(); 93 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 94 | LatencyHistogram histogram = factory.create(8, props); 95 | assertTrue(histogram instanceof LatencyStats); 96 | } 97 | 98 | 99 | @Test 100 | public void testUseHistogramAllSpecified() { 101 | Properties props = new Properties(); 102 | props.setProperty("use_hdr_histogram", "true"); 103 | props.setProperty("hdr_histogram_accuracy", "5"); 104 | props.setProperty("hdr_histogram_max_latency", "100"); 105 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 106 | 107 | LatencyHistogram histogram = factory.create(8, props); 108 | 109 | assertTrue(histogram instanceof HdrLatencyHistogram); 110 | } 111 | 112 | @Test(expected = IllegalArgumentException.class) 113 | public void testUseHistogramAccuracyTooHigh() { 114 | Properties props = new Properties(); 115 | props.setProperty("use_hdr_histogram", "true"); 116 | props.setProperty("hdr_histogram_accuracy", "100"); 117 | props.setProperty("hdr_histogram_max_latency", "100"); 118 | LatencyHistogramFactory factory = new LatencyHistogramFactory(logger); 119 | 120 | factory.create(8, props); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/TestStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import junit.framework.AssertionFailedError; 19 | import junit.framework.TestCase; 20 | 21 | import org.junit.Test; 22 | 23 | import com.facebook.LinkBench.stats.LatencyStats; 24 | 25 | public class TestStats extends TestCase { 26 | 27 | @Test 28 | public void testBucketing() { 29 | // 0 microseconds until 100 seconds 30 | for (long us = 0; us < 100 * 1000 * 1000; us += 100 ) { 31 | int bucket = LatencyStats.latencyToBucket(us); 32 | try { 33 | assertTrue(bucket >= 0); 34 | assertTrue(bucket < LatencyStats.NUM_BUCKETS); 35 | long range[] = LatencyStats.bucketBound(bucket); 36 | assertTrue(us >= range[0]); 37 | assertTrue(us < range[1]); 38 | } catch (AssertionFailedError e) { 39 | System.err.println("Failed for " + us + "us, bucket=" + bucket); 40 | throw e; 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/TimerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Random; 19 | 20 | import org.junit.Test; 21 | 22 | import junit.framework.TestCase; 23 | 24 | public class TimerTest extends TestCase { 25 | 26 | @Test 27 | public void testTimer1() { 28 | for (int i = 0; i < 100; i++) { 29 | long wakeTime = System.nanoTime() + i * 1000 * 500; 30 | Timer.waitUntil(wakeTime); 31 | long now = System.nanoTime(); 32 | assertTrue(now >= wakeTime); 33 | 34 | // Check that the precision isn't so awful that it would 35 | // indicate a definite bug (e.g. 100ms) 36 | assertTrue(now <= wakeTime + 1e7); 37 | } 38 | } 39 | 40 | /** 41 | * Test that we can use the timer to wait for short intervals 42 | * without getting far behind 43 | */ 44 | @Test 45 | public void testTimer2() { 46 | // Repeatedly wait for 10us 47 | 48 | long waits = 100 * 100; // 100ms total 49 | long time = System.nanoTime(); 50 | long startTime = time; 51 | for (int i = 0; i < waits; i++) { 52 | time += 1e4; // 10 us 53 | Timer.waitUntil(time); 54 | } 55 | long endTime = System.nanoTime(); 56 | System.err.println("took " + ((endTime - startTime) / 1000000) + "ms"); 57 | assertTrue(endTime - startTime >= 1e8); 58 | assertTrue(endTime - startTime < 1.02e8); // no longer than 102ms 59 | } 60 | 61 | @Test 62 | public void testExponentialArrivals() { 63 | long randSeed = System.currentTimeMillis(); 64 | System.err.println("Random seed: " + randSeed); 65 | Random rng = new Random(randSeed); 66 | 67 | int trials = 40000; 68 | int arrivalRate_s = 200000; 69 | double arrivalRate_ns = arrivalRate_s / (double)1e9; 70 | 71 | // Check that the exponential distribution is creating correct arrival rate. 72 | long startTime = System.nanoTime(); 73 | long time = startTime; 74 | for (int i = 0; i < trials; i++) { 75 | time = Timer.waitExpInterval(rng, time, arrivalRate_ns); 76 | } 77 | long endTime = System.nanoTime(); 78 | 79 | double actualArrivalRate_ns = trials / (double) (endTime - startTime); 80 | System.err.println("actual arrival rate: " 81 | + actualArrivalRate_ns * 1e9 + " /s " + 82 | " expected " + arrivalRate_s + "/s"); 83 | 84 | assertTrue(actualArrivalRate_ns >= arrivalRate_ns * 0.9); 85 | assertTrue(actualArrivalRate_ns <= arrivalRate_ns * 1.1); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/UniformDistTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Random; 19 | 20 | import org.junit.Test; 21 | 22 | import com.facebook.LinkBench.distributions.ProbabilityDistribution; 23 | import com.facebook.LinkBench.distributions.UniformDistribution; 24 | 25 | public class UniformDistTest extends DistributionTestBase { 26 | 27 | @Override 28 | public ProbabilityDistribution getDist() { 29 | return new UniformDistribution(); 30 | } 31 | 32 | @Test 33 | public void testInRange() { 34 | // Check 2^31 < n < 2^32 and n > 2^32 35 | long maxes[] = {(long)Math.pow(2, 31.5), (long)Math.pow(2, 34.23)}; 36 | int trials = 10000; 37 | Random rng = new Random(); 38 | 39 | for (long max: maxes) { 40 | UniformDistribution dist = new UniformDistribution(); 41 | dist.init(0, max, 1); 42 | for (int trial = 0; trial < trials; trial++) { 43 | long i = dist.choose(rng); 44 | assertTrue(i >= 0); 45 | assertTrue(i < max); 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/ZipfDistTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012, Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.facebook.LinkBench; 17 | 18 | import java.util.Properties; 19 | 20 | import com.facebook.LinkBench.distributions.ProbabilityDistribution; 21 | import com.facebook.LinkBench.distributions.ZipfDistribution; 22 | 23 | public class ZipfDistTest extends DistributionTestBase { 24 | 25 | @Override 26 | protected int cdfChecks() { 27 | // Don't do many checks, since its expensive 28 | return 5; 29 | } 30 | 31 | @Override 32 | public ProbabilityDistribution getDist() { 33 | return new ZipfDistribution(); 34 | } 35 | 36 | @Override 37 | public Properties getDistParams() { 38 | Properties props = new Properties(); 39 | props.setProperty("shape", "0.9"); 40 | return props; 41 | } 42 | 43 | 44 | @Override 45 | protected Bucketer getBucketer() { 46 | return new ZipfBucketer(); 47 | } 48 | 49 | @Override 50 | protected double tolerance() { 51 | /* 52 | * Method for choosing IDs isn't 100% precise 53 | * but something is more seriously wrong if it is more than 1% off 54 | */ 55 | return 0.01; 56 | } 57 | 58 | /** 59 | * Check distribution more closely at low values 60 | */ 61 | static class ZipfBucketer implements Bucketer { 62 | private static final long bucketBounds[] = 63 | {0, 1, 2, 3, 4, 10, 20, 30, 100, 1000, 10000, 100000, 1000000, 64 | 10000000, 100000000}; 65 | 66 | public int getBucketCount() { 67 | return bucketBounds.length + 1; 68 | } 69 | 70 | public int chooseBucket(long i, long n) { 71 | for (int j = 0; j < bucketBounds.length; j++) { 72 | if (i <= bucketBounds[j]) { 73 | return j; 74 | } 75 | } 76 | return bucketBounds.length; 77 | } 78 | 79 | public long bucketMax(int bucket, long n) { 80 | if (bucket >= bucketBounds.length) return n; 81 | else return bucketBounds[bucket]; 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/testtypes/MongoDbTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.testtypes; 2 | 3 | public interface MongoDbTest extends ProviderTest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/testtypes/MySqlTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.testtypes; 2 | 3 | public interface MySqlTest extends ProviderTest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/testtypes/PgsqlTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.testtypes; 2 | 3 | public interface PgsqlTest extends ProviderTest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/testtypes/ProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.testtypes; 2 | 3 | /** 4 | * Marker interface for tests for specific data store providers, particualrly 5 | * those that require an external data store to be setup. 6 | */ 7 | public interface ProviderTest { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/testtypes/RocksDbTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.testtypes; 2 | 3 | public interface RocksDbTest extends ProviderTest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/test/java/com/facebook/LinkBench/testtypes/SlowTest.java: -------------------------------------------------------------------------------- 1 | package com.facebook.LinkBench.testtypes; 2 | 3 | /** 4 | * Marker interface for slow unit tests that should not be run as part of default 5 | * unit tests 6 | */ 7 | public interface SlowTest { 8 | 9 | } 10 | --------------------------------------------------------------------------------