├── README.md ├── build.gradle ├── downloadDataset.sh ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── run.sh ├── settings.gradle └── src └── main ├── java └── com │ └── jerolba │ └── benchmark │ ├── CopyInsert.java │ ├── JdbcBatchInsert.java │ ├── JdbcBatchInsertStatementRewrite.java │ ├── JdbcSimpleInsert.java │ ├── JdbcSimpleInsertInBlocks.java │ ├── JpaBatchBatchSize.java │ ├── JpaBatchInsert.java │ ├── JpaBatchInsertStatementRewrite.java │ ├── JpaSimpleInsert.java │ ├── JpaSimpleInsertInBlocks.java │ ├── JpaSimpleInsertInBlocksFlush.java │ ├── LoadDataInsert.java │ └── shared │ ├── BenchmarkMeter.java │ ├── CityBikeParser.java │ ├── CityBikeReader.java │ ├── ConnectionProvider.java │ ├── CsvSplit.java │ ├── DataSourceFactory.java │ ├── DatabaseProperties.java │ ├── EntityManagerFactoryFactory.java │ ├── MutableInteger.java │ ├── PersistenceUnitInfoImpl.java │ ├── StreamCounter.java │ ├── StringBuilderReader.java │ ├── TableHelper.java │ ├── Trip.java │ ├── TripEntity.java │ ├── TripEntityInsert.java │ └── TripEntityJpa.java └── resources ├── logback.xml ├── mysql.properties ├── mysql2.properties ├── postgres.properties └── postgres2.properties /README.md: -------------------------------------------------------------------------------- 1 | # Persistence post 2 | 3 | http://www.jerolba.com/persistiendo-rapido-en-base-de-datos/ 4 | 5 | These instructions are written for *nix system. 6 | 7 | ## Prerequisites 8 | You must have Docker installed (for the databases). 9 | 10 | 11 | ## How to run the benchmark yourself 12 | 13 | 1. Download the dataset 14 | * `chmod +x downloadDataset.sh` 15 | * `./downloadDataset.sh` 16 | 17 | 2. Generate the JAR and copy it to the root folder of the repo 18 | * `gradle build` 19 | * `cp ./build/libs/persistence-post.jar .` 20 | 21 | 3. Depending on the database you want to try: 22 | 23 | 3.1. MySQL 24 | * Install the database client (in case you don't have it, just for checking the connection) 25 | * `sudo apt-get install mysql-client` 26 | * Run a MySQL from a Docker 27 | * `docker run --name jerolba-mysql -e MYSQL_ROOT_PASSWORD=my-secret-pw -e MYSQL_DATABASE=testdb -e MYSQL_USER=test -e MYSQL_PASSWORD=test --publish 127.0.0.1:3306:3306 -d mysql:5.6.35` 28 | * Check that it's up and running (password 'my-secret-pw'): `mysql -u root -p` 29 | 30 | 3.2. PostgreSQL 31 | * Install the database client (in case you don't have it, just for checking the connection) 32 | * `sudo apt-get install postgresql-client` 33 | * Run a PostgreSQL from a Docker 34 | * `docker run --name jerolba-postgres -e POSTGRES_USER=test -e POSTGRES_PASSWORD=test -e POSTGRES_DB=testdb --publish 127.0.0.1:5432:5432 -d postgres:9.6.8` 35 | * Check that it's up and running (password 'mysecretpassword'): `psql -h 127.0.0.1 -p 5432 -U postgres` 36 | 37 | 6. Run the benchmark 38 | * `chmod +x run.sh` 39 | * For MySQL: `./run.sh -m 127.0.0.1` 40 | * For PostgreSQL: `./run.sh -p 127.0.0.1` 41 | 42 | 7. Check that the benchmark is running 43 | 44 | You should see some output here: `tail -f log/JdbcSimpleInsert__1000_1.log`, something similar to: 45 | ``` 46 | ➜ persistence-post git:(master) ✗ tail -f log/JdbcSimpleInsert_postgres_1000_1.log 47 | 13:39:26.669 [main] INFO c.j.b.shared.ConnectionProvider - Connecting with {urlConnection=jdbc:postgresql://127.0.0.1:5432/testdb, password=mysecretpassword, driver=org.postgresql.Driver, user=postgres} 48 | 13:39:54.086 [main] INFO c.j.benchmark.shared.StreamCounter - Processed 10000 items in 27298 ms 49 | 13:40:21.100 [main] INFO c.j.benchmark.shared.StreamCounter - Processed 20000 items in 54311 ms 50 | ``` 51 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'com.github.johnrengelman.shadow' version '2.0.1' 3 | id 'application' 4 | } 5 | 6 | repositories { 7 | mavenLocal() 8 | jcenter() 9 | } 10 | 11 | sourceCompatibility = 1.8 12 | targetCompatibility = 1.8 13 | 14 | dependencies { 15 | compile group: 'mysql', name: 'mysql-connector-java', version: '5.1.43' 16 | compile group: 'org.postgresql', name: 'postgresql', version: '42.1.4' 17 | compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' 18 | compile group: 'org.hibernate', name: 'hibernate-core', version: '5.2.10.Final' 19 | compile group: 'com.zaxxer', name: 'HikariCP', version: '2.7.8' 20 | } 21 | 22 | mainClassName = 'com.jerolba.benchmark.JpaSimpleInsert' 23 | 24 | shadowJar { 25 | version = null 26 | classifier = null 27 | } 28 | -------------------------------------------------------------------------------- /downloadDataset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | wget https://s3.amazonaws.com/tripdata/201601-citibike-tripdata.zip -P /tmp -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jerolba/persistence-post/44a80903957ab1f204d10e97dee5ae8af3c3b43f/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.5-bin.zip 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | IP_POSTGRES="localhost" 5 | USER_POSTGRES="test" 6 | PASS_POSTGRES="test" 7 | IP_MYSQL="localhost" 8 | USER_MYSQL="test" 9 | PASS_MYSQL="test" 10 | 11 | TIMES=10 12 | 13 | dbs=() 14 | 15 | while getopts ":p:m:t:" opts 16 | do 17 | case $opts in 18 | t) 19 | TIMES=$OPTARG 20 | ;; 21 | p) 22 | echo "Postgres IP: $OPTARG" 23 | IP_POSTGRES=$OPTARG 24 | echo "driver=org.postgresql.Driver" > postgres.properties 25 | echo "user=$USER_POSTGRES" >> postgres.properties 26 | echo "password=$PASS_POSTGRES" >> postgres.properties 27 | echo "urlConnection=jdbc:postgresql://$IP_POSTGRES:5432/testdb" >> postgres.properties 28 | 29 | echo "driver=org.postgresql.Driver" > postgres2.properties 30 | echo "user=$USER_POSTGRES" >> postgres2.properties 31 | echo "password=$PASS_POSTGRES" >> postgres2.properties 32 | echo "urlConnection=jdbc:postgresql://$IP_POSTGRES:5432/testdb?reWriteBatchedInserts=true" >> postgres2.properties 33 | 34 | echo "$IP_POSTGRES:5432:testdb:$USER_POSTGRES:$PASS_POSTGRES" > ~/.pgpass 35 | chmod 600 ~/.pgpass 36 | dbs+=("postgres") 37 | ;; 38 | m) 39 | echo "MySQL IP: $OPTARG" 40 | IP_MYSQL=$OPTARG 41 | echo "driver=com.mysql.jdbc.Driver" > mysql.properties 42 | echo "user=$USER_MYSQL" >> mysql.properties 43 | echo "password=$PASS_MYSQL" >> mysql.properties 44 | echo "urlConnection=jdbc:mysql://$IP_MYSQL/testdb?useSSL=false" >> mysql.properties 45 | 46 | echo "driver=com.mysql.jdbc.Driver" > mysql2.properties 47 | echo "user=$USER_MYSQL" >> mysql2.properties 48 | echo "password=$PASS_MYSQL" >> mysql2.properties 49 | echo "urlConnection=jdbc:mysql://$IP_MYSQL/testdb?useSSL=false&rewriteBatchedStatements=true" >> mysql2.properties 50 | dbs+=("mysql") 51 | ;; 52 | esac 53 | done 54 | 55 | echo "Number of times: $TIMES" 56 | 57 | if [ ! -x "log" ] 58 | then 59 | mkdir log 60 | fi 61 | 62 | #classes=("JpaSimpleInsert" "JpaSimpleInsertInBlocks" "JpaSimpleInsertInBlocksFlush" "JpaBatchIdentity" "JpaBatchInsert" "JpaBatchInsertStatementRewrite") 63 | classes=("JdbcSimpleInsert" "JdbcSimpleInsertInBlocks" "JdbcBatchInsert" "JdbcBatchInsertStatementRewrite") 64 | sizes=(1000) 65 | for db in "${dbs[@]}" 66 | do 67 | for i in `seq 1 $TIMES`; 68 | do 69 | for class in "${classes[@]}" 70 | do 71 | echo "Running $class for $db" 72 | for size in "${sizes[@]}" 73 | do 74 | echo Iteration $i for size $size 75 | java -cp ./persistence-post.jar com.jerolba.benchmark.$class $db.properties $size > log/$class\_$db\_$size\_$i.log 76 | sleep 10 77 | done 78 | done 79 | done 80 | done 81 | 82 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'persistence-post' 2 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/CopyInsert.java: -------------------------------------------------------------------------------- 1 | package com.jerolba.benchmark; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.sql.Connection; 7 | import java.sql.SQLException; 8 | import java.text.SimpleDateFormat; 9 | import java.util.Iterator; 10 | import java.util.function.Supplier; 11 | 12 | import org.postgresql.copy.CopyManager; 13 | import org.postgresql.jdbc.PgConnection; 14 | 15 | import com.jerolba.benchmark.shared.BenchmarkMeter; 16 | import com.jerolba.benchmark.shared.CityBikeParser; 17 | import com.jerolba.benchmark.shared.CityBikeReader; 18 | import com.jerolba.benchmark.shared.ConnectionProvider; 19 | import com.jerolba.benchmark.shared.TableHelper; 20 | import com.jerolba.benchmark.shared.TripEntity; 21 | 22 | public class CopyInsert { 23 | 24 | private final static String COPY = "COPY bike_trip (tripduration, starttime, stoptime, " 25 | + "start_station_id, start_station_name, start_station_latitude, start_station_longitude, " 26 | + "end_station_id, end_station_name, end_station_latitude, end_station_longitude, bike_id, " 27 | + "user_type, birth_year, gender) FROM STDIN WITH (FORMAT TEXT, ENCODING 'UTF-8', DELIMITER '\t', HEADER false)"; 28 | 29 | private static final SimpleDateFormat sdfDateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 30 | 31 | public static void main(String[] args) throws IOException, SQLException { 32 | String properties = args[0]; 33 | int batchSize = Integer.parseInt(args[1]); 34 | 35 | Supplier connectionSuplier = new ConnectionProvider(properties); 36 | try (Connection connection = connectionSuplier.get()) { 37 | TableHelper.createTable(connection); 38 | 39 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 40 | CityBikeReader reader = new CityBikeReader<>("/tmp", str -> parser.parse(str)); 41 | 42 | PgConnection unwrapped = connection.unwrap(PgConnection.class); 43 | CopyManager copyManager = unwrapped.getCopyAPI(); 44 | 45 | BenchmarkMeter meter = new BenchmarkMeter(JpaSimpleInsert.class, properties, batchSize); 46 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 47 | try { 48 | int cont = 0; 49 | StringBuilder sb = new StringBuilder(); 50 | Iterator iterator = trips.iterator(); 51 | while (iterator.hasNext()) { 52 | TripEntity trip = iterator.next(); 53 | sb.append(trip.getTripduration()).append("\t"); 54 | sb.append(sdfDateTime.format(trip.getStarttime())).append("\t"); 55 | sb.append(sdfDateTime.format(trip.getStoptime())).append("\t"); 56 | sb.append(trip.getStartStationId()).append("\t"); 57 | sb.append(trip.getStartStationName()).append("\t"); 58 | sb.append(trip.getStartStationLatitude()).append("\t"); 59 | sb.append(trip.getStartStationLongitude()).append("\t"); 60 | sb.append(trip.getEndStationId()).append("\t"); 61 | sb.append(trip.getEndStationName()).append("\t"); 62 | sb.append(trip.getEndStationLatitude()).append("\t"); 63 | sb.append(trip.getEndStationLongitude()).append("\t"); 64 | sb.append(trip.getBikeId()).append("\t"); 65 | sb.append(trip.getUserType()).append("\t"); 66 | sb.append(nullify(trip.getBirthYear())).append("\t"); 67 | sb.append(trip.getGender()); 68 | sb.append("\n"); 69 | cont++; 70 | if (cont % batchSize == 0) { 71 | InputStream is = new ByteArrayInputStream(sb.toString().getBytes()); 72 | copyManager.copyIn(COPY, is); 73 | sb.setLength(0); 74 | } 75 | } 76 | } catch (Exception e) { 77 | e.printStackTrace(); 78 | }}) 79 | ); 80 | 81 | } 82 | } 83 | 84 | private static String nullify(Integer value) { 85 | if (value==null) { 86 | return "\\N"; 87 | } 88 | return value.toString(); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JdbcBatchInsert.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.sql.PreparedStatement; 21 | import java.sql.SQLException; 22 | import java.util.Iterator; 23 | import java.util.function.Supplier; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.ConnectionProvider; 29 | import com.jerolba.benchmark.shared.TableHelper; 30 | import com.jerolba.benchmark.shared.TripEntity; 31 | import com.jerolba.benchmark.shared.TripEntityInsert; 32 | 33 | public class JdbcBatchInsert { 34 | 35 | public static void main(String[] args) throws IOException, SQLException { 36 | String properties = args[0]; 37 | int batchSize = Integer.parseInt(args[1]); 38 | Supplier connectionSuplier = new ConnectionProvider(properties); 39 | try (Connection connection = connectionSuplier.get()) { 40 | TableHelper.createTable(connection); 41 | connection.setAutoCommit(false); 42 | 43 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 44 | CityBikeReader reader = new CityBikeReader<>("/tmp", 500_000, str -> parser.parse(str)); 45 | TripEntityInsert tripInsert = new TripEntityInsert(); 46 | 47 | BenchmarkMeter meter = new BenchmarkMeter(JdbcBatchInsert.class, properties, batchSize); 48 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 49 | try (PreparedStatement pstmt = connection.prepareStatement(TripEntityInsert.INSERT)) { 50 | int cont = 0; 51 | Iterator iterator = trips.iterator(); 52 | while (iterator.hasNext()) { 53 | TripEntity entity = iterator.next(); 54 | tripInsert.setParameters(pstmt, entity); 55 | pstmt.addBatch(); 56 | cont++; 57 | if (cont % batchSize == 0) { 58 | pstmt.executeBatch(); 59 | connection.commit(); 60 | } 61 | } 62 | connection.commit(); 63 | } catch (SQLException e) { 64 | e.printStackTrace(); 65 | } 66 | }) 67 | ); 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JdbcBatchInsertStatementRewrite.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.sql.PreparedStatement; 21 | import java.sql.SQLException; 22 | import java.util.Iterator; 23 | import java.util.function.Supplier; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.ConnectionProvider; 29 | import com.jerolba.benchmark.shared.TableHelper; 30 | import com.jerolba.benchmark.shared.TripEntity; 31 | import com.jerolba.benchmark.shared.TripEntityInsert; 32 | 33 | public class JdbcBatchInsertStatementRewrite { 34 | 35 | public static void main(String[] args) throws IOException, SQLException { 36 | String properties = args[0].replace(".properties", "2.properties"); //For script support 37 | int batchSize = Integer.parseInt(args[1]); 38 | Supplier connectionSuplier = new ConnectionProvider(properties); 39 | try (Connection connection = connectionSuplier.get()) { 40 | TableHelper.createTable(connection); 41 | connection.setAutoCommit(true); 42 | 43 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 44 | CityBikeReader reader = new CityBikeReader<>("/tmp", 1_000_000, str -> parser.parse(str)); 45 | TripEntityInsert jdbcInsert = new TripEntityInsert(); 46 | 47 | BenchmarkMeter meter = new BenchmarkMeter(JdbcBatchInsertStatementRewrite.class, properties, batchSize); 48 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 49 | try (PreparedStatement pstmt = connection.prepareStatement(TripEntityInsert.INSERT)) { 50 | int cont = 0; 51 | Iterator iterator = trips.iterator(); 52 | while (iterator.hasNext()) { 53 | TripEntity entity = iterator.next(); 54 | jdbcInsert.setParameters(pstmt, entity); 55 | pstmt.addBatch(); 56 | cont++; 57 | if (cont % batchSize == 0) { 58 | pstmt.executeBatch(); 59 | } 60 | } 61 | } catch (SQLException e) { 62 | e.printStackTrace(); 63 | } 64 | }) 65 | ); 66 | } 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JdbcSimpleInsert.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.sql.PreparedStatement; 21 | import java.sql.SQLException; 22 | import java.util.Iterator; 23 | import java.util.function.Supplier; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.ConnectionProvider; 29 | import com.jerolba.benchmark.shared.TableHelper; 30 | import com.jerolba.benchmark.shared.TripEntity; 31 | import com.jerolba.benchmark.shared.TripEntityInsert; 32 | 33 | public class JdbcSimpleInsert { 34 | 35 | public static void main(String[] args) throws IOException, SQLException { 36 | String properties = args[0]; 37 | Supplier connectionSuplier = new ConnectionProvider(properties); 38 | try (Connection connection = connectionSuplier.get()) { 39 | TableHelper.createTable(connection); 40 | connection.setAutoCommit(true); 41 | 42 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 43 | CityBikeReader reader = new CityBikeReader<>("/tmp", 100_000, str -> parser.parse(str)); 44 | TripEntityInsert tripInsert = new TripEntityInsert(); 45 | 46 | BenchmarkMeter meter = new BenchmarkMeter(JdbcSimpleInsert.class, properties, 1); 47 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 48 | try (PreparedStatement pstmt = connection.prepareStatement(TripEntityInsert.INSERT)) { 49 | Iterator iterator = trips.iterator(); 50 | while (iterator.hasNext()) { 51 | tripInsert.setParameters(pstmt, iterator.next()); 52 | pstmt.executeUpdate(); 53 | } 54 | } catch (SQLException e) { 55 | e.printStackTrace(); 56 | } 57 | }) 58 | ); 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JdbcSimpleInsertInBlocks.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.sql.PreparedStatement; 21 | import java.sql.SQLException; 22 | import java.util.Iterator; 23 | import java.util.function.Supplier; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.ConnectionProvider; 29 | import com.jerolba.benchmark.shared.TableHelper; 30 | import com.jerolba.benchmark.shared.TripEntity; 31 | import com.jerolba.benchmark.shared.TripEntityInsert; 32 | 33 | public class JdbcSimpleInsertInBlocks { 34 | 35 | public static void main(String[] args) throws IOException, SQLException { 36 | String properties = args[0]; 37 | int batchSize = Integer.parseInt(args[1]); 38 | Supplier connectionSuplier = new ConnectionProvider(properties); 39 | try (Connection connection = connectionSuplier.get()) { 40 | TableHelper.createTable(connection); 41 | connection.setAutoCommit(false); 42 | 43 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 44 | CityBikeReader reader = new CityBikeReader<>("/tmp", 100_000, str -> parser.parse(str)); 45 | TripEntityInsert tripInsert = new TripEntityInsert(); 46 | 47 | BenchmarkMeter meter = new BenchmarkMeter(JdbcSimpleInsertInBlocks.class, properties, batchSize); 48 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 49 | try (PreparedStatement pstmt = connection.prepareStatement(TripEntityInsert.INSERT)) { 50 | int cont = 0; 51 | Iterator iterator = trips.iterator(); 52 | while (iterator.hasNext()) { 53 | tripInsert.setParameters(pstmt, iterator.next()); 54 | pstmt.executeUpdate(); 55 | cont++; 56 | if (cont % batchSize == 0) { 57 | connection.commit(); 58 | } 59 | } 60 | connection.commit(); 61 | } catch (SQLException e) { 62 | e.printStackTrace(); 63 | } 64 | }) 65 | ); 66 | } 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JpaBatchBatchSize.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.Iterator; 21 | import java.util.Properties; 22 | 23 | import javax.persistence.EntityManager; 24 | import javax.persistence.EntityTransaction; 25 | 26 | import com.jerolba.benchmark.shared.BenchmarkMeter; 27 | import com.jerolba.benchmark.shared.CityBikeParser; 28 | import com.jerolba.benchmark.shared.CityBikeReader; 29 | import com.jerolba.benchmark.shared.DataSourceFactory; 30 | import com.jerolba.benchmark.shared.EntityManagerFactoryFactory; 31 | import com.jerolba.benchmark.shared.TableHelper; 32 | import com.jerolba.benchmark.shared.TripEntity; 33 | 34 | /* 35 | * Persist all information in PostgreSQL or MySQL using JPA batched inserts using autoincremental Id. 36 | */ 37 | public class JpaBatchBatchSize { 38 | 39 | public static void main(String[] args) throws IOException, SQLException { 40 | String properties = args[0]; 41 | int batchSize = Integer.parseInt(args[1]); 42 | DataSourceFactory dsFactory = new DataSourceFactory(properties); 43 | TableHelper.createTable(dsFactory); 44 | 45 | EntityManagerFactoryFactory factory = new EntityManagerFactoryFactory(dsFactory, TripEntity.class) { 46 | @Override 47 | public Properties properties() { 48 | Properties properties = super.properties(); 49 | properties.put("hibernate.jdbc.batch_size", batchSize); 50 | return properties; 51 | } 52 | }; 53 | EntityManager entityManager = factory.newEntityManagerFactory().createEntityManager(); 54 | 55 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 56 | CityBikeReader reader = new CityBikeReader<>("/tmp", 300_000, str -> parser.parse(str)); 57 | 58 | BenchmarkMeter meter = new BenchmarkMeter(JpaBatchBatchSize.class, properties, batchSize); 59 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 60 | EntityTransaction tx = entityManager.getTransaction(); 61 | Iterator iterator = trips.iterator(); 62 | tx.begin(); 63 | int cont = 0; 64 | while (iterator.hasNext()) { 65 | entityManager.persist(iterator.next()); 66 | cont++; 67 | if (cont % batchSize == 0) { 68 | entityManager.flush(); 69 | entityManager.clear(); 70 | tx.commit(); 71 | tx.begin(); 72 | } 73 | } 74 | tx.commit(); 75 | entityManager.close(); 76 | })); 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JpaBatchInsert.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.Iterator; 21 | import java.util.Properties; 22 | 23 | import javax.persistence.EntityManager; 24 | import javax.persistence.EntityTransaction; 25 | 26 | import com.jerolba.benchmark.shared.BenchmarkMeter; 27 | import com.jerolba.benchmark.shared.CityBikeParser; 28 | import com.jerolba.benchmark.shared.CityBikeReader; 29 | import com.jerolba.benchmark.shared.DataSourceFactory; 30 | import com.jerolba.benchmark.shared.EntityManagerFactoryFactory; 31 | import com.jerolba.benchmark.shared.MutableInteger; 32 | import com.jerolba.benchmark.shared.TableHelper; 33 | import com.jerolba.benchmark.shared.TripEntityJpa; 34 | 35 | /* 36 | * Persist all information in PostgreSQL or MySQL using JPA batched inserts. 37 | */ 38 | public class JpaBatchInsert { 39 | 40 | public static void main(String[] args) throws IOException, SQLException { 41 | String properties = args[0]; 42 | int batchSize = Integer.parseInt(args[1]); 43 | DataSourceFactory dsFactory = new DataSourceFactory(properties); 44 | TableHelper.createTable(dsFactory); 45 | 46 | EntityManagerFactoryFactory factory = new EntityManagerFactoryFactory(dsFactory, TripEntityJpa.class) { 47 | @Override 48 | public Properties properties() { 49 | Properties properties = super.properties(); 50 | properties.put("hibernate.jdbc.batch_size", batchSize); 51 | return properties; 52 | } 53 | }; 54 | EntityManager entityManager = factory.newEntityManagerFactory().createEntityManager(); 55 | 56 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntityJpa()); 57 | CityBikeReader reader = new CityBikeReader<>("/tmp", 300_000, str -> parser.parse(str)); 58 | 59 | MutableInteger idSeq = new MutableInteger(); 60 | BenchmarkMeter meter = new BenchmarkMeter(JpaBatchInsert.class, properties, batchSize); 61 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 62 | EntityTransaction tx = entityManager.getTransaction(); 63 | Iterator iterator = trips.iterator(); 64 | tx.begin(); 65 | int cont = 0; 66 | while (iterator.hasNext()) { 67 | TripEntityJpa trip = iterator.next(); 68 | trip.setId(idSeq.incAndGet()); 69 | entityManager.persist(trip); 70 | cont++; 71 | if (cont % batchSize == 0) { 72 | entityManager.flush(); 73 | entityManager.clear(); 74 | tx.commit(); 75 | tx.begin(); 76 | } 77 | } 78 | tx.commit(); 79 | entityManager.close(); 80 | })); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JpaBatchInsertStatementRewrite.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.Iterator; 21 | import java.util.Properties; 22 | 23 | import javax.persistence.EntityManager; 24 | import javax.persistence.EntityTransaction; 25 | 26 | import com.jerolba.benchmark.shared.BenchmarkMeter; 27 | import com.jerolba.benchmark.shared.CityBikeParser; 28 | import com.jerolba.benchmark.shared.CityBikeReader; 29 | import com.jerolba.benchmark.shared.DataSourceFactory; 30 | import com.jerolba.benchmark.shared.EntityManagerFactoryFactory; 31 | import com.jerolba.benchmark.shared.MutableInteger; 32 | import com.jerolba.benchmark.shared.TableHelper; 33 | import com.jerolba.benchmark.shared.TripEntityJpa; 34 | 35 | /* 36 | * Persist all information in PostgreSQL or MySQL using JPA batched inserts with Statement Rewrite 37 | */ 38 | public class JpaBatchInsertStatementRewrite { 39 | 40 | public static void main(String[] args) throws IOException, SQLException { 41 | String properties = args[0].replace(".properties", "2.properties"); //For script support 42 | int batchSize = Integer.parseInt(args[1]); 43 | DataSourceFactory dsFactory = new DataSourceFactory(properties); 44 | TableHelper.createTable(dsFactory); 45 | 46 | EntityManagerFactoryFactory factory = new EntityManagerFactoryFactory(dsFactory, TripEntityJpa.class) { 47 | @Override 48 | public Properties properties() { 49 | Properties properties = super.properties(); 50 | properties.put("hibernate.jdbc.batch_size", batchSize); 51 | return properties; 52 | } 53 | }; 54 | EntityManager entityManager = factory.newEntityManagerFactory().createEntityManager(); 55 | 56 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntityJpa()); 57 | CityBikeReader reader = new CityBikeReader<>("/tmp", 300_000, str -> parser.parse(str)); 58 | 59 | MutableInteger idSeq = new MutableInteger(); 60 | BenchmarkMeter meter = new BenchmarkMeter(JpaBatchInsertStatementRewrite.class, properties, batchSize); 61 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 62 | EntityTransaction tx = entityManager.getTransaction(); 63 | Iterator iterator = trips.iterator(); 64 | tx.begin(); 65 | int cont = 0; 66 | while (iterator.hasNext()) { 67 | TripEntityJpa trip = iterator.next(); 68 | trip.setId(idSeq.incAndGet()); 69 | entityManager.persist(trip); 70 | cont++; 71 | if (cont % batchSize == 0) { 72 | entityManager.flush(); 73 | entityManager.clear(); 74 | tx.commit(); 75 | tx.begin(); 76 | } 77 | } 78 | tx.commit(); 79 | entityManager.close(); 80 | })); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JpaSimpleInsert.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.Iterator; 21 | 22 | import javax.persistence.EntityManager; 23 | import javax.persistence.EntityTransaction; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.DataSourceFactory; 29 | import com.jerolba.benchmark.shared.EntityManagerFactoryFactory; 30 | import com.jerolba.benchmark.shared.TableHelper; 31 | import com.jerolba.benchmark.shared.TripEntity; 32 | 33 | /* 34 | * Persist all information in PostgreSQL or MySQL using simple JPA inserts. 35 | */ 36 | public class JpaSimpleInsert { 37 | 38 | public static void main(String[] args) throws IOException, SQLException { 39 | String properties = args[0]; 40 | DataSourceFactory dsFactory = new DataSourceFactory(properties); 41 | TableHelper.createTable(dsFactory); 42 | 43 | EntityManagerFactoryFactory factory = new EntityManagerFactoryFactory(dsFactory, TripEntity.class); 44 | EntityManager entityManager = factory.newEntityManagerFactory().createEntityManager(); 45 | 46 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 47 | CityBikeReader reader = new CityBikeReader<>("/tmp", 10_000, str -> parser.parse(str)); 48 | 49 | BenchmarkMeter meter = new BenchmarkMeter(JpaSimpleInsert.class, properties, 1); 50 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 51 | EntityTransaction tx = entityManager.getTransaction(); 52 | Iterator iterator = trips.iterator(); 53 | while (iterator.hasNext()) { 54 | tx.begin(); 55 | entityManager.persist(iterator.next()); 56 | tx.commit(); 57 | } 58 | entityManager.close(); 59 | }) 60 | ); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JpaSimpleInsertInBlocks.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.Iterator; 21 | 22 | import javax.persistence.EntityManager; 23 | import javax.persistence.EntityTransaction; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.DataSourceFactory; 29 | import com.jerolba.benchmark.shared.EntityManagerFactoryFactory; 30 | import com.jerolba.benchmark.shared.TableHelper; 31 | import com.jerolba.benchmark.shared.TripEntity; 32 | 33 | /* 34 | * Persist all information in PostgreSQL or MySQL using simple JPA inserts in transactions of N elements. 35 | */ 36 | public class JpaSimpleInsertInBlocks { 37 | 38 | public static void main(String[] args) throws IOException, SQLException { 39 | String properties = args[0]; 40 | int batchSize = Integer.parseInt(args[1]); 41 | DataSourceFactory dsFactory = new DataSourceFactory(properties); 42 | TableHelper.createTable(dsFactory); 43 | 44 | EntityManagerFactoryFactory factory = new EntityManagerFactoryFactory(dsFactory, TripEntity.class); 45 | EntityManager entityManager = factory.newEntityManagerFactory().createEntityManager(); 46 | 47 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 48 | CityBikeReader reader = new CityBikeReader<>("/tmp", 300_000, str -> parser.parse(str)); 49 | 50 | BenchmarkMeter meter = new BenchmarkMeter(JpaSimpleInsertInBlocks.class, properties, batchSize); 51 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 52 | EntityTransaction tx = entityManager.getTransaction(); 53 | Iterator iterator = trips.iterator(); 54 | tx.begin(); 55 | int cont = 0; 56 | while (iterator.hasNext()) { 57 | entityManager.persist(iterator.next()); 58 | cont++; 59 | if (cont % batchSize == 0) { 60 | tx.commit(); 61 | tx.begin(); 62 | } 63 | } 64 | tx.commit(); 65 | entityManager.close(); 66 | }) 67 | ); 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/JpaSimpleInsertInBlocksFlush.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark; 17 | 18 | import java.io.IOException; 19 | import java.sql.SQLException; 20 | import java.util.Iterator; 21 | 22 | import javax.persistence.EntityManager; 23 | import javax.persistence.EntityTransaction; 24 | 25 | import com.jerolba.benchmark.shared.BenchmarkMeter; 26 | import com.jerolba.benchmark.shared.CityBikeParser; 27 | import com.jerolba.benchmark.shared.CityBikeReader; 28 | import com.jerolba.benchmark.shared.DataSourceFactory; 29 | import com.jerolba.benchmark.shared.EntityManagerFactoryFactory; 30 | import com.jerolba.benchmark.shared.TableHelper; 31 | import com.jerolba.benchmark.shared.TripEntity; 32 | 33 | /* 34 | * Persist all information in PostgreSQL or MySQL using simple JPA inserts in transactions of N elements 35 | * flushing the Entity Manager 36 | */ 37 | public class JpaSimpleInsertInBlocksFlush { 38 | 39 | public static void main(String[] args) throws IOException, SQLException { 40 | String properties = args[0]; 41 | int batchSize = Integer.parseInt(args[1]); 42 | DataSourceFactory dsFactory = new DataSourceFactory(properties); 43 | TableHelper.createTable(dsFactory); 44 | 45 | EntityManagerFactoryFactory factory = new EntityManagerFactoryFactory(dsFactory, TripEntity.class); 46 | EntityManager entityManager = factory.newEntityManagerFactory().createEntityManager(); 47 | 48 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 49 | CityBikeReader reader = new CityBikeReader<>("/tmp", 300_000, str -> parser.parse(str)); 50 | 51 | BenchmarkMeter meter = new BenchmarkMeter(JpaSimpleInsertInBlocksFlush.class, properties, batchSize); 52 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 53 | EntityTransaction tx = entityManager.getTransaction(); 54 | Iterator iterator = trips.iterator(); 55 | tx.begin(); 56 | int cont = 0; 57 | while (iterator.hasNext()) { 58 | entityManager.persist(iterator.next()); 59 | cont++; 60 | if (cont % batchSize == 0) { 61 | entityManager.flush(); 62 | entityManager.clear(); 63 | tx.commit(); 64 | tx.begin(); 65 | } 66 | } 67 | tx.commit(); 68 | entityManager.close(); 69 | })); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/LoadDataInsert.java: -------------------------------------------------------------------------------- 1 | package com.jerolba.benchmark; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.sql.Connection; 7 | import java.sql.SQLException; 8 | import java.text.SimpleDateFormat; 9 | import java.util.Iterator; 10 | import java.util.function.Supplier; 11 | 12 | import com.jerolba.benchmark.shared.BenchmarkMeter; 13 | import com.jerolba.benchmark.shared.CityBikeParser; 14 | import com.jerolba.benchmark.shared.CityBikeReader; 15 | import com.jerolba.benchmark.shared.ConnectionProvider; 16 | import com.jerolba.benchmark.shared.TableHelper; 17 | import com.jerolba.benchmark.shared.TripEntity; 18 | import com.mysql.jdbc.Statement; 19 | 20 | public class LoadDataInsert { 21 | 22 | private final static String LOADDATA = "LOAD DATA LOCAL INFILE '' INTO TABLE `bike_trip` CHARACTER SET UTF8 " 23 | + "FIELDS TERMINATED BY '\t' ENCLOSED BY '' ESCAPED BY '\\\\' LINES TERMINATED BY '\n' " 24 | + "STARTING BY '' " 25 | + "(`tripduration`, `starttime`, `stoptime`, `start_station_id`, `start_station_name`, " 26 | + "`start_station_latitude`, `start_station_longitude`, `end_station_id`, `end_station_name`, " 27 | + "`end_station_latitude`, `end_station_longitude`, `bike_id`, `user_type`, `birth_year`, `gender`)"; 28 | 29 | private static final SimpleDateFormat sdfDateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 30 | 31 | public static void main(String[] args) throws IOException, SQLException { 32 | String properties = args[0]; 33 | int batchSize = Integer.parseInt(args[1]); 34 | Supplier connectionSuplier = new ConnectionProvider(properties); 35 | try (Connection connection = connectionSuplier.get()) { 36 | TableHelper.createTable(connection); 37 | 38 | CityBikeParser parser = new CityBikeParser<>(() -> new TripEntity()); 39 | CityBikeReader reader = new CityBikeReader<>("/tmp", str -> parser.parse(str)); 40 | 41 | com.mysql.jdbc.Connection unwrapped = connection.unwrap(com.mysql.jdbc.Connection.class); 42 | unwrapped.setAllowLoadLocalInfile(true); 43 | 44 | BenchmarkMeter meter = new BenchmarkMeter(JpaSimpleInsert.class, properties, batchSize); 45 | meter.meter(() -> reader.forEachCsvInZip(trips -> { 46 | try { 47 | int cont = 0; 48 | StringBuilder sb = new StringBuilder(); 49 | Iterator iterator = trips.iterator(); 50 | while (iterator.hasNext()) { 51 | TripEntity trip = iterator.next(); 52 | sb.append(trip.getTripduration()).append("\t"); 53 | sb.append(sdfDateTime.format(trip.getStarttime())).append("\t"); 54 | sb.append(sdfDateTime.format(trip.getStoptime())).append("\t"); 55 | sb.append(trip.getStartStationId()).append("\t"); 56 | sb.append(trip.getStartStationName()).append("\t"); 57 | sb.append(trip.getStartStationLatitude()).append("\t"); 58 | sb.append(trip.getStartStationLongitude()).append("\t"); 59 | sb.append(trip.getEndStationId()).append("\t"); 60 | sb.append(trip.getEndStationName()).append("\t"); 61 | sb.append(trip.getEndStationLatitude()).append("\t"); 62 | sb.append(trip.getEndStationLongitude()).append("\t"); 63 | sb.append(trip.getBikeId()).append("\t"); 64 | sb.append(trip.getUserType()).append("\t"); 65 | sb.append(nullify(trip.getBirthYear())).append("\t"); 66 | sb.append(trip.getGender()).append("\t"); 67 | sb.append("\n"); 68 | cont++; 69 | if (cont % batchSize == 0) { 70 | Statement statement = (Statement) unwrapped.createStatement(); 71 | InputStream is = new ByteArrayInputStream(sb.toString().getBytes()); 72 | statement.setLocalInfileInputStream(is); 73 | statement.execute(LOADDATA); 74 | sb.setLength(0); 75 | } 76 | } 77 | } catch (SQLException e) { 78 | e.printStackTrace(); 79 | }}) 80 | ); 81 | 82 | } 83 | } 84 | 85 | private static String nullify(Integer value) { 86 | if (value==null) { 87 | return "\\N"; 88 | } 89 | return value.toString(); 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/BenchmarkMeter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.util.function.Supplier; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | public class BenchmarkMeter { 24 | 25 | private static final Logger LOGGER = LoggerFactory.getLogger(BenchmarkMeter.class); 26 | 27 | private String test; 28 | private String db; 29 | private int size; 30 | 31 | public BenchmarkMeter(Class class1, String properties, int size) { 32 | this.test = class1.getSimpleName(); 33 | if (properties.toLowerCase().contains("my")) { 34 | this.db = "mysql"; 35 | } else { 36 | this.db = "postgres"; 37 | } 38 | this.size = size; 39 | } 40 | 41 | public void meter(Supplier a) { 42 | long before = System.currentTimeMillis(); 43 | Long lines = a.get(); 44 | long after = System.currentTimeMillis(); 45 | LOGGER.info(test+","+db+","+size+","+(after-before)/1000+","+lines); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/CityBikeParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.text.ParseException; 19 | import java.text.SimpleDateFormat; 20 | import java.util.Date; 21 | import java.util.function.Supplier; 22 | 23 | public class CityBikeParser { 24 | 25 | private final SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-M-d HH:mm:ss"); 26 | private final SimpleDateFormat sdf2 = new SimpleDateFormat("M/d/yyyy H:mm"); 27 | 28 | private Supplier constructor; 29 | 30 | public CityBikeParser(Supplier constructor) { 31 | this.constructor = constructor; 32 | } 33 | 34 | public T parse(String line) { 35 | String[] columns = CsvSplit.split(line, 15); 36 | T trip = constructor.get(); 37 | trip.setTripduration(parseInt(columns[0])); 38 | trip.setStarttime(parseDate(columns[1])); 39 | trip.setStoptime(parseDate(columns[2])); 40 | trip.setStartStationId(parseInt(columns[3])); 41 | trip.setStartStationName(columns[4]); 42 | trip.setStartStationLatitude(parseDouble(columns[5])); 43 | trip.setStartStationLongitude(parseDouble(columns[6])); 44 | trip.setEndStationId(parseInt(columns[7])); 45 | trip.setEndStationName(columns[8]); 46 | trip.setEndStationLatitude(parseDouble(columns[9])); 47 | trip.setEndStationLongitude(parseDouble(columns[10])); 48 | trip.setBikeId(parseLong(columns[11])); 49 | trip.setUserType(columns[12]); 50 | trip.setBirthYear(parseInt(columns[13])); 51 | trip.setGender(parseChar(columns[14])); 52 | return trip; 53 | } 54 | 55 | public Date parseDate(String date) { 56 | try { 57 | if (date.contains("-")) { 58 | return sdf1.parse(date); 59 | } else { 60 | return sdf2.parse(date); 61 | } 62 | } catch (ParseException e) { 63 | e.printStackTrace(); 64 | } 65 | return null; 66 | } 67 | 68 | private Long parseLong(String value) { 69 | if (value != null && value.length() > 0) { 70 | return Long.parseLong(value); 71 | } 72 | return null; 73 | } 74 | 75 | private Integer parseInt(String value) { 76 | if (value != null && value.length() > 0) { 77 | return Integer.parseInt(value); 78 | } 79 | return null; 80 | } 81 | 82 | private Double parseDouble(String value) { 83 | if (value != null && value.length() > 0) { 84 | return Double.parseDouble(value); 85 | } 86 | return null; 87 | } 88 | 89 | public Character parseChar(String str) { 90 | if (str == null || str.length() == 0) { 91 | return null; 92 | } 93 | return str.charAt(0); 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/CityBikeReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.io.BufferedReader; 19 | import java.io.File; 20 | import java.io.FileInputStream; 21 | import java.io.IOException; 22 | import java.io.InputStreamReader; 23 | import java.util.function.Consumer; 24 | import java.util.function.Function; 25 | import java.util.function.Predicate; 26 | import java.util.stream.Stream; 27 | import java.util.zip.ZipEntry; 28 | import java.util.zip.ZipInputStream; 29 | 30 | public class CityBikeReader { 31 | 32 | private final Function csvParser; 33 | private final File datasetDir; 34 | private final Predicate isCitibikeFile = f -> f.getName().contains("citibike-tripdata"); 35 | private final long maxElements; 36 | 37 | public CityBikeReader(String path, int elements, Function csvParser) { 38 | this.csvParser = csvParser; 39 | this.datasetDir = new File(path); 40 | this.maxElements = elements; 41 | } 42 | 43 | public CityBikeReader(String path, Function csvParser) { 44 | this(path, Integer.MAX_VALUE, csvParser); 45 | } 46 | 47 | /** 48 | * Read each ZIP file containing CSVs and apply to consumer an stream for each 49 | * file CSV found. Each CSV file is parsed in different stream. 50 | * 51 | * @param consumer 52 | */ 53 | public long forEachCsvInZip(Consumer> consumer) { 54 | StreamCounter globalCounter = new StreamCounter<>(0); 55 | Stream.of(datasetDir.listFiles()).filter(f -> f.getName().endsWith(".zip")).filter(isCitibikeFile) 56 | .forEach(f -> { 57 | try (ZipInputStream zis = new ZipInputStream(new FileInputStream(f))) { 58 | ZipEntry ze = zis.getNextEntry(); 59 | while (ze != null) { 60 | StreamCounter counter = new StreamCounter<>(); 61 | BufferedReader reader = new BufferedReader(new InputStreamReader(zis)); 62 | long remaining = Math.max(0, maxElements-globalCounter.count()); 63 | Stream trips = reader.lines().skip(1).map(l -> csvParser.apply(l)).map(counter).map(globalCounter).limit(remaining); 64 | if (globalCounter.count() < maxElements) { 65 | consumer.accept(trips); 66 | } 67 | counter.close(); 68 | ze = zis.getNextEntry(); 69 | } 70 | zis.closeEntry(); 71 | } catch (IOException e) { 72 | e.printStackTrace(); 73 | } 74 | }); 75 | return globalCounter.count(); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/ConnectionProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.io.IOException; 19 | import java.sql.Connection; 20 | import java.sql.DriverManager; 21 | import java.sql.SQLException; 22 | import java.util.function.Supplier; 23 | 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | public class ConnectionProvider extends DatabaseProperties implements Supplier { 28 | 29 | private static final Logger LOGGER = LoggerFactory.getLogger(ConnectionProvider.class); 30 | 31 | public ConnectionProvider(String propertiesName) throws IOException { 32 | super(propertiesName); 33 | } 34 | 35 | @Override 36 | public Connection get() { 37 | LOGGER.info("Connecting with {}", prop); 38 | String driver = prop.get("driver"); 39 | try { 40 | Class.forName(driver).newInstance(); 41 | return DriverManager.getConnection(prop.get("urlConnection"), prop.get("user"), prop.get("password")); 42 | } catch (ClassNotFoundException | IllegalAccessException | InstantiationException ex) { 43 | throw new RuntimeException("Can not instantiate driver " + driver, ex); 44 | } catch (SQLException ex) { 45 | throw new RuntimeException("Can not connect to database", ex); 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/CsvSplit.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | public class CsvSplit { 19 | 20 | public static String[] split(String line, int fieldsNumber) { 21 | int col = 0; 22 | int idx = 0; 23 | String[] cols = new String[fieldsNumber]; 24 | while (idx < line.length()) { 25 | if (line.charAt(idx) == '"') { 26 | idx++; 27 | int ini = idx; 28 | while (idx < line.length() && line.charAt(idx) != '"') { 29 | idx++; 30 | } 31 | cols[col++] = line.substring(ini, idx); 32 | idx++; // Skip " 33 | idx++; // Skip , 34 | } else { 35 | int ini = idx; 36 | while (idx < line.length() && line.charAt(idx) != ',') { 37 | idx++; 38 | } 39 | cols[col++] = line.substring(ini, idx); 40 | idx++; // Skip , 41 | } 42 | } 43 | return cols; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/DataSourceFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.io.IOException; 19 | import java.util.function.Supplier; 20 | 21 | import javax.sql.DataSource; 22 | 23 | import com.zaxxer.hikari.HikariDataSource; 24 | 25 | public class DataSourceFactory extends DatabaseProperties implements Supplier { 26 | 27 | public static enum DatabaseType { 28 | mysql, postgres 29 | }; 30 | 31 | public DataSourceFactory(String properties) throws IOException { 32 | super(properties); 33 | } 34 | 35 | @Override 36 | public DataSource get() { 37 | HikariDataSource ds = new HikariDataSource(); 38 | ds.setDriverClassName(prop.get("driver")); 39 | ds.setJdbcUrl(prop.get("urlConnection")); 40 | ds.setUsername(prop.get("user")); 41 | ds.setPassword(prop.get("password")); 42 | return ds; 43 | } 44 | 45 | public DatabaseType getDatabaseType() { 46 | String driver = prop.get("driver"); 47 | if (driver.contains("mysql")) { 48 | return DatabaseType.mysql; 49 | } 50 | if (driver.contains("postgresql")) { 51 | return DatabaseType.postgres; 52 | } 53 | return null; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/DatabaseProperties.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.io.File; 19 | import java.io.FileInputStream; 20 | import java.io.FileNotFoundException; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | import java.util.Properties; 26 | 27 | public class DatabaseProperties { 28 | 29 | protected Map prop = new HashMap<>(); 30 | 31 | public DatabaseProperties(String propertiesName) throws IOException { 32 | InputStream is = locateInputStream(propertiesName); 33 | Properties p = new Properties(); 34 | p.load(is); 35 | p.forEach((k, v) -> prop.put(k.toString(), v.toString())); 36 | } 37 | 38 | private InputStream locateInputStream(String propertiesName) throws FileNotFoundException { 39 | File f = new File(propertiesName); 40 | if (f.exists()) { 41 | return new FileInputStream(f); 42 | } 43 | InputStream is = this.getClass().getClassLoader().getResourceAsStream(propertiesName); 44 | if (is == null) { 45 | throw new FileNotFoundException(propertiesName); 46 | } 47 | return is; 48 | } 49 | 50 | public void with(String key, String value) { 51 | prop.put(key, value); 52 | } 53 | 54 | public String getProperty(String key) { 55 | return prop.get(key); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/EntityManagerFactoryFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.util.Arrays; 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Properties; 22 | import java.util.stream.Collectors; 23 | 24 | import javax.persistence.EntityManagerFactory; 25 | import javax.persistence.spi.PersistenceUnitInfo; 26 | import javax.sql.DataSource; 27 | 28 | import org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl; 29 | import org.hibernate.jpa.boot.internal.PersistenceUnitInfoDescriptor; 30 | 31 | import com.jerolba.benchmark.shared.DataSourceFactory.DatabaseType; 32 | 33 | public class EntityManagerFactoryFactory { 34 | 35 | private Class[] entities; 36 | private DataSource dataSource; 37 | private DatabaseType databaseType; 38 | 39 | public EntityManagerFactoryFactory(DataSourceFactory dataSourceFactory, Class... entities) { 40 | this.entities = entities; 41 | this.dataSource = dataSourceFactory.get(); 42 | this.databaseType = dataSourceFactory.getDatabaseType(); 43 | } 44 | 45 | public EntityManagerFactory newEntityManagerFactory() { 46 | String name = getClass().getSimpleName(); 47 | List entiesClassNames = Arrays.asList(entities).stream().map(Class::getName) 48 | .collect(Collectors.toList()); 49 | PersistenceUnitInfo persistenceUnitInfo = new PersistenceUnitInfoImpl(name, entiesClassNames, properties()); 50 | PersistenceUnitInfoDescriptor puiDesc = new PersistenceUnitInfoDescriptor(persistenceUnitInfo); 51 | EntityManagerFactoryBuilderImpl entityManagerFactoryBuilder = new EntityManagerFactoryBuilderImpl(puiDesc, 52 | new HashMap<>()); 53 | return entityManagerFactoryBuilder.build(); 54 | } 55 | 56 | protected DataSource getDataSource() { 57 | return dataSource; 58 | } 59 | 60 | protected Properties properties() { 61 | Properties properties = new Properties(); 62 | properties.put("hibernate.hbm2ddl.auto", "none"); 63 | properties.put("hibernate.connection.datasource", getDataSource()); 64 | properties.put("hibernate.generate_statistics", Boolean.TRUE.toString()); 65 | if (databaseType == DatabaseType.mysql) { 66 | properties.put("hibernate.dialect.storage_engine", "innodb"); 67 | } else if (databaseType == DatabaseType.postgres) { 68 | properties.put("hibernate.dialect", "org.hibernate.dialect.PostgreSQL95Dialect"); 69 | } 70 | return properties; 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/MutableInteger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | public class MutableInteger { 19 | 20 | private int value; 21 | 22 | public MutableInteger() { 23 | this(0); 24 | } 25 | 26 | public MutableInteger(int initial) { 27 | this.value = initial; 28 | } 29 | 30 | public int incAndGet() { 31 | this.value++; 32 | return value; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/PersistenceUnitInfoImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Vlad Mihalcea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.net.URL; 19 | import java.util.ArrayList; 20 | import java.util.Collections; 21 | import java.util.List; 22 | import java.util.Properties; 23 | 24 | import javax.persistence.SharedCacheMode; 25 | import javax.persistence.ValidationMode; 26 | import javax.persistence.spi.ClassTransformer; 27 | import javax.persistence.spi.PersistenceUnitInfo; 28 | import javax.persistence.spi.PersistenceUnitTransactionType; 29 | import javax.sql.DataSource; 30 | 31 | import org.hibernate.jpa.HibernatePersistenceProvider; 32 | 33 | /** 34 | * @author Vlad Mihalcea 35 | */ 36 | public class PersistenceUnitInfoImpl implements PersistenceUnitInfo { 37 | 38 | private final String persistenceUnitName; 39 | 40 | private PersistenceUnitTransactionType transactionType = PersistenceUnitTransactionType.RESOURCE_LOCAL; 41 | 42 | private final List managedClassNames; 43 | 44 | private final List mappingFileNames = new ArrayList<>(); 45 | 46 | private final Properties properties; 47 | 48 | private DataSource jtaDataSource; 49 | 50 | private DataSource nonJtaDataSource; 51 | 52 | public PersistenceUnitInfoImpl(String persistenceUnitName, List managedClassNames, Properties properties) { 53 | this.persistenceUnitName = persistenceUnitName; 54 | this.managedClassNames = managedClassNames; 55 | this.properties = properties; 56 | } 57 | 58 | @Override 59 | public String getPersistenceUnitName() { 60 | return persistenceUnitName; 61 | } 62 | 63 | @Override 64 | public String getPersistenceProviderClassName() { 65 | return HibernatePersistenceProvider.class.getName(); 66 | } 67 | 68 | @Override 69 | public PersistenceUnitTransactionType getTransactionType() { 70 | return transactionType; 71 | } 72 | 73 | @Override 74 | public DataSource getJtaDataSource() { 75 | return jtaDataSource; 76 | } 77 | 78 | public PersistenceUnitInfoImpl setJtaDataSource(DataSource jtaDataSource) { 79 | this.jtaDataSource = jtaDataSource; 80 | this.nonJtaDataSource = null; 81 | this.transactionType = PersistenceUnitTransactionType.JTA; 82 | return this; 83 | } 84 | 85 | @Override 86 | public DataSource getNonJtaDataSource() { 87 | return nonJtaDataSource; 88 | } 89 | 90 | public PersistenceUnitInfoImpl setNonJtaDataSource(DataSource nonJtaDataSource) { 91 | this.nonJtaDataSource = nonJtaDataSource; 92 | this.jtaDataSource = null; 93 | transactionType = PersistenceUnitTransactionType.RESOURCE_LOCAL; 94 | return this; 95 | } 96 | 97 | @Override 98 | public List getMappingFileNames() { 99 | return mappingFileNames; 100 | } 101 | 102 | @Override 103 | public List getJarFileUrls() { 104 | return Collections.emptyList(); 105 | } 106 | 107 | @Override 108 | public URL getPersistenceUnitRootUrl() { 109 | return null; 110 | } 111 | 112 | @Override 113 | public List getManagedClassNames() { 114 | return managedClassNames; 115 | } 116 | 117 | @Override 118 | public boolean excludeUnlistedClasses() { 119 | return false; 120 | } 121 | 122 | @Override 123 | public SharedCacheMode getSharedCacheMode() { 124 | return SharedCacheMode.UNSPECIFIED; 125 | } 126 | 127 | @Override 128 | public ValidationMode getValidationMode() { 129 | return ValidationMode.AUTO; 130 | } 131 | 132 | @Override 133 | public Properties getProperties() { 134 | return properties; 135 | } 136 | 137 | @Override 138 | public String getPersistenceXMLSchemaVersion() { 139 | return "2.1"; 140 | } 141 | 142 | @Override 143 | public ClassLoader getClassLoader() { 144 | return Thread.currentThread().getContextClassLoader(); 145 | } 146 | 147 | @Override 148 | public void addTransformer(ClassTransformer transformer) { 149 | 150 | } 151 | 152 | @Override 153 | public ClassLoader getNewTempClassLoader() { 154 | return null; 155 | } 156 | } -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/StreamCounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.io.Closeable; 19 | import java.util.function.Function; 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | * Utility class to log the number of items processed in a Stream and meter the 26 | * time elaspsed from creation. 27 | */ 28 | public class StreamCounter implements Closeable, Function { 29 | 30 | private static final Logger LOGGER = LoggerFactory.getLogger(StreamCounter.class); 31 | 32 | private final long batchSize; 33 | private final long init; 34 | private long counter; 35 | 36 | public StreamCounter() { 37 | this(10_000); 38 | } 39 | 40 | public StreamCounter(long batchSize) { 41 | this.batchSize = batchSize; 42 | this.init = System.nanoTime(); 43 | this.counter = 0; 44 | } 45 | 46 | @Override 47 | public T apply(T t) { 48 | return count(t); 49 | } 50 | 51 | public T count(T item) { 52 | counter++; 53 | if (batchSize>0 && counter % batchSize == 0) { 54 | trace(); 55 | } 56 | return item; 57 | } 58 | 59 | public void trace() { 60 | LOGGER.info("Processed " + counter + " items in " + (System.nanoTime() - init) / 1_000_000 + " ms"); 61 | } 62 | 63 | public long count() { 64 | return counter; 65 | } 66 | 67 | @Override 68 | public void close() { 69 | trace(); 70 | } 71 | 72 | } -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/StringBuilderReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.io.IOException; 19 | import java.io.Reader; 20 | 21 | public class StringBuilderReader extends Reader { 22 | 23 | private final StringBuilder sb; 24 | private int next = 0; 25 | 26 | public StringBuilderReader(StringBuilder sb) { 27 | this.sb = sb; 28 | } 29 | 30 | @Override 31 | public int read(char cbuf[], int off, int len) throws IOException { 32 | if (off < 0 || off > cbuf.length || len < 0 || off + len > cbuf.length) { 33 | throw new IndexOutOfBoundsException("off=" + off + " len=" + len + " cbuf.length=" + cbuf.length); 34 | } 35 | if (len == 0) { 36 | return 0; 37 | } 38 | if (next >= sb.length()) { 39 | return -1; 40 | } 41 | int n = Math.min(sb.length() - next, len); 42 | sb.getChars(next, next + n, cbuf, off); 43 | next += n; 44 | return n; 45 | } 46 | 47 | @Override 48 | public void close() throws IOException { 49 | sb.setLength(0); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/TableHelper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.sql.Connection; 19 | import java.sql.SQLException; 20 | import java.sql.Statement; 21 | 22 | import javax.sql.DataSource; 23 | 24 | import org.postgresql.jdbc.PgConnection; 25 | 26 | public class TableHelper { 27 | 28 | public static void createTable(DataSourceFactory dsFactory) throws SQLException { 29 | DataSource dataSource = dsFactory.get(); 30 | try (Connection conn = dataSource.getConnection()) { 31 | TableHelper.createTable(conn); 32 | } 33 | } 34 | 35 | public static void createTable(Connection connection) throws SQLException { 36 | String create = ""; 37 | if (isType(connection, com.mysql.jdbc.Connection.class)) { 38 | create = "CREATE TABLE bike_trip (id INT NOT NULL AUTO_INCREMENT, tripduration INT NOT NULL, " 39 | + "starttime DATETIME, stoptime DATETIME, start_station_id INT NOT NULL, start_station_name VARCHAR(255), " 40 | + "start_station_latitude DOUBLE NOT NULL, start_station_longitude DOUBLE NOT NULL, " 41 | + "end_station_id INT NOT NULL, end_station_name VARCHAR(255), end_station_latitude DOUBLE NOT NULL, " 42 | + "end_station_longitude DOUBLE NOT NULL, bike_id BIGINT NOT NULL, user_type VARCHAR(255), " 43 | + "birth_year INT, gender CHAR, PRIMARY KEY (id))"; 44 | } else if (isType(connection, PgConnection.class)) { 45 | create = "CREATE TABLE bike_trip (id SERIAL, tripduration INT NOT NULL, " 46 | + "starttime DATE, stoptime DATE, start_station_id INT NOT NULL, start_station_name VARCHAR(255), " 47 | + "start_station_latitude FLOAT NOT NULL, start_station_longitude FLOAT NOT NULL, " 48 | + "end_station_id INT NOT NULL, end_station_name VARCHAR(255), end_station_latitude FLOAT NOT NULL, " 49 | + "end_station_longitude FLOAT NOT NULL, bike_id BIGINT NOT NULL, user_type VARCHAR(255), " 50 | + "birth_year INT, gender CHAR, PRIMARY KEY (id))"; 51 | } 52 | try (Statement stmt = connection.createStatement()) { 53 | stmt.execute("DROP TABLE IF EXISTS bike_trip"); 54 | stmt.execute(create); 55 | } 56 | }; 57 | 58 | private static boolean isType(Connection connection, Class clasz) { 59 | try { 60 | Object unwrap = connection.unwrap(clasz); 61 | return unwrap!=null; 62 | } catch (SQLException e) { 63 | return false; 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/Trip.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.util.Date; 19 | 20 | public interface Trip { 21 | 22 | void setTripduration(int tripduration); 23 | 24 | void setStarttime(Date starttime); 25 | 26 | void setStoptime(Date stoptime); 27 | 28 | void setStartStationId(int startStationId); 29 | 30 | void setStartStationName(String startStationName); 31 | 32 | void setStartStationLatitude(double startStationLatitude); 33 | 34 | void setStartStationLongitude(double startStationLongitude); 35 | 36 | void setEndStationId(int endStationId); 37 | 38 | void setEndStationName(String endStationName); 39 | 40 | void setEndStationLatitude(double endStationLatitude); 41 | 42 | void setEndStationLongitude(double endStationLongitude); 43 | 44 | void setBikeId(long bikeId); 45 | 46 | void setUserType(String userType); 47 | 48 | void setBirthYear(Integer birthYear); 49 | 50 | void setGender(Character gender); 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/TripEntity.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.util.Date; 19 | 20 | import javax.persistence.Column; 21 | import javax.persistence.Entity; 22 | import javax.persistence.GeneratedValue; 23 | import javax.persistence.GenerationType; 24 | import javax.persistence.Id; 25 | import javax.persistence.Table; 26 | 27 | @Entity 28 | @Table(name = "bike_trip") 29 | public class TripEntity implements Trip { 30 | 31 | @Id 32 | @GeneratedValue(strategy = GenerationType.IDENTITY) 33 | private Integer id; 34 | 35 | @Column(name = "tripduration") 36 | private int tripduration; 37 | 38 | @Column(name = "starttime") 39 | private Date starttime; 40 | 41 | @Column(name = "stoptime") 42 | private Date stoptime; 43 | 44 | @Column(name = "start_station_id") 45 | private int startStationId; 46 | 47 | @Column(name = "start_station_name") 48 | private String startStationName; 49 | 50 | @Column(name = "start_station_latitude") 51 | private double startStationLatitude; 52 | 53 | @Column(name = "start_station_longitude") 54 | private double startStationLongitude; 55 | 56 | @Column(name = "end_station_id") 57 | private int endStationId; 58 | 59 | @Column(name = "end_station_name") 60 | private String endStationName; 61 | 62 | @Column(name = "end_station_latitude") 63 | private double endStationLatitude; 64 | 65 | @Column(name = "end_station_longitude") 66 | private double endStationLongitude; 67 | 68 | @Column(name = "bike_id") 69 | private long bikeId; 70 | 71 | @Column(name = "user_type") 72 | private String userType; 73 | 74 | @Column(name = "birth_year") 75 | private Integer birthYear; 76 | 77 | @Column(name = "gender") 78 | private Character gender; 79 | 80 | public Integer getId() { 81 | return id; 82 | } 83 | 84 | public void setId(Integer id) { 85 | this.id = id; 86 | } 87 | 88 | public int getTripduration() { 89 | return tripduration; 90 | } 91 | 92 | @Override 93 | public void setTripduration(int tripduration) { 94 | this.tripduration = tripduration; 95 | } 96 | 97 | public Date getStarttime() { 98 | return starttime; 99 | } 100 | 101 | @Override 102 | public void setStarttime(Date starttime) { 103 | this.starttime = starttime; 104 | } 105 | 106 | public Date getStoptime() { 107 | return stoptime; 108 | } 109 | 110 | @Override 111 | public void setStoptime(Date stoptime) { 112 | this.stoptime = stoptime; 113 | } 114 | 115 | public int getStartStationId() { 116 | return startStationId; 117 | } 118 | 119 | @Override 120 | public void setStartStationId(int startStationId) { 121 | this.startStationId = startStationId; 122 | } 123 | 124 | public String getStartStationName() { 125 | return startStationName; 126 | } 127 | 128 | @Override 129 | public void setStartStationName(String startStationName) { 130 | this.startStationName = startStationName; 131 | } 132 | 133 | public double getStartStationLatitude() { 134 | return startStationLatitude; 135 | } 136 | 137 | @Override 138 | public void setStartStationLatitude(double startStationLatitude) { 139 | this.startStationLatitude = startStationLatitude; 140 | } 141 | 142 | public double getStartStationLongitude() { 143 | return startStationLongitude; 144 | } 145 | 146 | @Override 147 | public void setStartStationLongitude(double startStationLongitude) { 148 | this.startStationLongitude = startStationLongitude; 149 | } 150 | 151 | public int getEndStationId() { 152 | return endStationId; 153 | } 154 | 155 | @Override 156 | public void setEndStationId(int endStationId) { 157 | this.endStationId = endStationId; 158 | } 159 | 160 | public String getEndStationName() { 161 | return endStationName; 162 | } 163 | 164 | @Override 165 | public void setEndStationName(String endStationName) { 166 | this.endStationName = endStationName; 167 | } 168 | 169 | public double getEndStationLatitude() { 170 | return endStationLatitude; 171 | } 172 | 173 | @Override 174 | public void setEndStationLatitude(double endStationLatitude) { 175 | this.endStationLatitude = endStationLatitude; 176 | } 177 | 178 | public double getEndStationLongitude() { 179 | return endStationLongitude; 180 | } 181 | 182 | @Override 183 | public void setEndStationLongitude(double endStationLongitude) { 184 | this.endStationLongitude = endStationLongitude; 185 | } 186 | 187 | public long getBikeId() { 188 | return bikeId; 189 | } 190 | 191 | @Override 192 | public void setBikeId(long bikeId) { 193 | this.bikeId = bikeId; 194 | } 195 | 196 | public String getUserType() { 197 | return userType; 198 | } 199 | 200 | @Override 201 | public void setUserType(String userType) { 202 | this.userType = userType; 203 | } 204 | 205 | public Integer getBirthYear() { 206 | return birthYear; 207 | } 208 | 209 | @Override 210 | public void setBirthYear(Integer birthYear) { 211 | this.birthYear = birthYear; 212 | } 213 | 214 | public Character getGender() { 215 | return gender; 216 | } 217 | 218 | @Override 219 | public void setGender(Character gender) { 220 | this.gender = gender; 221 | } 222 | 223 | } 224 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/TripEntityInsert.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.math.BigDecimal; 19 | import java.math.BigInteger; 20 | import java.sql.PreparedStatement; 21 | import java.sql.SQLException; 22 | import java.sql.Time; 23 | import java.sql.Timestamp; 24 | import java.time.LocalDate; 25 | import java.time.LocalDateTime; 26 | import java.time.LocalTime; 27 | 28 | public class TripEntityInsert { 29 | 30 | public static final String INSERT = "INSERT INTO bike_trip (tripduration, starttime, stoptime, start_station_id, start_station_name, " 31 | + "start_station_latitude, start_station_longitude, end_station_id, end_station_name, end_station_latitude, " 32 | + "end_station_longitude, bike_id, user_type, birth_year, gender) " 33 | + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; 34 | 35 | public void setParameters(PreparedStatement pstmt, TripEntity entity) throws SQLException { 36 | setObject(pstmt, 1, entity.getTripduration()); 37 | setObject(pstmt, 2, entity.getStarttime()); 38 | setObject(pstmt, 3, entity.getStoptime()); 39 | setObject(pstmt, 4, entity.getStartStationId()); 40 | setObject(pstmt, 5, entity.getStartStationName()); 41 | setObject(pstmt, 6, entity.getStartStationLatitude()); 42 | setObject(pstmt, 7, entity.getStartStationLongitude()); 43 | setObject(pstmt, 8, entity.getEndStationId()); 44 | setObject(pstmt, 9, entity.getEndStationName()); 45 | setObject(pstmt, 10, entity.getEndStationLatitude()); 46 | setObject(pstmt, 11, entity.getEndStationLongitude()); 47 | setObject(pstmt, 12, entity.getBikeId()); 48 | setObject(pstmt, 13, entity.getUserType()); 49 | setObject(pstmt, 14, entity.getBirthYear()); 50 | setObject(pstmt, 15, entity.getGender()); 51 | } 52 | 53 | /* 54 | * Own setObject implementation to avoid problems between different setObject 55 | * implementations in each driver. Extracted from JFleet implementation. 56 | */ 57 | public void setObject(PreparedStatement pstmt, int parameterIndex, Object parameterObj) throws SQLException { 58 | if (parameterObj == null) { 59 | pstmt.setNull(parameterIndex, java.sql.Types.OTHER); 60 | } else { 61 | if (parameterObj instanceof Integer) { 62 | pstmt.setInt(parameterIndex, ((Integer) parameterObj).intValue()); 63 | } else if (parameterObj instanceof String) { 64 | pstmt.setString(parameterIndex, (String) parameterObj); 65 | } else if (parameterObj instanceof Long) { 66 | pstmt.setLong(parameterIndex, ((Long) parameterObj).longValue()); 67 | } else if (parameterObj instanceof Boolean) { 68 | pstmt.setBoolean(parameterIndex, ((Boolean) parameterObj).booleanValue()); 69 | } else if (parameterObj instanceof java.util.Date) { 70 | pstmt.setTimestamp(parameterIndex, new Timestamp(((java.util.Date) parameterObj).getTime())); 71 | } else if (parameterObj instanceof BigDecimal) { 72 | pstmt.setBigDecimal(parameterIndex, (BigDecimal) parameterObj); 73 | } else if (parameterObj instanceof Byte) { 74 | pstmt.setInt(parameterIndex, ((Byte) parameterObj).intValue()); 75 | } else if (parameterObj instanceof Character) { 76 | pstmt.setString(parameterIndex, ((Character) parameterObj).toString()); 77 | } else if (parameterObj instanceof Short) { 78 | pstmt.setShort(parameterIndex, ((Short) parameterObj).shortValue()); 79 | } else if (parameterObj instanceof Float) { 80 | pstmt.setFloat(parameterIndex, ((Float) parameterObj).floatValue()); 81 | } else if (parameterObj instanceof Double) { 82 | pstmt.setDouble(parameterIndex, ((Double) parameterObj).doubleValue()); 83 | } else if (parameterObj instanceof java.sql.Date) { 84 | pstmt.setDate(parameterIndex, (java.sql.Date) parameterObj); 85 | } else if (parameterObj instanceof Time) { 86 | pstmt.setTime(parameterIndex, (Time) parameterObj); 87 | } else if (parameterObj instanceof Timestamp) { 88 | pstmt.setTimestamp(parameterIndex, (Timestamp) parameterObj); 89 | } else if (parameterObj instanceof BigInteger) { 90 | pstmt.setObject(parameterIndex, parameterObj); 91 | } else if (parameterObj instanceof LocalDate) { 92 | pstmt.setObject(parameterIndex, parameterObj); 93 | } else if (parameterObj instanceof LocalTime) { 94 | pstmt.setObject(parameterIndex, parameterObj); 95 | } else if (parameterObj instanceof LocalDateTime) { 96 | pstmt.setObject(parameterIndex, parameterObj); 97 | } else { 98 | throw new RuntimeException("No type mapper for " + parameterObj.getClass()); 99 | } 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/jerolba/benchmark/shared/TripEntityJpa.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Jerónimo López Bezanilla 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.jerolba.benchmark.shared; 17 | 18 | import java.util.Date; 19 | 20 | import javax.persistence.Column; 21 | import javax.persistence.Entity; 22 | import javax.persistence.Id; 23 | import javax.persistence.Table; 24 | 25 | @Entity 26 | @Table(name = "bike_trip") 27 | public class TripEntityJpa implements Trip { 28 | 29 | @Id 30 | private Integer id; 31 | 32 | @Column(name = "tripduration") 33 | private int tripduration; 34 | 35 | @Column(name = "starttime") 36 | private Date starttime; 37 | 38 | @Column(name = "stoptime") 39 | private Date stoptime; 40 | 41 | @Column(name = "start_station_id") 42 | private int startStationId; 43 | 44 | @Column(name = "start_station_name") 45 | private String startStationName; 46 | 47 | @Column(name = "start_station_latitude") 48 | private double startStationLatitude; 49 | 50 | @Column(name = "start_station_longitude") 51 | private double startStationLongitude; 52 | 53 | @Column(name = "end_station_id") 54 | private int endStationId; 55 | 56 | @Column(name = "end_station_name") 57 | private String endStationName; 58 | 59 | @Column(name = "end_station_latitude") 60 | private double endStationLatitude; 61 | 62 | @Column(name = "end_station_longitude") 63 | private double endStationLongitude; 64 | 65 | @Column(name = "bike_id") 66 | private long bikeId; 67 | 68 | @Column(name = "user_type") 69 | private String userType; 70 | 71 | @Column(name = "birth_year") 72 | private Integer birthYear; 73 | 74 | @Column(name = "gender") 75 | private Character gender; 76 | 77 | public Integer getId() { 78 | return id; 79 | } 80 | 81 | public void setId(Integer id) { 82 | this.id = id; 83 | } 84 | 85 | public int getTripduration() { 86 | return tripduration; 87 | } 88 | 89 | @Override 90 | public void setTripduration(int tripduration) { 91 | this.tripduration = tripduration; 92 | } 93 | 94 | public Date getStarttime() { 95 | return starttime; 96 | } 97 | 98 | @Override 99 | public void setStarttime(Date starttime) { 100 | this.starttime = starttime; 101 | } 102 | 103 | public Date getStoptime() { 104 | return stoptime; 105 | } 106 | 107 | @Override 108 | public void setStoptime(Date stoptime) { 109 | this.stoptime = stoptime; 110 | } 111 | 112 | public int getStartStationId() { 113 | return startStationId; 114 | } 115 | 116 | @Override 117 | public void setStartStationId(int startStationId) { 118 | this.startStationId = startStationId; 119 | } 120 | 121 | public String getStartStationName() { 122 | return startStationName; 123 | } 124 | 125 | @Override 126 | public void setStartStationName(String startStationName) { 127 | this.startStationName = startStationName; 128 | } 129 | 130 | public double getStartStationLatitude() { 131 | return startStationLatitude; 132 | } 133 | 134 | @Override 135 | public void setStartStationLatitude(double startStationLatitude) { 136 | this.startStationLatitude = startStationLatitude; 137 | } 138 | 139 | public double getStartStationLongitude() { 140 | return startStationLongitude; 141 | } 142 | 143 | @Override 144 | public void setStartStationLongitude(double startStationLongitude) { 145 | this.startStationLongitude = startStationLongitude; 146 | } 147 | 148 | public int getEndStationId() { 149 | return endStationId; 150 | } 151 | 152 | @Override 153 | public void setEndStationId(int endStationId) { 154 | this.endStationId = endStationId; 155 | } 156 | 157 | public String getEndStationName() { 158 | return endStationName; 159 | } 160 | 161 | @Override 162 | public void setEndStationName(String endStationName) { 163 | this.endStationName = endStationName; 164 | } 165 | 166 | public double getEndStationLatitude() { 167 | return endStationLatitude; 168 | } 169 | 170 | @Override 171 | public void setEndStationLatitude(double endStationLatitude) { 172 | this.endStationLatitude = endStationLatitude; 173 | } 174 | 175 | public double getEndStationLongitude() { 176 | return endStationLongitude; 177 | } 178 | 179 | @Override 180 | public void setEndStationLongitude(double endStationLongitude) { 181 | this.endStationLongitude = endStationLongitude; 182 | } 183 | 184 | public long getBikeId() { 185 | return bikeId; 186 | } 187 | 188 | @Override 189 | public void setBikeId(long bikeId) { 190 | this.bikeId = bikeId; 191 | } 192 | 193 | public String getUserType() { 194 | return userType; 195 | } 196 | 197 | @Override 198 | public void setUserType(String userType) { 199 | this.userType = userType; 200 | } 201 | 202 | public Integer getBirthYear() { 203 | return birthYear; 204 | } 205 | 206 | @Override 207 | public void setBirthYear(Integer birthYear) { 208 | this.birthYear = birthYear; 209 | } 210 | 211 | public Character getGender() { 212 | return gender; 213 | } 214 | 215 | @Override 216 | public void setGender(Character gender) { 217 | this.gender = gender; 218 | } 219 | 220 | } 221 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | 23 | 24 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/resources/mysql.properties: -------------------------------------------------------------------------------- 1 | driver=com.mysql.jdbc.Driver 2 | user=test 3 | password=test 4 | urlConnection=jdbc:mysql://localhost/testdb?useSSL=false 5 | -------------------------------------------------------------------------------- /src/main/resources/mysql2.properties: -------------------------------------------------------------------------------- 1 | driver=com.mysql.jdbc.Driver 2 | user=test 3 | password=test 4 | urlConnection=jdbc:mysql://localhost/testdb?useSSL=false&rewriteBatchedStatements=true 5 | -------------------------------------------------------------------------------- /src/main/resources/postgres.properties: -------------------------------------------------------------------------------- 1 | driver=org.postgresql.Driver 2 | user=test 3 | password=test 4 | urlConnection=jdbc:postgresql://localhost:5432/testdb 5 | -------------------------------------------------------------------------------- /src/main/resources/postgres2.properties: -------------------------------------------------------------------------------- 1 | driver=org.postgresql.Driver 2 | user=test 3 | password=test 4 | urlConnection=jdbc:postgresql://localhost:5432/testdb?reWriteBatchedInserts=true 5 | --------------------------------------------------------------------------------