├── .github └── workflows │ └── maven-verify-build.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── pom.xml └── src └── main ├── csharp ├── HelloWorld │ ├── .gitignore │ ├── HelloWorld.csproj │ ├── HelloWorld.sln │ ├── Program.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ └── packages.config ├── README.md └── StockTicker │ ├── CLIOptions.cs │ ├── DBUtil.cs │ ├── Dockerfile │ ├── Properties │ └── AssemblyInfo.cs │ ├── StockTicker.cs │ ├── StockTicker.csproj │ ├── StockTicker.sln │ ├── TickerInfo.cs │ ├── packages.config │ └── run.sh └── java └── com └── yugabyte └── sample ├── Main.java ├── apps ├── AppBase.java ├── AppConfig.java ├── CassandraBatchKeyValue.java ├── CassandraBatchTimeseries.java ├── CassandraEventData.java ├── CassandraHelloWorld.java ├── CassandraInserts.java ├── CassandraKeyValue.java ├── CassandraKeyValueBase.java ├── CassandraPersonalization.java ├── CassandraRangeKeyValue.java ├── CassandraSecondaryIndex.java ├── CassandraStockTicker.java ├── CassandraTimeseries.java ├── CassandraTransactionalKeyValue.java ├── CassandraTransactionalRestartRead.java ├── CassandraUniqueSecondaryIndex.java ├── CassandraUserId.java ├── MultiTableMultiIndexInserts.java ├── RedisHashPipelined.java ├── RedisKeyValue.java ├── RedisPipelinedKeyValue.java ├── RedisYBClientKeyValue.java ├── SQLAppBase.java ├── SqlBankTransfers.java ├── SqlConsistentHashing.java ├── SqlDataLoad.java ├── SqlEventCounter.java ├── SqlForeignKeysAndJoins.java ├── SqlGeoPartitionedTable.java ├── SqlInserts.java ├── SqlMessageQueue.java ├── SqlSecondaryIndex.java ├── SqlSnapshotTxns.java ├── SqlStaleReadDetector.java ├── SqlUpdates.java ├── SqlWarehouseStock.java └── anomalies │ └── SqlQueryLatencyIncrease.java └── common ├── CmdLineOpts.java ├── IOPSThread.java ├── LogUtil.java ├── RedisHashLoadGenerator.java ├── SimpleLoadGenerator.java ├── Throttler.java ├── TimeseriesLoadGenerator.java └── metrics ├── JsonStatsMetric.java ├── Metric.java ├── MetricsTracker.java ├── Observation.java ├── PromMetrics.java ├── ReadableStatsMetric.java ├── StatsTracker.java ├── ThroughputObserver.java └── ThroughputStats.java /.github/workflows/maven-verify-build.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Java CI with Maven 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up JDK 1.8 20 | uses: actions/setup-java@v1 21 | with: 22 | java-version: 1.8 23 | - name: Build with Maven 24 | run: mvn -B -DskipDockerBuild package --file pom.xml 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .idea 3 | yb-sample-apps.iml 4 | /.classpath 5 | /.project 6 | /.settings/ 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre-alpine 2 | MAINTAINER YugaByte 3 | ENV container=yb-sample-apps 4 | 5 | WORKDIR /opt/yugabyte 6 | 7 | ARG JAR_FILE 8 | ADD target/${JAR_FILE} /opt/yugabyte/yb-sample-apps.jar 9 | 10 | ENTRYPOINT ["/usr/bin/java", "-jar", "/opt/yugabyte/yb-sample-apps.jar"] 11 | -------------------------------------------------------------------------------- /src/main/csharp/HelloWorld/.gitignore: -------------------------------------------------------------------------------- 1 | # Autosave files 2 | *~ 3 | 4 | # build 5 | [Oo]bj/ 6 | [Bb]in/ 7 | packages/ 8 | TestResults/ 9 | 10 | # globs 11 | Makefile.in 12 | *.DS_Store 13 | *.sln.cache 14 | *.suo 15 | *.cache 16 | *.pidb 17 | *.userprefs 18 | *.usertasks 19 | config.log 20 | config.make 21 | config.status 22 | aclocal.m4 23 | install-sh 24 | autom4te.cache/ 25 | *.user 26 | *.tar.gz 27 | tarballs/ 28 | test-results/ 29 | Thumbs.db 30 | 31 | # Mac bundle stuff 32 | *.dmg 33 | *.app 34 | 35 | # resharper 36 | *_Resharper.* 37 | *.Resharper 38 | 39 | # dotCover 40 | *.dotCover 41 | -------------------------------------------------------------------------------- /src/main/csharp/HelloWorld/HelloWorld.csproj: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | 15 | 16 | 17 | Debug 18 | x86 19 | {977BE115-CEDD-43D9-AC92-688A9D1E813B} 20 | Exe 21 | HelloWorld 22 | HelloWorld 23 | v4.5 24 | 25 | 26 | true 27 | full 28 | false 29 | bin\Debug 30 | DEBUG; 31 | prompt 32 | 4 33 | true 34 | x86 35 | 36 | 37 | true 38 | bin\Release 39 | prompt 40 | 4 41 | true 42 | x86 43 | 44 | 45 | 46 | 47 | packages\Mono.Options.4.4.0.0\lib\net4-client\Mono.Options.dll 48 | 49 | 50 | packages\lz4net.1.0.10.93\lib\net4-client\LZ4.dll 51 | 52 | 53 | 54 | packages\Microsoft.Extensions.DependencyInjection.Abstractions.1.0.0\lib\netstandard1.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll 55 | 56 | 57 | packages\Microsoft.Extensions.Logging.Abstractions.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.Abstractions.dll 58 | 59 | 60 | packages\Microsoft.Extensions.Logging.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.dll 61 | 62 | 63 | packages\CassandraCSharpDriver.3.2.1\lib\net45\Cassandra.dll 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /src/main/csharp/HelloWorld/HelloWorld.sln: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | Microsoft Visual Studio Solution File, Format Version 12.00 15 | # Visual Studio 2012 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HelloWorld", "HelloWorld.csproj", "{977BE115-CEDD-43D9-AC92-688A9D1E813B}" 17 | EndProject 18 | Global 19 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 20 | Debug|x86 = Debug|x86 21 | Release|x86 = Release|x86 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Debug|x86.ActiveCfg = Debug|x86 25 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Debug|x86.Build.0 = Debug|x86 26 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Release|x86.ActiveCfg = Release|x86 27 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Release|x86.Build.0 = Release|x86 28 | EndGlobalSection 29 | GlobalSection(MonoDevelopProperties) = preSolution 30 | Policies = $0 31 | $0.TextStylePolicy = $1 32 | $1.FileWidth = 120 33 | $1.TabWidth = 2 34 | $1.IndentWidth = 2 35 | $1.inheritsSet = VisualStudio 36 | $1.inheritsScope = text/plain 37 | $1.scope = text/x-csharp 38 | $0.CSharpFormattingPolicy = $2 39 | $2.IndentSwitchSection = True 40 | $2.NewLinesForBracesInProperties = True 41 | $2.NewLinesForBracesInAccessors = True 42 | $2.NewLinesForBracesInAnonymousMethods = True 43 | $2.NewLinesForBracesInControlBlocks = True 44 | $2.NewLinesForBracesInAnonymousTypes = True 45 | $2.NewLinesForBracesInObjectCollectionArrayInitializers = True 46 | $2.NewLinesForBracesInLambdaExpressionBody = True 47 | $2.NewLineForElse = True 48 | $2.NewLineForCatch = True 49 | $2.NewLineForFinally = True 50 | $2.NewLineForMembersInObjectInit = True 51 | $2.NewLineForMembersInAnonymousTypes = True 52 | $2.NewLineForClausesInQuery = True 53 | $2.SpacingAfterMethodDeclarationName = False 54 | $2.SpaceAfterMethodCallName = False 55 | $2.SpaceBeforeOpenSquareBracket = False 56 | $2.inheritsSet = Mono 57 | $2.inheritsScope = text/x-csharp 58 | $2.scope = text/x-csharp 59 | EndGlobalSection 60 | EndGlobal 61 | -------------------------------------------------------------------------------- /src/main/csharp/HelloWorld/Program.cs: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | // Copyright (c) YugaByte, Inc. 15 | 16 | using System; 17 | using System.Net; 18 | using System.Collections.Generic; 19 | using Cassandra; 20 | 21 | /** 22 | * Simple C# client which connects to the yugabyte cluster, creates a employee table, inserts 23 | * sample data into it and reads it back. 24 | */ 25 | namespace YB 26 | { 27 | class HelloWorld 28 | { 29 | public static void Main(string[] args) 30 | { 31 | List hostIpAndPorts = new List(); 32 | // We can also add multiple tserver endpoints here. Also replace the local ip 33 | // with appropriate yugabyte cluster ip. 34 | hostIpAndPorts.Add(new IPEndPoint(IPAddress.Parse("127.0.0.1"), 9042)); 35 | 36 | var cluster = Cluster.Builder() 37 | .AddContactPoints(hostIpAndPorts) 38 | .Build(); 39 | 40 | // Currently our default keyspace name is $$$_DEFAULT, we will create our tables 41 | // inside of that keyspace. 42 | var session = cluster.Connect("$$$_DEFAULT"); 43 | 44 | // Create a employee table with id as primary key 45 | session.Execute("CREATE TABLE IF NOT EXISTS employee (id int primary key, name varchar, age int)"); 46 | 47 | // Insert some dummy data into the table. 48 | 49 | session.Execute("INSERT INTO employee(id, name, age) values(1, 'John', 35)"); 50 | 51 | // Read the data from the table using id. 52 | var statement = session.Prepare("SELECT * from employee where id = ?").Bind(1); 53 | RowSet rows = session.Execute(statement); 54 | Console.WriteLine("Id\tName\tAge"); 55 | foreach (Row row in rows) 56 | Console.WriteLine("{0}\t{1}\t{2}", row["id"], row["name"], row["age"]); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/csharp/HelloWorld/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | using System.Reflection; 15 | using System.Runtime.CompilerServices; 16 | 17 | // Information about this assembly is defined by the following attributes. 18 | // Change them to the values specific to your project. 19 | 20 | [assembly: AssemblyTitle("HelloWorld")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("")] 24 | [assembly: AssemblyProduct("")] 25 | [assembly: AssemblyCopyright("${AuthorCopyright}")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | 29 | // The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}". 30 | // The form "{Major}.{Minor}.*" will automatically update the build and revision, 31 | // and "{Major}.{Minor}.{Build}.*" will update just the revision. 32 | 33 | [assembly: AssemblyVersion("1.0.*")] 34 | 35 | // The following attributes are used to specify the signing key for the assembly, 36 | // if desired. See the Mono documentation for more information about signing. 37 | 38 | //[assembly: AssemblyDelaySign(false)] 39 | //[assembly: AssemblyKeyFile("")] 40 | -------------------------------------------------------------------------------- /src/main/csharp/HelloWorld/packages.config: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 |  15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/main/csharp/README.md: -------------------------------------------------------------------------------- 1 | Developer Documentation 2 | 3 | ## Building and running CSharp client 4 | 5 | ### Pre-requisites 6 | 7 | #### General Pre-requisites 8 | * Download and install MonoDevelop (http://www.monodevelop.com/) 9 | 10 | #### YugabyteDB Universe 11 | * Have a local or remote YugabyteDB universe running with IP and Port information. 12 | 13 | #### Running HelloWorld csharp client. 14 | * Open the HelloWorld.csproj in MonoDevelop IDE. 15 | * Click on Run icon to start the project. 16 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/CLIOptions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | using CommandLine; 15 | using CommandLine.Text; 16 | using System.Collections.Generic; 17 | 18 | namespace YB 19 | { 20 | class CLIOptions 21 | { 22 | [OptionList ("nodes", Separator = ',', Required = true, HelpText = "Yugabyte node ip and port")] 23 | public List Nodes { get; set; } 24 | 25 | [Option ("command", Required = true, HelpText = "Command Type (run, create-table, drop-table).")] 26 | public string Command { get; set; } 27 | 28 | [Option ("stock_symbols", DefaultValue = 1000, HelpText = "Number of stock symbols to load.")] 29 | public int NumStockSymbols { get; set; } 30 | 31 | [Option ("num_keys_to_write", DefaultValue = -1, HelpText = "Number of keys to write.")] 32 | public int NumKeysToWrite { get; set; } 33 | 34 | [Option ("num_keys_to_read", DefaultValue = -1, HelpText = "Number of keys to reads.")] 35 | public int NumKeysToRead { get; set; } 36 | 37 | [Option ("num_writer_threads", DefaultValue = 4, HelpText = "Number of writer threads.")] 38 | public int NumWriterThreads { get; set; } 39 | 40 | [Option ("num_reader_threads", DefaultValue = 4, HelpText = "Number of reader threads.")] 41 | public int NumReaderThreads { get; set; } 42 | 43 | 44 | [Option ("data_emit_rate", DefaultValue = 1000, HelpText = "Data emit rate in milliseconds.")] 45 | public int DataEmitRate { get; set; } 46 | 47 | 48 | [HelpOption] 49 | public string GetUsage () 50 | { 51 | var help = new HelpText { 52 | Heading = new HeadingInfo ("C# Stock Ticker App", "1.0"), 53 | Copyright = new CopyrightInfo ("YugaByte, Inc", 2016), 54 | AdditionalNewLineAfterOption = true, 55 | AddDashesToOption = true 56 | }; 57 | help.AddPreOptionsLine ("Usage: StockTicker --nodes 127.0.0.1:9042"); 58 | help.AddOptions (this); 59 | return help; 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/DBUtil.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | using Cassandra; 15 | using System; 16 | using System.Collections.Generic; 17 | using System.Net; 18 | using System.Linq; 19 | using System.Collections.Concurrent; 20 | using System.Threading.Tasks; 21 | 22 | namespace YB 23 | { 24 | public class DBUtil 25 | { 26 | readonly String KeyspaceName = "ybdemo"; 27 | protected List hostIpAndPorts = new List (); 28 | protected ISession dbSession; 29 | protected Cluster dbCluster; 30 | private readonly ConcurrentDictionary>> _cachedStatements; 31 | 32 | public DBUtil (List nodes) 33 | { 34 | nodes.ForEach ((node) => hostIpAndPorts.Add (ParseIPEndPoint (node))); 35 | dbCluster = Cluster.Builder ().AddContactPoints (hostIpAndPorts).Build (); 36 | dbSession = Connect (); 37 | _cachedStatements = new ConcurrentDictionary>> (); 38 | } 39 | 40 | private static IPEndPoint ParseIPEndPoint (string node) 41 | { 42 | if (Uri.TryCreate (node, UriKind.Absolute, out Uri uri)) 43 | return new IPEndPoint (IPAddress.Parse (uri.Host), uri.Port < 0 ? 0 : uri.Port); 44 | if (Uri.TryCreate ($"tcp://{node}", UriKind.Absolute, out uri)) 45 | return new IPEndPoint (IPAddress.Parse (uri.Host), uri.Port < 0 ? 0 : uri.Port); 46 | if (Uri.TryCreate ($"tcp://[{node}]", UriKind.Absolute, out uri)) 47 | return new IPEndPoint (IPAddress.Parse (uri.Host), uri.Port < 0 ? 0 : uri.Port); 48 | throw new FormatException ("Failed to parse text to IPEndPoint"); 49 | } 50 | 51 | public Task GetOrAddQuery (string cql) 52 | { 53 | return _cachedStatements.GetOrAdd (cql, CreateQueryTask).Value; 54 | } 55 | 56 | private Lazy> CreateQueryTask (string cql) 57 | { 58 | return new Lazy> (() => dbSession.PrepareAsync (cql)); 59 | } 60 | 61 | ISession Connect () 62 | { 63 | if (dbSession == null) { 64 | Console.WriteLine ("Connect To Cluster"); 65 | try { 66 | dbSession = dbCluster.Connect (KeyspaceName); 67 | } catch (InvalidQueryException e) { 68 | if (e.Message.Contains ("Keyspace Not Found")) { 69 | Console.WriteLine ($"Keyspace {KeyspaceName} not found creating.."); 70 | CreateKeyspaceAndConnect (); 71 | } 72 | } 73 | 74 | } 75 | return dbSession; 76 | } 77 | 78 | public void Disconnect() 79 | { 80 | Console.WriteLine ("Disconnect from Cluster"); 81 | dbCluster.Shutdown (); 82 | } 83 | 84 | public RowSet ExecuteQuery (String queryStr) 85 | { 86 | return dbSession.Execute (queryStr); 87 | } 88 | 89 | public RowSet ExecuteQuery (BoundStatement statement) 90 | { 91 | return dbSession.Execute (statement); 92 | } 93 | 94 | private void CreateKeyspaceAndConnect() 95 | { 96 | dbSession = dbCluster.Connect (); 97 | dbSession.CreateKeyspace (KeyspaceName, new Dictionary { 98 | { "class", "SimpleStrategy" }, 99 | { "replication_factor", "3" } 100 | }); 101 | dbSession = dbCluster.Connect (KeyspaceName); 102 | } 103 | 104 | public void DropTable (String TableName) 105 | { 106 | Console.WriteLine ("Dropping Stock Ticker table {0}", TableName); 107 | String statementStr = $"DROP TABLE IF EXISTS {TableName}"; 108 | dbSession.Execute (statementStr); 109 | } 110 | 111 | public bool CheckTableExists (String TableName) 112 | { 113 | String statementStr = "SELECT table_name FROM system_schema.tables " + 114 | " WHERE keyspace_name = ? AND table_name = ?"; 115 | BoundStatement stmt = dbSession.Prepare(statementStr) 116 | .Bind (KeyspaceName, TableName); 117 | Row result = dbSession.Execute (stmt).FirstOrDefault (); 118 | if (result != null) { 119 | Console.WriteLine ("Stock Ticker table {0} exits.", result ["table_name"]); 120 | return true; 121 | } 122 | return false; 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | FROM mono:4.2.3.4 15 | 16 | RUN mkdir -p code 17 | COPY bin/Release/ /code 18 | COPY run.sh /code 19 | 20 | ENTRYPOINT ["/code/run.sh"] 21 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | using System.Reflection; 15 | using System.Runtime.CompilerServices; 16 | 17 | // Information about this assembly is defined by the following attributes. 18 | // Change them to the values specific to your project. 19 | 20 | [assembly: AssemblyTitle("StockTicker")] 21 | [assembly: AssemblyDescription("")] 22 | [assembly: AssemblyConfiguration("")] 23 | [assembly: AssemblyCompany("")] 24 | [assembly: AssemblyProduct("")] 25 | [assembly: AssemblyCopyright("${AuthorCopyright}")] 26 | [assembly: AssemblyTrademark("")] 27 | [assembly: AssemblyCulture("")] 28 | 29 | // The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}". 30 | // The form "{Major}.{Minor}.*" will automatically update the build and revision, 31 | // and "{Major}.{Minor}.{Build}.*" will update just the revision. 32 | 33 | [assembly: AssemblyVersion("1.0.*")] 34 | 35 | // The following attributes are used to specify the signing key for the assembly, 36 | // if desired. See the Mono documentation for more information about signing. 37 | 38 | //[assembly: AssemblyDelaySign(false)] 39 | //[assembly: AssemblyKeyFile("")] 40 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/StockTicker.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | using Cassandra; 15 | using CommandLine; 16 | using System; 17 | using System.Collections.Concurrent; 18 | using System.Linq; 19 | using System.Threading; 20 | 21 | namespace YB 22 | { 23 | class StockTicker 24 | { 25 | static int readCount = 0; 26 | static int writeCount = 0; 27 | readonly ConcurrentDictionary tickers = new ConcurrentDictionary (); 28 | static DBUtil dbUtil; 29 | Thread [] threads; 30 | readonly CLIOptions cliOptions; 31 | readonly String StockTickerTbl = "stock_ticker_1min"; 32 | readonly int NUM_CURRENCIES = 30; 33 | 34 | public static void Main (string [] args) 35 | { 36 | CLIOptions options = new CLIOptions (); 37 | if (Parser.Default.ParseArguments (args, options)) { 38 | StockTicker st = new StockTicker (options); 39 | st.Run (); 40 | } 41 | } 42 | 43 | public StockTicker (CLIOptions options) 44 | { 45 | cliOptions = options; 46 | } 47 | 48 | void Run() 49 | { 50 | dbUtil = new DBUtil (cliOptions.Nodes); 51 | 52 | switch (cliOptions.Command) { 53 | case "create-table": 54 | CreateTable (StockTickerTbl); 55 | break; 56 | case "drop-table": 57 | dbUtil.DropTable (StockTickerTbl); 58 | break; 59 | case "run": 60 | if (!dbUtil.CheckTableExists (StockTickerTbl)) { 61 | throw new Exception ("Stock Ticker Table doesn't exists, run create-table command."); 62 | } 63 | Initalize (); 64 | Array.ForEach (threads, (Thread thread) => thread.Start ()); 65 | Array.ForEach (threads, (Thread thread) => thread.Join ()); 66 | break; 67 | default: 68 | throw new Exception ("Invalid Command " + cliOptions.Command); 69 | } 70 | dbUtil.Disconnect (); 71 | } 72 | 73 | private void CreateTable (String TableName) 74 | { 75 | Console.WriteLine ("Creating Stock Ticker table {0}", TableName); 76 | String statementStr = $"CREATE TABLE IF NOT EXISTS {TableName} (identifier_id int, " + 77 | "time_stamp timestamp, first float, last float, high float, low float, " + 78 | "volume float, vwap float, twap float, trades bigint, currency int, " + 79 | "primary key ((identifier_id), time_stamp)) " + 80 | "WITH CLUSTERING ORDER BY (time_stamp DESC)"; 81 | dbUtil.ExecuteQuery (statementStr); 82 | } 83 | 84 | // We cache the prepared statement and using prepared statement yields higher performance 85 | private PreparedStatement GetInsertStatement () 86 | { 87 | return dbUtil.GetOrAddQuery ($"INSERT INTO {StockTickerTbl} (identifier_id, time_stamp, first, " + 88 | "last, high, low, volume, vwap, twap, trades, currency) " + 89 | "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").Result; 90 | } 91 | 92 | // We cache the prepared statement and using prepared statement yields higher performance 93 | private PreparedStatement GetSelectStatement () 94 | { 95 | return dbUtil.GetOrAddQuery ($"SELECT * FROM {StockTickerTbl} WHERE identifier_id = ? " + 96 | "AND time_stamp > ? AND time_stamp < ?").Result; 97 | } 98 | 99 | void Initalize () 100 | { 101 | for (int i = 0; i < cliOptions.NumStockSymbols; i++) { 102 | tickers.TryAdd (i, new TickerInfo (i)); 103 | } 104 | 105 | threads = new Thread [cliOptions.NumWriterThreads + cliOptions.NumReaderThreads]; 106 | int threadIdx = 0; 107 | for (int i = 0; i < cliOptions.NumWriterThreads; i++) { 108 | threads.SetValue (new Thread (DoWrite), threadIdx++); 109 | } 110 | for (int i = 0; i < cliOptions.NumReaderThreads; i++) { 111 | threads.SetValue (new Thread (DoRead), threadIdx++); 112 | } 113 | } 114 | 115 | public void DoWrite () 116 | { 117 | bool writeDone = false; 118 | var random = new Random (); 119 | while (!writeDone) { 120 | int randomIdx = random.Next (tickers.Count); 121 | float first = (float) random.NextDouble (); 122 | float last = (float) random.NextDouble (); 123 | float high = (float) random.NextDouble (); 124 | float low = (float) random.NextDouble (); 125 | float volume = (float) random.NextDouble (); 126 | float vwap = (float) random.NextDouble (); 127 | float twap = (float) random.NextDouble (); 128 | long trades = (long)random.Next (); 129 | int currency = random.Next (NUM_CURRENCIES); 130 | 131 | if (tickers.TryRemove (randomIdx, out TickerInfo dataSource)) { 132 | long ts = dataSource.GetDataEmitTs (); 133 | if (ts == -1) { 134 | Thread.Sleep (100); 135 | continue; 136 | } 137 | 138 | var counter = Interlocked.Increment (ref writeCount); 139 | if (cliOptions.NumKeysToWrite > 0 && counter >= cliOptions.NumKeysToWrite) break; 140 | BoundStatement stmt = GetInsertStatement ().Bind (dataSource.GetTickerId (), 141 | dataSource.GetDataEmitTime (), 142 | first, last, high, low, volume, 143 | vwap, twap, trades, currency); 144 | dbUtil.ExecuteQuery (stmt); 145 | dataSource.SetLastEmittedTs (ts); 146 | tickers.TryAdd (randomIdx, dataSource); 147 | } 148 | } 149 | } 150 | 151 | public void DoRead () 152 | { 153 | bool readDone = false; 154 | var random = new Random (); 155 | while (!readDone) { 156 | int randomIdx = random.Next (tickers.Count); 157 | if (tickers.TryRemove (randomIdx, out TickerInfo dataSource)) { 158 | if (!dataSource.HasEmittedData ()) { 159 | Thread.Sleep (100); 160 | continue; 161 | } 162 | var newReadCount = Interlocked.Increment (ref readCount); 163 | if (cliOptions.NumKeysToRead > 0 && newReadCount >= cliOptions.NumKeysToRead) break; 164 | BoundStatement stmt = GetSelectStatement ().Bind (dataSource.GetTickerId (), 165 | dataSource.GetStartTime (), 166 | dataSource.GetEndTime ()); 167 | if (dbUtil.ExecuteQuery (stmt).GetRows ().FirstOrDefault () == null) { 168 | Console.WriteLine ("READ Failed Ticker Id {0}, timestamp {1} to {2}", 169 | dataSource.GetTickerId (), 170 | dataSource.GetStartTime (), 171 | dataSource.GetEndTime ()); 172 | } 173 | 174 | tickers.TryAdd (randomIdx, dataSource); 175 | } 176 | } 177 | } 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/StockTicker.csproj: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | 15 | 16 | 17 | Debug 18 | x86 19 | {977BE115-CEDD-43D9-AC92-688A9D1E813B} 20 | Exe 21 | StockTicker 22 | StockTicker 23 | v4.5 24 | 25 | 26 | true 27 | full 28 | false 29 | bin\Debug 30 | DEBUG; 31 | prompt 32 | 4 33 | true 34 | x86 35 | 36 | 37 | true 38 | bin\Release 39 | prompt 40 | 4 41 | true 42 | x86 43 | 44 | 45 | 46 | 47 | packages\Mono.Options.4.4.0.0\lib\net4-client\Mono.Options.dll 48 | 49 | 50 | packages\lz4net.1.0.10.93\lib\net4-client\LZ4.dll 51 | 52 | 53 | 54 | packages\Microsoft.Extensions.DependencyInjection.Abstractions.1.0.0\lib\netstandard1.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll 55 | 56 | 57 | packages\Microsoft.Extensions.Logging.Abstractions.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.Abstractions.dll 58 | 59 | 60 | packages\Microsoft.Extensions.Logging.1.0.0\lib\netstandard1.1\Microsoft.Extensions.Logging.dll 61 | 62 | 63 | packages\CassandraCSharpDriver.3.2.1\lib\net45\Cassandra.dll 64 | 65 | 66 | 67 | 68 | 69 | packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/StockTicker.sln: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | Microsoft Visual Studio Solution File, Format Version 12.00 15 | # Visual Studio 2012 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StockTicker", "StockTicker.csproj", "{977BE115-CEDD-43D9-AC92-688A9D1E813B}" 17 | EndProject 18 | Global 19 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 20 | Debug|x86 = Debug|x86 21 | Release|x86 = Release|x86 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Debug|x86.ActiveCfg = Debug|x86 25 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Debug|x86.Build.0 = Debug|x86 26 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Release|x86.ActiveCfg = Release|x86 27 | {977BE115-CEDD-43D9-AC92-688A9D1E813B}.Release|x86.Build.0 = Release|x86 28 | EndGlobalSection 29 | GlobalSection(MonoDevelopProperties) = preSolution 30 | Policies = $0 31 | $0.TextStylePolicy = $1 32 | $1.TabWidth = 2 33 | $1.IndentWidth = 2 34 | $1.scope = text/x-csharp 35 | $1.TabsToSpaces = True 36 | $0.CSharpFormattingPolicy = $2 37 | $2.scope = text/x-csharp 38 | EndGlobalSection 39 | EndGlobal 40 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/TickerInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | using System; 15 | 16 | namespace YB 17 | { 18 | class TickerInfo 19 | { 20 | int tickerId; 21 | readonly long dataEmitRateMs; 22 | long lastEmittedTs = -1; 23 | long dataEmitStartTs = -1; 24 | public static DateTimeOffset dateOrigin = 25 | new DateTimeOffset (1970, 1, 1, 0, 0, 0, TimeSpan.Zero); 26 | 27 | public TickerInfo (int tickerId, long dataEmitRateMs = 1000) 28 | { 29 | this.tickerId = tickerId; 30 | this.dataEmitRateMs = dataEmitRateMs; 31 | } 32 | 33 | public int GetTickerId () 34 | { 35 | return tickerId; 36 | } 37 | 38 | public void SetLastEmittedTs (long ts) 39 | { 40 | if (dataEmitStartTs == -1) { 41 | dataEmitStartTs = ts; 42 | } 43 | if (lastEmittedTs < ts) { 44 | lastEmittedTs = ts; 45 | } 46 | } 47 | 48 | public long GetLastEmittedTs () 49 | { 50 | return lastEmittedTs; 51 | } 52 | 53 | public bool HasEmittedData () 54 | { 55 | return lastEmittedTs > -1; 56 | } 57 | 58 | public long GetDataEmitTs () 59 | { 60 | long ts = (DateTimeOffset.Now.Ticks - dateOrigin.Ticks) / TimeSpan.TicksPerMillisecond; 61 | if ((ts - lastEmittedTs) < dataEmitRateMs) { 62 | return -1; 63 | } 64 | if (lastEmittedTs == -1) { 65 | return ts - (ts % dataEmitRateMs); 66 | } 67 | return (lastEmittedTs + dataEmitRateMs); 68 | } 69 | 70 | public DateTimeOffset GetDataEmitTime() 71 | { 72 | return dateOrigin.AddMilliseconds (GetDataEmitTs()); 73 | } 74 | 75 | public DateTimeOffset GetEndTime () 76 | { 77 | // We would just add 1 second for time being until we have >= and <= 78 | return dateOrigin.AddMilliseconds (GetLastEmittedTs() + dataEmitRateMs); 79 | } 80 | 81 | public DateTimeOffset GetStartTime () 82 | { 83 | long deltaT = -1 * (60 + new Random ().Next (540)) * dataEmitRateMs; 84 | return GetEndTime ().AddMilliseconds (deltaT) ; 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/packages.config: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) YugaByte, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations 12 | # under the License. 13 | # 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/main/csharp/StockTicker/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright (c) YugaByte, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations 13 | # under the License. 14 | # 15 | if [ $# -eq 0 ] 16 | then 17 | echo "App not provided. Valid Apps:" 18 | cd code 19 | for app in *.exe 20 | do 21 | echo "$app" 22 | done 23 | exit 24 | fi 25 | mono code/$@ & 26 | PID=$! 27 | wait $PID 28 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraBatchKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.util.Arrays; 18 | import java.util.HashSet; 19 | import java.util.List; 20 | 21 | import org.apache.log4j.Logger; 22 | 23 | import com.datastax.driver.core.BatchStatement; 24 | import com.datastax.driver.core.PreparedStatement; 25 | import com.datastax.driver.core.ResultSet; 26 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 27 | 28 | /** 29 | * This workload writes and reads some random string keys from a CQL server in batches. By default, 30 | * this app inserts a million keys, and reads/updates them indefinitely. 31 | */ 32 | public class CassandraBatchKeyValue extends CassandraKeyValue { 33 | private static final Logger LOG = Logger.getLogger(CassandraBatchKeyValue.class); 34 | 35 | // Static initialization of this workload's config. 36 | static { 37 | // The number of keys to write in each batch. 38 | appConfig.batchSize = 10; 39 | } 40 | 41 | // Buffers for each key in a batch. 42 | private final byte[][] buffers; 43 | 44 | public CassandraBatchKeyValue() { 45 | buffers = new byte[appConfig.batchSize][appConfig.valueSize]; 46 | } 47 | 48 | @Override 49 | public long doWrite(int threadIdx) { 50 | BatchStatement batch = new BatchStatement(); 51 | HashSet keys = new HashSet(); 52 | PreparedStatement insert = getPreparedInsert(); 53 | try { 54 | for (int i = 0; i < appConfig.batchSize; i++) { 55 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 56 | ByteBuffer value = null; 57 | if (appConfig.valueSize == 0) { 58 | value = ByteBuffer.wrap(key.getValueStr().getBytes()); 59 | } else { 60 | buffer = buffers[i]; 61 | value = ByteBuffer.wrap(getRandomValue(key)); 62 | } 63 | keys.add(key); 64 | batch.add(insert.bind(key.asString(), value)); 65 | } 66 | // Do the write to Cassandra. 67 | ResultSet resultSet = getCassandraClient().execute(batch); 68 | LOG.debug("Wrote keys count: " + keys.size() + ", return code: " + resultSet.toString()); 69 | for (Key key : keys) { 70 | getSimpleLoadGenerator().recordWriteSuccess(key); 71 | } 72 | return keys.size(); 73 | } catch (Exception e) { 74 | for (Key key : keys) { 75 | getSimpleLoadGenerator().recordWriteFailure(key); 76 | } 77 | throw e; 78 | } 79 | } 80 | 81 | @Override 82 | public List getWorkloadDescription() { 83 | return Arrays.asList( 84 | "Sample batch key-value app built on Cassandra with concurrent reader and writer threads. ", 85 | "Note that writes are batched, while reads perform lookups by a single key. The batch size ", 86 | "to write and the number of readers and writers can be specified as parameters."); 87 | } 88 | 89 | @Override 90 | public List getWorkloadOptionalArguments() { 91 | return Arrays.asList( 92 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 93 | "--num_reads " + appConfig.numKeysToRead, 94 | "--num_writes " + appConfig.numKeysToWrite, 95 | "--value_size " + appConfig.valueSize, 96 | "--num_threads_read " + appConfig.numReaderThreads, 97 | "--num_threads_write " + appConfig.numWriterThreads, 98 | "--batch_size " + appConfig.batchSize, 99 | "--table_ttl_seconds " + appConfig.tableTTLSeconds); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraHelloWorld.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.util.Arrays; 17 | import java.util.List; 18 | 19 | import org.apache.log4j.Logger; 20 | 21 | import com.datastax.driver.core.Cluster; 22 | import com.datastax.driver.core.ResultSet; 23 | import com.datastax.driver.core.Row; 24 | import com.datastax.driver.core.Session; 25 | 26 | /** 27 | * A very simple app that creates an Employees table, inserts an employee record and reads it back. 28 | */ 29 | public class CassandraHelloWorld extends AppBase { 30 | private static final Logger LOG = Logger.getLogger(CassandraHelloWorld.class); 31 | 32 | /** 33 | * Run this simple app. 34 | */ 35 | @Override 36 | public void run() { 37 | try { 38 | // Create a Cassandra client. 39 | Session session = getCassandraClient(); 40 | 41 | // Create the keyspace and use it. 42 | String createKeyspaceStatement = 43 | "CREATE KEYSPACE IF NOT EXISTS \"my_keyspace\";"; 44 | ResultSet createKeyspaceResult = session.execute(createKeyspaceStatement); 45 | LOG.info("Created a keyspace with CQL statement: " + createKeyspaceStatement); 46 | String useKeyspaceStatement = "USE \"my_keyspace\";"; 47 | ResultSet useKeyspaceResult = session.execute(useKeyspaceStatement); 48 | LOG.info("Use the new keyspace with CQL statement: " + useKeyspaceStatement); 49 | 50 | // Create the table with a key column 'k' and a value column 'v'. 51 | String createTableStatement = 52 | "CREATE TABLE IF NOT EXISTS Employee (id int primary key, name varchar, age int);"; 53 | ResultSet createResult = session.execute(createTableStatement); 54 | LOG.info("Created a table with CQL statement: " + createTableStatement); 55 | 56 | // Write a key-value pair. 57 | String insertStatement = "INSERT INTO Employee (id, name, age) VALUES (1, 'John', 35);"; 58 | ResultSet insertResult = session.execute(insertStatement); 59 | LOG.info("Inserted a key-value pair with CQL statement: " + insertStatement); 60 | 61 | // Read the key-value pair back. 62 | String selectStatement = "SELECT name, age FROM Employee WHERE id = 1;"; 63 | ResultSet selectResult = session.execute(selectStatement); 64 | List rows = selectResult.all(); 65 | String name = rows.get(0).getString(0); 66 | int age = rows.get(0).getInt(1); 67 | // We expect only one row, with the key and value. 68 | LOG.info("Read data with CQL statement: " + selectStatement); 69 | LOG.info("Got result: row-count=" + rows.size() + ", name=" + name + ", age=" + age); 70 | 71 | // Close the client. 72 | destroyClients(); 73 | } catch (Exception e) { 74 | LOG.error("Error running CassandraHelloWorld" + e.getMessage(), e); 75 | } 76 | } 77 | 78 | // Static initialization of this workload's config. 79 | static { 80 | // Set the app type to simple. 81 | appConfig.appType = AppConfig.Type.Simple; 82 | } 83 | 84 | @Override 85 | public List getWorkloadDescription() { 86 | return Arrays.asList( 87 | "A very simple app that writes and reads one employee record into an 'Employee' table"); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraInserts.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import com.datastax.driver.core.*; 17 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 18 | import org.apache.log4j.Logger; 19 | 20 | import java.util.*; 21 | 22 | /** 23 | * This workload writes and reads some random string keys from a YCQL table with a secondary index 24 | * on a non-primary-key column. When it reads a key, it queries the key by its associated value 25 | * which is indexed. 26 | */ 27 | public class CassandraInserts extends CassandraKeyValue { 28 | private static final Logger LOG = Logger.getLogger(CassandraInserts.class); 29 | 30 | static { 31 | appConfig.readIOPSPercentage = -1; 32 | appConfig.numReaderThreads = 2; 33 | appConfig.numWriterThreads = 2; 34 | appConfig.numKeysToRead = -1; 35 | appConfig.numKeysToWrite = -1; 36 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS; 37 | } 38 | 39 | // The default table name to create and use for CRUD ops. 40 | private static final String DEFAULT_TABLE_NAME = CassandraInserts.class.getSimpleName(); 41 | // The strings used for insert and select operations. 42 | private static String insertStatement; 43 | private static String selectStatement; 44 | 45 | // The number of value columns. 46 | private static int countValCols; 47 | 48 | public CassandraInserts() { 49 | } 50 | 51 | @Override 52 | public void initializeConnectionsAndStatements(int numThreads) { 53 | countValCols = appConfig.numIndexes > 0 ? appConfig.numIndexes : 1; 54 | 55 | StringBuilder sb = new StringBuilder(); 56 | sb.append("INSERT INTO " + getTableName()); 57 | 58 | StringBuilder fields = new StringBuilder(); 59 | StringBuilder values = new StringBuilder(); 60 | 61 | fields.append("(k"); 62 | values.append("(?"); 63 | 64 | for (int i = 0; i < countValCols; ++i) { 65 | fields.append(", v" + i); 66 | values.append(", ?"); 67 | } 68 | fields.append(")"); 69 | values.append(")"); 70 | 71 | sb.append(fields); 72 | sb.append(" VALUES "); 73 | sb.append(values); 74 | sb.append(";"); 75 | insertStatement = sb.toString(); 76 | selectStatement = String.format("SELECT * FROM %s WHERE k = ?;", getTableName()); 77 | } 78 | 79 | @Override 80 | public void dropTable() { 81 | dropCassandraTable(getTableName()); 82 | } 83 | 84 | @Override 85 | public List getCreateTableStatements() { 86 | List createStmts = new ArrayList(); 87 | 88 | StringBuilder create = new StringBuilder(); 89 | create.append("CREATE TABLE IF NOT EXISTS "); 90 | create.append(getTableName()); 91 | create.append("(k text,"); 92 | 93 | for (int i = 0; i < countValCols; ++i) { 94 | create.append(" v"); 95 | create.append(i); 96 | create.append(" text,"); 97 | } 98 | create.append(" primary key (k)) "); 99 | if (!appConfig.nonTransactionalIndex) { 100 | create.append("WITH transactions = { 'enabled' : true };"); 101 | } 102 | createStmts.add(create.toString()); 103 | 104 | for (int i = 0; i < appConfig.numIndexes; ++i) { 105 | String index = String.format( 106 | "CREATE INDEX IF NOT EXISTS idx%d ON %s (v%d) %s;", i, getTableName(), i, 107 | appConfig.nonTransactionalIndex ? 108 | "WITH transactions = {'enabled' : false, 'consistency_level' : 'user_enforced'}" : ""); 109 | createStmts.add(index); 110 | } 111 | return createStmts; 112 | } 113 | 114 | public String getTableName() { 115 | return appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 116 | } 117 | 118 | private PreparedStatement getPreparedSelect() { 119 | return getPreparedSelect(selectStatement, appConfig.localReads); 120 | } 121 | 122 | @Override 123 | protected PreparedStatement getPreparedInsert() { 124 | return getPreparedInsert(insertStatement); 125 | } 126 | 127 | private static BoundStatement getBoundInsertStatement(PreparedStatement ps, Key key) { 128 | BoundStatement bs = ps.bind(); 129 | bs.setString(0, key.asString()); 130 | for (int i = 0; i < countValCols; ++i) { 131 | bs.setString(1 + i, key.getValueStr(i, appConfig.valueSize)); 132 | } 133 | return bs; 134 | } 135 | 136 | @Override 137 | public long doRead() { 138 | Key key = getSimpleLoadGenerator().getKeyToRead(); 139 | if (key == null) { 140 | return 0; 141 | } 142 | BoundStatement bound = getPreparedSelect().bind(key.asString()); 143 | ResultSet rs = getCassandraClient().execute(bound); 144 | List rows = rs.all(); 145 | LOG.debug("Read key: " + key.asString() + " return code" + rs.toString()); 146 | if (rows.size() != 1) { 147 | LOG.fatal("Read key: " + key.asString() + " expected 1 row in result, got " + rows.size()); 148 | } 149 | if (!key.asString().equals(rows.get(0).getString(0))) { 150 | LOG.fatal("Read key: " + key.asString() + ", got " + rows.get(0).getString(0)); 151 | } 152 | return 1; 153 | } 154 | 155 | private int doBatchedWrite() { 156 | Set keys = new HashSet(); 157 | try { 158 | BatchStatement batch = new BatchStatement(); 159 | PreparedStatement insert = getPreparedInsert(); 160 | for (int i = 0; i < appConfig.batchSize; i++) { 161 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 162 | keys.add(key); 163 | batch.add(getBoundInsertStatement(insert, key)); 164 | } 165 | ResultSet resultSet = getCassandraClient().execute(batch); 166 | LOG.debug("Wrote keys count: " + keys.size() + ", return code: " + resultSet.toString()); 167 | for (Key key : keys) { 168 | getSimpleLoadGenerator().recordWriteSuccess(key); 169 | } 170 | return keys.size(); 171 | } catch (Exception e) { 172 | for (Key key : keys) { 173 | getSimpleLoadGenerator().recordWriteFailure(key); 174 | } 175 | throw e; 176 | } 177 | } 178 | 179 | private int doSingleWrite() { 180 | Key key = null; 181 | try { 182 | key = getSimpleLoadGenerator().getKeyToWrite(); 183 | if (key == null) { 184 | return 0; 185 | } 186 | ResultSet resultSet = 187 | getCassandraClient().execute(getBoundInsertStatement(getPreparedInsert(), key)); 188 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + resultSet.toString()); 189 | getSimpleLoadGenerator().recordWriteSuccess(key); 190 | return 1; 191 | } catch (Exception e) { 192 | getSimpleLoadGenerator().recordWriteFailure(key); 193 | throw e; 194 | } 195 | } 196 | 197 | @Override 198 | public long doWrite(int threadIdx) { 199 | if (appConfig.batchWrite) { 200 | return doBatchedWrite(); 201 | } 202 | return doSingleWrite(); 203 | } 204 | 205 | @Override 206 | public List getWorkloadDescription() { 207 | return Arrays.asList( 208 | "Secondary index on key-value YCQL table. Writes unique keys with an index on values. ", 209 | " Supports a flag to run at app enforced consistency level which is ideal for batch ", 210 | "loading data."); 211 | } 212 | 213 | @Override 214 | public List getWorkloadOptionalArguments() { 215 | return Arrays.asList( 216 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 217 | "--num_reads " + appConfig.numKeysToRead, 218 | "--num_writes " + appConfig.numKeysToWrite, 219 | "--num_threads_read " + appConfig.numReaderThreads, 220 | "--num_threads_write " + appConfig.numWriterThreads, 221 | "--batch_size " + appConfig.batchSize, 222 | "--num_indexes" + appConfig.numIndexes, 223 | "--value_size " + appConfig.valueSize); 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.util.Arrays; 18 | import java.util.List; 19 | 20 | import org.apache.log4j.Logger; 21 | 22 | import com.datastax.driver.core.BoundStatement; 23 | import com.datastax.driver.core.PreparedStatement; 24 | 25 | /** 26 | * This workload writes and reads some random string keys from a CQL server. By default, this app 27 | * inserts a million keys, and reads/updates them indefinitely. 28 | */ 29 | public class CassandraKeyValue extends CassandraKeyValueBase { 30 | 31 | private static final Logger LOG = Logger.getLogger(CassandraKeyValue.class); 32 | // The default table name to create and use for CRUD ops. 33 | private static final String DEFAULT_TABLE_NAME = CassandraKeyValue.class.getSimpleName(); 34 | static { 35 | // The number of keys to read. 36 | appConfig.numKeysToRead = NUM_KEYS_TO_READ_FOR_YSQL_AND_YCQL; 37 | // The number of keys to write. This is the combined total number of inserts and updates. 38 | appConfig.numKeysToWrite = NUM_KEYS_TO_WRITE_FOR_YSQL_AND_YCQL; 39 | // The number of unique keys to write. This determines the number of inserts (as opposed to 40 | // updates). 41 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS_FOR_YSQL_AND_YCQL; 42 | } 43 | private long initialRowCount = 0; 44 | 45 | @Override 46 | public List getCreateTableStatements() { 47 | String create_stmt = String.format( 48 | "CREATE TABLE IF NOT EXISTS %s (k varchar, v blob, primary key (k))", getTableName()); 49 | 50 | if (appConfig.tableTTLSeconds > 0) { 51 | create_stmt += " WITH default_time_to_live = " + appConfig.tableTTLSeconds; 52 | } 53 | create_stmt += ";"; 54 | return Arrays.asList(create_stmt); 55 | } 56 | 57 | @Override 58 | protected String getDefaultTableName() { 59 | return DEFAULT_TABLE_NAME; 60 | } 61 | 62 | protected PreparedStatement getPreparedInsert() { 63 | return getPreparedInsert(String.format("INSERT INTO %s (k, v) VALUES (?, ?);", getTableName())); 64 | } 65 | 66 | @Override 67 | protected BoundStatement bindSelect(String key) { 68 | PreparedStatement prepared_stmt = getPreparedSelect( 69 | String.format("SELECT k, v FROM %s WHERE k = ?;", getTableName()), appConfig.localReads); 70 | return prepared_stmt.bind(key); 71 | } 72 | 73 | @Override 74 | protected BoundStatement bindInsert(String key, ByteBuffer value) { 75 | return getPreparedInsert().bind(key, value); 76 | } 77 | 78 | @Override 79 | public void verifyTotalRowsWritten() throws Exception { 80 | if (appConfig.numKeysToWrite >= 0) { 81 | long actual = getRowCount(); 82 | LOG.info("Total rows count in table " + getTableName() + ": " + actual); 83 | long expected = numKeysWritten.get(); 84 | if (actual != (expected + initialRowCount)) { 85 | LOG.fatal("New rows count does not match! Expected: " + (expected + initialRowCount) + ", actual: " + actual); 86 | } else { 87 | LOG.info("Table row count verified successfully"); 88 | } 89 | } 90 | } 91 | 92 | @Override 93 | public void recordExistingRowCount() throws Exception { 94 | initialRowCount = getRowCount(); 95 | } 96 | 97 | @Override 98 | public List getWorkloadDescription() { 99 | return Arrays.asList( 100 | "Sample key-value app built on Cassandra with concurrent reader and writer threads.", 101 | " Each of these threads operates on a single key-value pair. The number of readers ", 102 | " and writers, the value size, the number of inserts vs updates are configurable. " , 103 | "By default number of reads and writes operations are configured to "+AppBase.appConfig.numKeysToRead+" and "+AppBase.appConfig.numKeysToWrite+" respectively." , 104 | " User can run read/write(both) operations indefinitely by passing -1 to --num_reads or --num_writes or both"); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraKeyValueBase.java: -------------------------------------------------------------------------------- 1 | package com.yugabyte.sample.apps; 2 | 3 | import com.datastax.driver.core.*; 4 | import com.yugabyte.sample.common.SimpleLoadGenerator; 5 | import org.apache.log4j.Logger; 6 | 7 | import java.nio.ByteBuffer; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | /** 12 | * Base class for all workloads that are based on key value tables. 13 | */ 14 | public abstract class CassandraKeyValueBase extends AppBase { 15 | private static final Logger LOG = Logger.getLogger(CassandraKeyValueBase.class); 16 | 17 | // Static initialization of this workload's config. These are good defaults for getting a decent 18 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 19 | // depending on the machine and what resources it has to spare. 20 | static { 21 | // Disable the read-write percentage. 22 | appConfig.readIOPSPercentage = -1; 23 | // Set the read and write threads to 1 each. 24 | appConfig.numReaderThreads = 24; 25 | appConfig.numWriterThreads = 2; 26 | // The number of keys to read. 27 | appConfig.numKeysToRead = 1500000; 28 | // The number of keys to write. This is the combined total number of inserts and updates. 29 | appConfig.numKeysToWrite = 2000000; 30 | // The number of unique keys to write. This determines the number of inserts (as opposed to 31 | // updates). 32 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS; 33 | } 34 | 35 | // The shared prepared select statement for fetching the data. 36 | private static volatile PreparedStatement preparedSelect; 37 | 38 | // The shared prepared statement for inserting into the table. 39 | private static volatile PreparedStatement preparedInsert; 40 | 41 | // Lock for initializing prepared statement objects. 42 | private static final Object prepareInitLock = new Object(); 43 | 44 | public CassandraKeyValueBase() { 45 | buffer = new byte[appConfig.valueSize]; 46 | } 47 | 48 | /** 49 | * Drop the table created by this app. 50 | */ 51 | @Override 52 | public void dropTable() { 53 | dropCassandraTable(getTableName()); 54 | } 55 | 56 | protected PreparedStatement getPreparedSelect(String selectStmt, boolean localReads) { 57 | PreparedStatement preparedSelectLocal = preparedSelect; 58 | if (preparedSelectLocal == null) { 59 | synchronized (prepareInitLock) { 60 | if (preparedSelect == null) { 61 | // Create the prepared statement object. 62 | preparedSelect = getCassandraClient().prepare(selectStmt); 63 | if (localReads) { 64 | LOG.debug("Doing local reads"); 65 | preparedSelect.setConsistencyLevel(ConsistencyLevel.ONE); 66 | } 67 | } 68 | preparedSelectLocal = preparedSelect; 69 | } 70 | } 71 | return preparedSelectLocal; 72 | } 73 | 74 | protected abstract String getDefaultTableName(); 75 | 76 | public String getTableName() { 77 | return appConfig.tableName != null ? appConfig.tableName : getDefaultTableName(); 78 | } 79 | 80 | @Override 81 | public synchronized void resetClients() { 82 | synchronized (prepareInitLock) { 83 | preparedInsert = null; 84 | preparedSelect = null; 85 | } 86 | super.resetClients(); 87 | } 88 | 89 | @Override 90 | public synchronized void destroyClients() { 91 | synchronized (prepareInitLock) { 92 | preparedInsert = null; 93 | preparedSelect = null; 94 | } 95 | super.destroyClients(); 96 | } 97 | 98 | protected abstract BoundStatement bindSelect(String key); 99 | 100 | @Override 101 | public long doRead() { 102 | SimpleLoadGenerator.Key key = getSimpleLoadGenerator().getKeyToRead(); 103 | if (key == null) { 104 | // There are no keys to read yet. 105 | return 0; 106 | } 107 | // Do the read from Cassandra. 108 | // Bind the select statement. 109 | BoundStatement select = bindSelect(key.asString()); 110 | ResultSet rs = getCassandraClient().execute(select); 111 | List rows = rs.all(); 112 | if (rows.size() != 1) { 113 | // If TTL is disabled, turn on correctness validation. 114 | if (appConfig.tableTTLSeconds <= 0) { 115 | LOG.fatal("Read key: " + key.asString() + " expected 1 row in result, got " + rows.size()); 116 | } 117 | return 1; 118 | } 119 | if (appConfig.valueSize == 0) { 120 | ByteBuffer buf = rows.get(0).getBytes(1); 121 | String value = new String(buf.array()); 122 | key.verify(value); 123 | } else { 124 | ByteBuffer value = rows.get(0).getBytes(1); 125 | byte[] bytes = new byte[value.capacity()]; 126 | value.get(bytes); 127 | verifyRandomValue(key, bytes); 128 | } 129 | LOG.debug("Read key: " + key.toString()); 130 | return 1; 131 | } 132 | 133 | protected PreparedStatement getPreparedInsert(String insertStmt) { 134 | PreparedStatement preparedInsertLocal = preparedInsert; 135 | if (preparedInsert == null) { 136 | synchronized (prepareInitLock) { 137 | if (preparedInsert == null) { 138 | // Create the prepared statement object. 139 | preparedInsert = getCassandraClient().prepare(insertStmt); 140 | } 141 | preparedInsertLocal = preparedInsert; 142 | } 143 | } 144 | return preparedInsertLocal; 145 | } 146 | 147 | protected abstract BoundStatement bindInsert(String key, ByteBuffer value); 148 | 149 | @Override 150 | public long doWrite(int threadIdx) { 151 | SimpleLoadGenerator.Key key = getSimpleLoadGenerator().getKeyToWrite(); 152 | if (key == null) { 153 | return 0; 154 | } 155 | 156 | try { 157 | // Do the write to Cassandra. 158 | BoundStatement insert; 159 | if (appConfig.valueSize == 0) { 160 | String value = key.getValueStr(); 161 | insert = bindInsert(key.asString(), ByteBuffer.wrap(value.getBytes())); 162 | } else { 163 | byte[] value = getRandomValue(key); 164 | insert = bindInsert(key.asString(), ByteBuffer.wrap(value)); 165 | } 166 | ResultSet resultSet = getCassandraClient().execute(insert); 167 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + resultSet.toString()); 168 | getSimpleLoadGenerator().recordWriteSuccess(key); 169 | return 1; 170 | } catch (Exception e) { 171 | getSimpleLoadGenerator().recordWriteFailure(key); 172 | throw e; 173 | } 174 | } 175 | 176 | protected long getRowCount() { 177 | ResultSet rs = getCassandraClient().execute("SELECT COUNT(*) FROM " + getTableName()); 178 | List rows = rs.all(); 179 | long actual = rows.get(0).getLong(0); 180 | LOG.info("Found " + actual + " rows in " + getTableName()); 181 | return actual; 182 | } 183 | 184 | @Override 185 | public void appendMessage(StringBuilder sb) { 186 | super.appendMessage(sb); 187 | sb.append("maxWrittenKey: " + getSimpleLoadGenerator().getMaxWrittenKey() + " | "); 188 | sb.append("maxGeneratedKey: " + getSimpleLoadGenerator().getMaxGeneratedKey() + " | "); 189 | } 190 | 191 | public void appendParentMessage(StringBuilder sb) { 192 | super.appendMessage(sb); 193 | } 194 | 195 | @Override 196 | public List getWorkloadDescription() { 197 | return Arrays.asList( 198 | "Sample key-value app built on Cassandra. The app writes out 1M unique string keys", 199 | "each with a string value. There are multiple readers and writers that update these", 200 | "keys and read them indefinitely. Note that the number of reads and writes to", 201 | "perform can be specified as a parameter."); 202 | } 203 | 204 | @Override 205 | public List getWorkloadOptionalArguments() { 206 | return Arrays.asList( 207 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 208 | "--num_reads " + appConfig.numKeysToRead, 209 | "--num_writes " + appConfig.numKeysToWrite, 210 | "--value_size " + appConfig.valueSize, 211 | "--num_threads_read " + appConfig.numReaderThreads, 212 | "--num_threads_write " + appConfig.numWriterThreads, 213 | "--table_ttl_seconds " + appConfig.tableTTLSeconds); 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraRangeKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.nio.ByteBuffer; 17 | import java.util.Arrays; 18 | import java.util.List; 19 | 20 | import com.yugabyte.sample.common.SimpleLoadGenerator; 21 | import org.apache.log4j.Logger; 22 | 23 | import com.datastax.driver.core.BoundStatement; 24 | import com.datastax.driver.core.ConsistencyLevel; 25 | import com.datastax.driver.core.PreparedStatement; 26 | import com.datastax.driver.core.ResultSet; 27 | import com.datastax.driver.core.Row; 28 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 29 | 30 | /** 31 | * This workload writes and reads some random string keys from a CQL server. By default, this app 32 | * inserts a million keys, and reads/updates them indefinitely. 33 | */ 34 | public class CassandraRangeKeyValue extends CassandraKeyValueBase { 35 | // The default table name to create and use for CRUD ops. 36 | private static final String DEFAULT_TABLE_NAME = CassandraRangeKeyValue.class.getSimpleName(); 37 | 38 | @Override 39 | public List getCreateTableStatements() { 40 | String create_stmt = String.format( 41 | "CREATE TABLE IF NOT EXISTS %s (k varchar, r1 varchar, r2 varchar, r3 varchar, v blob, " + 42 | "primary key ((k), r1, r2, r3))", getTableName()); 43 | 44 | if (appConfig.tableTTLSeconds > 0) { 45 | create_stmt += " WITH default_time_to_live = " + appConfig.tableTTLSeconds; 46 | } 47 | create_stmt += ";"; 48 | return Arrays.asList(create_stmt); 49 | } 50 | 51 | @Override 52 | protected String getDefaultTableName() { 53 | return DEFAULT_TABLE_NAME; 54 | } 55 | 56 | @Override 57 | protected BoundStatement bindSelect(String key) { 58 | PreparedStatement prepared_stmt = getPreparedSelect(String.format( 59 | "SELECT k, r1, r2, r3, v FROM %s WHERE k = ? AND r1 = ? AND r2 = ? AND r3 = ?;", 60 | getTableName()), appConfig.localReads); 61 | return prepared_stmt.bind(key, key, key, key); 62 | } 63 | 64 | @Override 65 | protected BoundStatement bindInsert(String key, ByteBuffer value) { 66 | PreparedStatement prepared_stmt = getPreparedInsert(String.format( 67 | "INSERT INTO %s (k, r1, r2, r3, v) VALUES (?, ?, ?, ?, ?);", 68 | getTableName())); 69 | return prepared_stmt.bind(key, key, key, key, value); 70 | } 71 | 72 | @Override 73 | public List getWorkloadDescription() { 74 | return Arrays.asList( 75 | "Sample key-value app built on Cassandra. The app writes out unique keys", 76 | "each has one hash and three range string parts.", 77 | "There are multiple readers and writers that update these", 78 | "keys and read them indefinitely. Note that the number of reads and writes to", 79 | "perform can be specified as a parameter."); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraSecondaryIndex.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.util.Arrays; 17 | import java.util.HashSet; 18 | import java.util.List; 19 | 20 | import org.apache.log4j.Logger; 21 | 22 | import com.datastax.driver.core.BatchStatement; 23 | import com.datastax.driver.core.BoundStatement; 24 | import com.datastax.driver.core.PreparedStatement; 25 | import com.datastax.driver.core.ResultSet; 26 | import com.datastax.driver.core.Row; 27 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 28 | 29 | /** 30 | * This workload writes and reads some random string keys from a YCQL table with a secondary index 31 | * on a non-primary-key column. When it reads a key, it queries the key by its associated value 32 | * which is indexed. 33 | */ 34 | public class CassandraSecondaryIndex extends CassandraKeyValue { 35 | private static final Logger LOG = Logger.getLogger(CassandraSecondaryIndex.class); 36 | 37 | // Static initialization of this workload's config. These are good defaults for getting a decent 38 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 39 | // depending on the machine and what resources it has to spare. 40 | static { 41 | // Disable the read-write percentage. 42 | appConfig.readIOPSPercentage = -1; 43 | // Set the read and write threads to 1 each. 44 | appConfig.numReaderThreads = 24; 45 | appConfig.numWriterThreads = 2; 46 | // The number of keys to read. 47 | appConfig.numKeysToRead = -1; 48 | // The number of keys to write. This is the combined total number of inserts and updates. 49 | appConfig.numKeysToWrite = -1; 50 | // The number of unique keys to write. This determines the number of inserts (as opposed to 51 | // updates). 52 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS; 53 | } 54 | 55 | // The default table name to create and use for CRUD ops. 56 | private static final String DEFAULT_TABLE_NAME = CassandraSecondaryIndex.class.getSimpleName(); 57 | 58 | public CassandraSecondaryIndex() { 59 | } 60 | 61 | @Override 62 | public List getCreateTableStatements() { 63 | return Arrays.asList( 64 | String.format( 65 | "CREATE TABLE IF NOT EXISTS %s (k varchar, v varchar, primary key (k)) %s;", 66 | getTableName(), 67 | appConfig.nonTransactionalIndex ? "" : "WITH transactions = { 'enabled' : true }"), 68 | String.format( 69 | "CREATE INDEX IF NOT EXISTS %sByValue ON %s (v) %s;", getTableName(), getTableName(), 70 | appConfig.nonTransactionalIndex ? 71 | "WITH transactions = { 'enabled' : false, 'consistency_level' : 'user_enforced' }" : 72 | "")); 73 | } 74 | 75 | public String getTableName() { 76 | return appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 77 | } 78 | 79 | private PreparedStatement getPreparedSelect() { 80 | return getPreparedSelect(String.format("SELECT k, v FROM %s WHERE v = ?;", getTableName()), 81 | appConfig.localReads); 82 | } 83 | 84 | @Override 85 | public long doRead() { 86 | Key key = getSimpleLoadGenerator().getKeyToRead(); 87 | if (key == null) { 88 | // There are no keys to read yet. 89 | return 0; 90 | } 91 | // Do the read from Cassandra. 92 | // Bind the select statement. 93 | BoundStatement select = getPreparedSelect().bind(key.getValueStr()); 94 | ResultSet rs = getCassandraClient().execute(select); 95 | List rows = rs.all(); 96 | if (rows.size() != 1) { 97 | LOG.fatal("Read key: " + key.asString() + " expected 1 row in result, got " + rows.size()); 98 | } 99 | if (!key.asString().equals(rows.get(0).getString(0))) { 100 | LOG.fatal("Read key: " + key.asString() + ", got " + rows.get(0).getString(0)); 101 | } 102 | LOG.debug("Read key: " + key.toString()); 103 | return 1; 104 | } 105 | 106 | protected PreparedStatement getPreparedInsert() { 107 | return getPreparedInsert(String.format("INSERT INTO %s (k, v) VALUES (?, ?);", 108 | getTableName())); 109 | } 110 | 111 | @Override 112 | public long doWrite(int threadIdx) { 113 | HashSet keys = new HashSet(); 114 | 115 | try { 116 | if (appConfig.batchWrite) { 117 | BatchStatement batch = new BatchStatement(); 118 | PreparedStatement insert = getPreparedInsert(); 119 | for (int i = 0; i < appConfig.batchSize; i++) { 120 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 121 | keys.add(key); 122 | batch.add(insert.bind(key.asString(), key.getValueStr())); 123 | } 124 | // Do the write to Cassandra. 125 | ResultSet resultSet = getCassandraClient().execute(batch); 126 | LOG.debug("Wrote keys count: " + keys.size() + ", return code: " + resultSet.toString()); 127 | for (Key key : keys) { 128 | getSimpleLoadGenerator().recordWriteSuccess(key); 129 | } 130 | return keys.size(); 131 | } else { 132 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 133 | if (key == null) { 134 | return 0; 135 | } 136 | keys.add(key); 137 | 138 | // Do the write to Cassandra. 139 | BoundStatement insert = getPreparedInsert().bind(key.asString(), key.getValueStr()); 140 | ResultSet resultSet = getCassandraClient().execute(insert); 141 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + resultSet.toString()); 142 | getSimpleLoadGenerator().recordWriteSuccess(key); 143 | return 1; 144 | } 145 | } catch (Exception e) { 146 | for (Key key : keys) { 147 | getSimpleLoadGenerator().recordWriteFailure(key); 148 | } 149 | throw e; 150 | } 151 | } 152 | 153 | @Override 154 | public List getWorkloadDescription() { 155 | return Arrays.asList( 156 | "Secondary index on key-value YCQL table. Writes unique keys with an index on values. Query keys by values ", 157 | "Note that these are strongly consistent, global secondary indexes. Supports a flag to run at", 158 | " app enforced consistency level which is ideal for batch loading data. The number of reads ", 159 | "and writes to perform can be specified as a parameter."); 160 | } 161 | 162 | @Override 163 | public List getWorkloadOptionalArguments() { 164 | return Arrays.asList( 165 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 166 | "--num_reads " + appConfig.numKeysToRead, 167 | "--num_writes " + appConfig.numKeysToWrite, 168 | "--num_threads_read " + appConfig.numReaderThreads, 169 | "--num_threads_write " + appConfig.numWriterThreads, 170 | "--batch_write", 171 | "--batch_size " + appConfig.batchSize); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraTransactionalKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.concurrent.ThreadLocalRandom; 19 | 20 | import org.apache.log4j.Logger; 21 | 22 | import com.datastax.driver.core.BoundStatement; 23 | import com.datastax.driver.core.PreparedStatement; 24 | import com.datastax.driver.core.ResultSet; 25 | import com.datastax.driver.core.Row; 26 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 27 | 28 | /** 29 | * This workload writes and reads some random string keys from a CQL server. By default, this app 30 | * inserts a million keys, and reads/updates them indefinitely. 31 | */ 32 | public class CassandraTransactionalKeyValue extends CassandraKeyValue { 33 | private static final Logger LOG = Logger.getLogger(CassandraTransactionalKeyValue.class); 34 | 35 | // Static initialization of this workload's config. These are good defaults for getting a decent 36 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 37 | // depending on the machine and what resources it has to spare. 38 | static { 39 | // Disable the read-write percentage. 40 | appConfig.readIOPSPercentage = -1; 41 | // Set the read and write threads to 1 each. 42 | appConfig.numReaderThreads = 24; 43 | appConfig.numWriterThreads = 2; 44 | // The number of keys to read. 45 | appConfig.numKeysToRead = -1; 46 | // The number of keys to write. This is the combined total number of inserts and updates. 47 | appConfig.numKeysToWrite = -1; 48 | // The number of unique keys to write. This determines the number of inserts (as opposed to 49 | // updates). 50 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS; 51 | } 52 | 53 | // The default table name to create and use for CRUD ops. 54 | private static final String DEFAULT_TABLE_NAME = 55 | CassandraTransactionalKeyValue.class.getSimpleName(); 56 | 57 | public CassandraTransactionalKeyValue() { 58 | } 59 | 60 | @Override 61 | public List getCreateTableStatements() { 62 | String createStmt = String.format( 63 | "CREATE TABLE IF NOT EXISTS %s (k varchar, v bigint, primary key (k)) " + 64 | "WITH transactions = { 'enabled' : true };", getTableName()); 65 | return Arrays.asList(createStmt); 66 | } 67 | 68 | public String getTableName() { 69 | return appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 70 | } 71 | 72 | private PreparedStatement getPreparedSelect() { 73 | return getPreparedSelect(String.format("SELECT k, v, writetime(v) FROM %s WHERE k in :k;", 74 | getTableName()), 75 | appConfig.localReads); 76 | } 77 | 78 | private void verifyValue(Key key, long value1, long value2) { 79 | long keyNumber = key.asNumber(); 80 | if (keyNumber != value1 + value2) { 81 | LOG.fatal("Value mismatch for key: " + key.toString() + 82 | " != " + value1 + " + " + value2); 83 | } 84 | } 85 | 86 | @Override 87 | public long doRead() { 88 | Key key = getSimpleLoadGenerator().getKeyToRead(); 89 | if (key == null) { 90 | // There are no keys to read yet. 91 | return 0; 92 | } 93 | // Do the read from Cassandra. 94 | // Bind the select statement. 95 | BoundStatement select = getPreparedSelect().bind(Arrays.asList(key.asString() + "_1", 96 | key.asString() + "_2")); 97 | ResultSet rs = getCassandraClient().execute(select); 98 | List rows = rs.all(); 99 | if (rows.size() != 2) { 100 | LOG.fatal("Read key: " + key.asString() + " expected 2 row in result, got " + rows.size()); 101 | return 1; 102 | } 103 | verifyValue(key, rows.get(0).getLong(1), rows.get(1).getLong(1)); 104 | if (rows.get(0).getLong(2) != rows.get(1).getLong(2)) { 105 | LOG.fatal("Writetime mismatch for key: " + key.toString() + ", " + 106 | rows.get(0).getLong(2) + " vs " + rows.get(1).getLong(2)); 107 | } 108 | 109 | LOG.debug("Read key: " + key.toString()); 110 | return 1; 111 | } 112 | 113 | protected PreparedStatement getPreparedInsert() { 114 | return getPreparedInsert(String.format( 115 | "BEGIN TRANSACTION" + 116 | " INSERT INTO %s (k, v) VALUES (:k1, :v1);" + 117 | " INSERT INTO %s (k, v) VALUES (:k2, :v2);" + 118 | "END TRANSACTION;", 119 | getTableName(), 120 | getTableName())); 121 | } 122 | 123 | @Override 124 | public long doWrite(int threadIdx) { 125 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 126 | try { 127 | // Do the write to Cassandra. 128 | BoundStatement insert = null; 129 | long keyNum = key.asNumber(); 130 | /* Generate two numbers that add up to the key, and use them as values */ 131 | long value1 = ThreadLocalRandom.current().nextLong(keyNum + 1); 132 | long value2 = keyNum - value1; 133 | insert = getPreparedInsert() 134 | .bind() 135 | .setString("k1", key.asString() + "_1") 136 | .setString("k2", key.asString() + "_2") 137 | .setLong("v1", value1) 138 | .setLong("v2", value2); 139 | 140 | ResultSet resultSet = getCassandraClient().execute(insert); 141 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + resultSet.toString()); 142 | getSimpleLoadGenerator().recordWriteSuccess(key); 143 | return 1; 144 | } catch (Exception e) { 145 | getSimpleLoadGenerator().recordWriteFailure(key); 146 | throw e; 147 | } 148 | } 149 | 150 | @Override 151 | public List getWorkloadDescription() { 152 | return Arrays.asList( 153 | "Key-value app with multi-row transactions. Each write txn inserts a pair of unique string keys with the same value.", 154 | " There are multiple readers and writers that update these keys and read them in pair ", 155 | "indefinitely. The number of reads and writes to perform can be specified as a parameter."); 156 | } 157 | 158 | @Override 159 | public List getWorkloadOptionalArguments() { 160 | return Arrays.asList( 161 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 162 | "--num_reads " + appConfig.numKeysToRead, 163 | "--num_writes " + appConfig.numKeysToWrite, 164 | "--value_size " + appConfig.valueSize, 165 | "--num_threads_read " + appConfig.numReaderThreads, 166 | "--num_threads_write " + appConfig.numWriterThreads); 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraTransactionalRestartRead.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.concurrent.atomic.AtomicLongArray; 19 | 20 | import com.datastax.driver.core.*; 21 | import com.datastax.driver.core.policies.RoundRobinPolicy; 22 | import com.yugabyte.sample.common.SimpleLoadGenerator; 23 | import org.apache.log4j.Logger; 24 | 25 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 26 | 27 | /** 28 | * This workload writes one key per thread, each time incrementing it's value and storing it 29 | * in array. 30 | * During read with pick random key and check that it's value at least is as big as was stored 31 | * before read started. 32 | */ 33 | public class CassandraTransactionalRestartRead extends CassandraKeyValue { 34 | private static final Logger LOG = Logger.getLogger(CassandraTransactionalRestartRead.class); 35 | // The default table name to create and use for CRUD ops. 36 | private static final String DEFAULT_TABLE_NAME = 37 | CassandraTransactionalRestartRead.class.getSimpleName(); 38 | 39 | // Static initialization of this workload's config. These are good defaults for getting a decent 40 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 41 | // depending on the machine and what resources it has to spare. 42 | static { 43 | // Disable the read-write percentage. 44 | appConfig.readIOPSPercentage = -1; 45 | // Set the read and write threads to 1 each. 46 | appConfig.numReaderThreads = 6; 47 | appConfig.numWriterThreads = 12; 48 | // The number of keys to read. 49 | appConfig.numKeysToRead = -1; 50 | // The number of keys to write. This is the combined total number of inserts and updates. 51 | appConfig.numKeysToWrite = -1; 52 | // The number of unique keys to write. This determines the number of inserts (as opposed to 53 | // updates). 54 | appConfig.numUniqueKeysToWrite = 10; 55 | } 56 | 57 | private AtomicLongArray lastValues = new AtomicLongArray((int)appConfig.numWriterThreads); 58 | 59 | public CassandraTransactionalRestartRead() { 60 | } 61 | 62 | @Override 63 | public List getCreateTableStatements() { 64 | String createStmt = String.format( 65 | "CREATE TABLE IF NOT EXISTS %s (k varchar, v bigint, primary key (k)) " + 66 | "WITH transactions = { 'enabled' : true };", getTableName()); 67 | return Arrays.asList(createStmt); 68 | } 69 | 70 | public String getTableName() { 71 | return appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 72 | } 73 | 74 | private PreparedStatement getPreparedSelect() { 75 | return getPreparedSelect(String.format( 76 | "SELECT k, v FROM %s WHERE k = :k;", 77 | getTableName()), 78 | appConfig.localReads); 79 | } 80 | 81 | @Override 82 | public long doRead() { 83 | Key key = getSimpleLoadGenerator().getKeyToRead(); 84 | if (key == null) { 85 | // There are no keys to read yet. 86 | return 0; 87 | } 88 | // Do the read from Cassandra. 89 | // Bind the select statement. 90 | BoundStatement select = getPreparedSelect().bind(key.asString()); 91 | long minExpected = lastValues.get((int) key.asNumber()); 92 | ResultSet rs = getCassandraClient().execute(select); 93 | List rows = rs.all(); 94 | if (rows.size() != 1) { 95 | LOG.fatal("Read key: " + key.asString() + " expected 1 row in result, got " + 96 | rows.size()); 97 | return 1; 98 | } 99 | long readValue = rows.get(0).getLong(1); 100 | if (readValue < minExpected) { 101 | LOG.fatal("Value too small for key: " + key.toString() + ": " + readValue + " < " + 102 | minExpected); 103 | } 104 | 105 | LOG.debug("Read key: " + key.toString()); 106 | return 1; 107 | } 108 | 109 | protected PreparedStatement getPreparedInsert() { 110 | return getPreparedInsert(String.format( 111 | "BEGIN TRANSACTION" + 112 | " INSERT INTO %s (k, v) VALUES (:k, :v);" + 113 | "END TRANSACTION;", 114 | getTableName())); 115 | } 116 | 117 | @Override 118 | public long doWrite(int threadIdx) { 119 | SimpleLoadGenerator generator = getSimpleLoadGenerator(); 120 | Key key = generator.generateKey(threadIdx); 121 | try { 122 | // Do the write to Cassandra. 123 | long keyNum = key.asNumber(); 124 | long newValue = lastValues.get((int) (keyNum)) + 1; 125 | BoundStatement insert = getPreparedInsert() 126 | .bind() 127 | .setString("k", key.asString()) 128 | .setLong("v", newValue); 129 | 130 | ResultSet resultSet = getCassandraClient().execute(insert); 131 | lastValues.set((int) keyNum, newValue); 132 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + resultSet.toString()); 133 | getSimpleLoadGenerator().recordWriteSuccess(key); 134 | return 1; 135 | } catch (Exception e) { 136 | getSimpleLoadGenerator().recordWriteFailure(key); 137 | throw e; 138 | } 139 | } 140 | 141 | @Override 142 | public List getWorkloadDescription() { 143 | return Arrays.asList( 144 | "This workload writes one key per thread, each time incrementing it's value and " + 145 | "storing it in array.", 146 | "During read with pick random key and check that it's value at least is as big " + 147 | "as was stored before read started."); 148 | } 149 | 150 | @Override 151 | public List getWorkloadOptionalArguments() { 152 | return Arrays.asList( 153 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 154 | "--num_reads " + appConfig.numKeysToRead, 155 | "--num_writes " + appConfig.numKeysToWrite, 156 | "--num_threads_read " + appConfig.numReaderThreads, 157 | "--num_threads_write " + appConfig.numWriterThreads); 158 | } 159 | 160 | protected void setupLoadBalancingPolicy(Cluster.Builder builder) { 161 | builder.withLoadBalancingPolicy(new RoundRobinPolicy()); 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraUniqueSecondaryIndex.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.util.Arrays; 17 | import java.util.List; 18 | 19 | import org.apache.log4j.Logger; 20 | 21 | /** 22 | * This workload writes and reads some random string keys from a YCQL table with a unique secondary 23 | * index on a non-primary-key column. When it reads a key, it queries the key by its associated 24 | * value 25 | * which is indexed. 26 | */ 27 | public class CassandraUniqueSecondaryIndex extends CassandraSecondaryIndex { 28 | private static final Logger LOG = Logger.getLogger(CassandraUniqueSecondaryIndex.class); 29 | 30 | // The default table name to create and use for CRUD ops. 31 | private static final String DEFAULT_TABLE_NAME = 32 | CassandraUniqueSecondaryIndex.class.getSimpleName(); 33 | 34 | public CassandraUniqueSecondaryIndex() { 35 | } 36 | 37 | @Override 38 | public List getCreateTableStatements() { 39 | return Arrays.asList( 40 | String.format( 41 | "CREATE TABLE IF NOT EXISTS %s (k varchar, v varchar, PRIMARY KEY (k)) " + 42 | "WITH transactions = { 'enabled' : true };", getTableName()), 43 | String.format( 44 | "CREATE UNIQUE INDEX IF NOT EXISTS %sByValue ON %s (v);", getTableName(), getTableName())); 45 | } 46 | 47 | @Override 48 | public String getTableName() { 49 | return appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 50 | } 51 | 52 | @Override 53 | public List getWorkloadDescription() { 54 | return Arrays.asList( 55 | "Sample key-value app built on Cassandra. The app writes out unique string keys", 56 | "each with a string value to a YCQL table with a unique index on the value column.", 57 | "There are multiple readers and writers that update these keys and read them", 58 | "indefinitely, with the readers query the keys by the associated values that are", 59 | "indexed. Note that the number of reads and writes to perform can be specified as", 60 | "a parameter."); 61 | } 62 | 63 | @Override 64 | public List getWorkloadOptionalArguments() { 65 | return Arrays.asList( 66 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 67 | "--num_reads " + appConfig.numKeysToRead, 68 | "--num_writes " + appConfig.numKeysToWrite, 69 | "--num_threads_read " + appConfig.numReaderThreads, 70 | "--num_threads_write " + appConfig.numWriterThreads); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/CassandraUserId.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.util.Arrays; 17 | import java.util.Date; 18 | import java.util.List; 19 | 20 | import org.apache.log4j.Logger; 21 | 22 | import com.datastax.driver.core.BoundStatement; 23 | import com.datastax.driver.core.PreparedStatement; 24 | import com.datastax.driver.core.ResultSet; 25 | import com.datastax.driver.core.Row; 26 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 27 | 28 | /** 29 | * This workload writes and reads some random string keys from a CQL server. By default, this app 30 | * inserts a million keys, and reads/updates them indefinitely. 31 | */ 32 | public class CassandraUserId extends CassandraKeyValue { 33 | private static final Logger LOG = Logger.getLogger(CassandraUserId.class); 34 | 35 | // The default table name. 36 | private final String DEFAULT_TABLE_NAME = CassandraUserId.class.getSimpleName(); 37 | 38 | // The newest timestamp we have read. 39 | static Date maxTimestamp = new Date(0); 40 | 41 | // Lock for updating maxTimestamp. 42 | private Object updateMaxTimestampLock = new Object(); 43 | 44 | public String getTableName() { 45 | return appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 46 | } 47 | 48 | /** 49 | * Drop the table created by this app. 50 | */ 51 | @Override 52 | public void dropTable() { 53 | dropCassandraTable(getTableName()); 54 | } 55 | 56 | @Override 57 | public List getCreateTableStatements() { 58 | String create_stmt = String.format("CREATE TABLE IF NOT EXISTS %s " + 59 | "(user_name varchar, password varchar, update_time timestamp, primary key (user_name));", 60 | getTableName()); 61 | return Arrays.asList(create_stmt); 62 | } 63 | 64 | private PreparedStatement getPreparedSelect() { 65 | return getPreparedSelect(String.format( 66 | "SELECT user_name, password, update_time FROM %s WHERE user_name = ?;", getTableName()), 67 | appConfig.localReads); 68 | } 69 | 70 | @Override 71 | public long doRead() { 72 | Key key = getSimpleLoadGenerator().getKeyToRead(); 73 | if (key == null) { 74 | // There are no keys to read yet. 75 | return 0; 76 | } 77 | // Do the read from Cassandra. 78 | // Bind the select statement. 79 | BoundStatement select = getPreparedSelect().bind(key.asString()); 80 | ResultSet rs = getCassandraClient().execute(select); 81 | List rows = rs.all(); 82 | if (rows.size() != 1) { 83 | // If TTL is enabled, turn off correctness validation. 84 | if (appConfig.tableTTLSeconds <= 0) { 85 | LOG.fatal("Read user_name: " + key.asString() + 86 | " expected 1 row in result, got " + rows.size()); 87 | } 88 | return 1; 89 | } 90 | String password = rows.get(0).getString("password"); 91 | if (!password.equals((new StringBuilder(key.asString()).reverse().toString()))) { 92 | LOG.fatal("Invalid password for user_name: " + key.asString() + 93 | ", expected: " + key.asString() + 94 | ", got: " + password); 95 | }; 96 | Date timestamp = rows.get(0).getTimestamp("update_time"); 97 | synchronized (updateMaxTimestampLock) { 98 | if (timestamp.after(maxTimestamp)) { 99 | maxTimestamp.setTime(timestamp.getTime()); 100 | } 101 | } 102 | LOG.debug("Read user_name: " + key.toString()); 103 | return 1; 104 | } 105 | 106 | protected PreparedStatement getPreparedInsert() { 107 | return getPreparedInsert(String.format( 108 | "INSERT INTO %s (user_name, password, update_time) VALUES (?, ?, ?);", getTableName())); 109 | } 110 | 111 | @Override 112 | public long doWrite(int threadIdx) { 113 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 114 | try { 115 | // Do the write to Cassandra. 116 | BoundStatement insert = getPreparedInsert().bind( 117 | key.asString(), (new StringBuilder(key.asString()).reverse().toString()), 118 | new Date(System.currentTimeMillis())); 119 | ResultSet resultSet = getCassandraClient().execute(insert); 120 | LOG.debug("Wrote user_name: " + key.toString() + ", return code: " + resultSet.toString()); 121 | getSimpleLoadGenerator().recordWriteSuccess(key); 122 | return 1; 123 | } catch (Exception e) { 124 | getSimpleLoadGenerator().recordWriteFailure(key); 125 | throw e; 126 | } 127 | } 128 | 129 | @Override 130 | public void appendMessage(StringBuilder sb) { 131 | super.appendParentMessage(sb); 132 | sb.append("maxWrittenKey: " + getSimpleLoadGenerator().getMaxWrittenKey() + " | "); 133 | sb.append("max timestamp: " + maxTimestamp.toString()); 134 | } 135 | 136 | @Override 137 | public List getWorkloadDescription() { 138 | return Arrays.asList( 139 | "Sample user id app built on Cassandra. The app writes out 1M unique user ids", 140 | "each with a string password. There are multiple readers and writers that update", 141 | "these user ids and passwords them indefinitely. Note that the number of reads and", 142 | "writes to perform can be specified as a parameter."); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/RedisKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import org.apache.log4j.Logger; 17 | 18 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 19 | 20 | import java.util.Arrays; 21 | import java.util.List; 22 | 23 | /** 24 | * This workload writes and reads some random string keys from a Redis server. One reader and one 25 | * writer thread thread each is spawned. 26 | */ 27 | public class RedisKeyValue extends AppBase { 28 | private static final Logger LOG = Logger.getLogger(RedisKeyValue.class); 29 | 30 | // Static initialization of this workload's config. 31 | static { 32 | // Disable the read-write percentage. 33 | appConfig.readIOPSPercentage = -1; 34 | // Set the read and write threads to 1 each. 35 | appConfig.numReaderThreads = 32; 36 | appConfig.numWriterThreads = 2; 37 | // Set the number of keys to read and write. 38 | appConfig.numKeysToRead = -1; 39 | appConfig.numKeysToWrite = -1; 40 | appConfig.numUniqueKeysToWrite = AppBase.NUM_UNIQUE_KEYS; 41 | } 42 | 43 | public RedisKeyValue() { 44 | buffer = new byte[appConfig.valueSize]; 45 | } 46 | 47 | @Override 48 | public long doRead() { 49 | Key key = getSimpleLoadGenerator().getKeyToRead(); 50 | if (key == null) { 51 | // There are no keys to read yet. 52 | return 0; 53 | } 54 | if (appConfig.valueSize == 0) { 55 | String value; 56 | if (appConfig.useRedisCluster) { 57 | value = getRedisCluster().get(key.asString()); 58 | } else { 59 | value = getJedisClient().get(key.asString()); 60 | } 61 | key.verify(value); 62 | } else { 63 | byte[] value; 64 | if (appConfig.useRedisCluster) { 65 | value = getRedisCluster().get(key.asString().getBytes()); 66 | } else { 67 | value = getJedisClient().get(key.asString().getBytes()); 68 | } 69 | verifyRandomValue(key, value); 70 | } 71 | LOG.debug("Read key: " + key.toString()); 72 | return 1; 73 | } 74 | 75 | @Override 76 | public long doWrite(int threadIdx) { 77 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 78 | try { 79 | String retVal; 80 | if (appConfig.valueSize == 0) { 81 | String value = key.getValueStr(); 82 | if (appConfig.useRedisCluster) { 83 | retVal = getRedisCluster().set(key.asString(), value); 84 | } else { 85 | retVal = getJedisClient().set(key.asString(), value); 86 | } 87 | } else { 88 | if (appConfig.useRedisCluster) { 89 | retVal = getRedisCluster().set(key.asString().getBytes(), getRandomValue(key)); 90 | } else { 91 | retVal = getJedisClient().set(key.asString().getBytes(), getRandomValue(key)); 92 | } 93 | } 94 | if (retVal == null) { 95 | getSimpleLoadGenerator().recordWriteFailure(key); 96 | return 0; 97 | } 98 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + retVal); 99 | getSimpleLoadGenerator().recordWriteSuccess(key); 100 | return 1; 101 | } catch (Exception e) { 102 | getSimpleLoadGenerator().recordWriteFailure(key); 103 | throw e; 104 | } 105 | } 106 | 107 | @Override 108 | public List getWorkloadDescription() { 109 | return Arrays.asList( 110 | "Sample key-value app built on Redis. The app writes out unique string keys each with a string value.", 111 | " There are multiple readers and writers that insert and update these keys.", 112 | " The number of reads and writes to perform can be specified as a parameter."); 113 | } 114 | 115 | @Override 116 | public List getWorkloadOptionalArguments() { 117 | return Arrays.asList( 118 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 119 | "--num_reads " + appConfig.numKeysToRead, 120 | "--num_writes " + appConfig.numKeysToWrite, 121 | "--num_threads_read " + appConfig.numReaderThreads, 122 | "--num_threads_write " + appConfig.numWriterThreads); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/RedisPipelinedKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import org.apache.log4j.Logger; 17 | 18 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 19 | 20 | import redis.clients.jedis.Response; 21 | import redis.clients.jedis.Jedis; 22 | import redis.clients.jedis.Pipeline; 23 | 24 | import java.util.Arrays; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | import java.util.Vector; 28 | import java.util.concurrent.Callable; 29 | import java.util.concurrent.ExecutorService; 30 | import java.util.concurrent.Executors; 31 | import java.util.concurrent.atomic.AtomicLong; 32 | 33 | /** 34 | * This workload writes and reads some random string keys from a Redis server. One reader and one 35 | * writer thread each is spawned. 36 | */ 37 | public class RedisPipelinedKeyValue extends RedisKeyValue { 38 | private static final Logger LOG = Logger.getLogger(RedisPipelinedKeyValue.class); 39 | 40 | ExecutorService flushPool; 41 | AtomicLong numOpsRespondedThisRound; 42 | AtomicLong numPipelinesCreated; 43 | AtomicLong pendingPipelineBatches; 44 | boolean shouldSleepBeforeNextRound = false; 45 | // Responses for pipelined redis operations. 46 | ArrayList> pipelinedOpResponseCallables; 47 | 48 | public RedisPipelinedKeyValue() { 49 | pipelinedOpResponseCallables = new ArrayList>(appConfig.redisPipelineLength); 50 | flushPool = Executors.newCachedThreadPool(); 51 | numOpsRespondedThisRound = new AtomicLong(); 52 | numPipelinesCreated = new AtomicLong(); 53 | pendingPipelineBatches = new AtomicLong(); 54 | } 55 | 56 | @Override 57 | public long doRead() { 58 | Key key = getSimpleLoadGenerator().getKeyToRead(); 59 | if (key == null) { 60 | // There are no keys to read yet. 61 | return 0; 62 | } 63 | if (appConfig.valueSize == 0) { 64 | Response value = doActualReadString(key); 65 | verifyReadString(key, value); 66 | } else { 67 | Response value = doActualReadBytes(key); 68 | verifyReadBytes(key, value); 69 | } 70 | return flushPipelineIfNecessary(); 71 | } 72 | 73 | protected long flushPipelineIfNecessary() { 74 | if (pipelinedOpResponseCallables.size() % appConfig.redisPipelineLength == 0) { 75 | ArrayList> callbacksToWaitFor = pipelinedOpResponseCallables; 76 | long batch = numPipelinesCreated.addAndGet(1); 77 | Pipeline currPipeline = getRedisPipeline(); 78 | Jedis currJedis = getJedisClient(); 79 | if (appConfig.sleepTime == 0) { // 0 disables running pipelines in parallel. 80 | doActualFlush(currJedis, currPipeline, callbacksToWaitFor, batch, false); 81 | } else { 82 | flushPool.submit(new Runnable() { 83 | public void run() { 84 | doActualFlush(currJedis, currPipeline, callbacksToWaitFor, batch, true); 85 | } 86 | }); 87 | LOG.debug("Submitted a runnable. Now resetting clients"); 88 | resetClients(); 89 | shouldSleepBeforeNextRound = true; 90 | } 91 | pipelinedOpResponseCallables = 92 | new ArrayList>(appConfig.redisPipelineLength); 93 | } 94 | 95 | return numOpsRespondedThisRound.getAndSet(0); 96 | } 97 | 98 | public void performWrite(int threadIdx) { 99 | super.performWrite(threadIdx); 100 | sleepIfNeededBeforeNextBatch(); 101 | } 102 | 103 | public void performRead() { 104 | super.performRead(); 105 | sleepIfNeededBeforeNextBatch(); 106 | } 107 | 108 | private void sleepIfNeededBeforeNextBatch() { 109 | if (shouldSleepBeforeNextRound) { 110 | try { 111 | LOG.debug("Sleeping for " + appConfig.sleepTime + " ms"); 112 | Thread.sleep(appConfig.sleepTime); 113 | } catch (Exception e) { 114 | LOG.error("Caught exception while sleeping ... ", e); 115 | } 116 | shouldSleepBeforeNextRound = false; 117 | } 118 | } 119 | 120 | protected void doActualFlush(Jedis jedis, Pipeline pipeline, 121 | ArrayList> pipelineCallables, 122 | long batch, boolean closeAfterFlush) { 123 | LOG.debug("Flushing pipeline. batch " + batch + " size = " + 124 | pipelineCallables.size() + " pending batches " + 125 | pendingPipelineBatches.addAndGet(1)); 126 | int count = 0; 127 | try { 128 | pipeline.sync(); 129 | for (Callable c : pipelineCallables) { 130 | count += c.call(); 131 | } 132 | if (closeAfterFlush) { 133 | pipeline.close(); 134 | jedis.close(); 135 | } 136 | } catch (Exception e) { 137 | throw new RuntimeException( 138 | "Caught Exception from redis pipeline " + getRedisServerInUse(), e); 139 | } finally { 140 | pipelineCallables.clear(); 141 | numOpsRespondedThisRound.addAndGet(count); 142 | } 143 | LOG.debug("Processed batch " + batch + " count " + count + " responses." 144 | + " pending batches " + pendingPipelineBatches.addAndGet(-1)); 145 | } 146 | 147 | public Response doActualReadString(Key key) { 148 | return getRedisPipeline().get(key.asString()); 149 | } 150 | 151 | public Response doActualReadBytes(Key key) { 152 | return getRedisPipeline().get(key.asString().getBytes()); 153 | } 154 | 155 | private void verifyReadString(final Key key, final Response value) { 156 | pipelinedOpResponseCallables.add(new Callable() { 157 | @Override 158 | public Integer call() throws Exception { 159 | key.verify(value.get()); 160 | return 1; 161 | } 162 | }); 163 | } 164 | 165 | private void verifyReadBytes(final Key key, final Response value) { 166 | pipelinedOpResponseCallables.add(new Callable() { 167 | @Override 168 | public Integer call() throws Exception { 169 | verifyRandomValue(key, value.get()); 170 | return 1; 171 | } 172 | }); 173 | } 174 | 175 | @Override 176 | public long doWrite(int threadIdx) { 177 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 178 | if (key != null) { 179 | Response retVal = doActualWrite(key); 180 | verifyWriteResult(key, retVal); 181 | return flushPipelineIfNecessary(); 182 | } else { 183 | return 0; 184 | } 185 | } 186 | 187 | public Response doActualWrite(Key key) { 188 | Response retVal; 189 | if (appConfig.valueSize == 0) { 190 | String value = key.getValueStr(); 191 | retVal = getRedisPipeline().set(key.asString(), value); 192 | } else { 193 | retVal = getRedisPipeline().set(key.asString().getBytes(), 194 | getRandomValue(key)); 195 | } 196 | return retVal; 197 | } 198 | 199 | private long verifyWriteResult(final Key key, final Response retVal) { 200 | pipelinedOpResponseCallables.add(new Callable() { 201 | @Override 202 | public Integer call() throws Exception { 203 | if (retVal.get() == null) { 204 | getSimpleLoadGenerator().recordWriteFailure(key); 205 | return 0; 206 | } 207 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + 208 | retVal.get()); 209 | getSimpleLoadGenerator().recordWriteSuccess(key); 210 | return 1; 211 | } 212 | }); 213 | return 1; 214 | } 215 | 216 | @Override 217 | public List getWorkloadDescription() { 218 | return Arrays.asList( 219 | "Sample batched key-value app built on Redis. The app reads and writes a batch of key-value pairs.", 220 | " There are multiple readers and writers that insert and update these keys. The number of reads", 221 | " and writes to perform and the batch size can be specified as a parameter."); 222 | } 223 | 224 | @Override 225 | public List getWorkloadOptionalArguments() { 226 | Vector usage = new Vector(super.getWorkloadOptionalArguments()); 227 | usage.add("--pipeline_length " + appConfig.redisPipelineLength); 228 | return usage; 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/RedisYBClientKeyValue.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import org.apache.log4j.Logger; 17 | 18 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 19 | 20 | import java.util.Arrays; 21 | import java.util.List; 22 | 23 | /** 24 | * This workload writes and reads some random string keys from a Redis server. One reader and one 25 | * writer thread thread each is spawned. 26 | * Same as RedisKeyValue but uses the YBJedis client instead of Jedis. 27 | */ 28 | public class RedisYBClientKeyValue extends AppBase { 29 | private static final Logger LOG = Logger.getLogger(RedisKeyValue.class); 30 | 31 | // Static initialization of this workload's config. 32 | static { 33 | // Disable the read-write percentage. 34 | appConfig.readIOPSPercentage = -1; 35 | // Set the read and write threads to 1 each. 36 | appConfig.numReaderThreads = 32; 37 | appConfig.numWriterThreads = 2; 38 | // Set the number of keys to read and write. 39 | appConfig.numKeysToRead = -1; 40 | appConfig.numKeysToWrite = -1; 41 | appConfig.numUniqueKeysToWrite = AppBase.NUM_UNIQUE_KEYS; 42 | } 43 | 44 | public RedisYBClientKeyValue() { 45 | buffer = new byte[appConfig.valueSize]; 46 | } 47 | 48 | @Override 49 | public long doRead() { 50 | Key key = getSimpleLoadGenerator().getKeyToRead(); 51 | if (key == null) { 52 | // There are no keys to read yet. 53 | return 0; 54 | } 55 | if (appConfig.valueSize == 0) { 56 | String value = getYBJedisClient().get(key.asString()); 57 | key.verify(value); 58 | } else { 59 | byte[] value = getYBJedisClient().get(key.asString().getBytes()); 60 | verifyRandomValue(key, value); 61 | } 62 | LOG.debug("Read key: " + key.toString()); 63 | return 1; 64 | } 65 | 66 | @Override 67 | public long doWrite(int threadIdx) { 68 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 69 | try { 70 | String retVal; 71 | if (appConfig.valueSize == 0) { 72 | String value = key.getValueStr(); 73 | retVal = getYBJedisClient().set(key.asString(), value); 74 | } else { 75 | retVal = getYBJedisClient().set(key.asString().getBytes(), getRandomValue(key)); 76 | } 77 | if (retVal == null) { 78 | getSimpleLoadGenerator().recordWriteFailure(key); 79 | return 0; 80 | } 81 | LOG.debug("Wrote key: " + key.toString() + ", return code: " + retVal); 82 | getSimpleLoadGenerator().recordWriteSuccess(key); 83 | return 1; 84 | } catch (Exception e) { 85 | getSimpleLoadGenerator().recordWriteFailure(key); 86 | throw e; 87 | } 88 | } 89 | 90 | @Override 91 | public List getWorkloadDescription() { 92 | return Arrays.asList( 93 | "Sample key-value app built on Redis that uses the YBJedis (multi-node) client instead ", 94 | "of the default (one-node) Jedis one. The app writes out 1M unique string keys each", 95 | "with a string value. There are multiple readers and writers that update these keys", 96 | "and read them indefinitely. Note that the number of reads and writes to perform", 97 | "can be specified as a parameter."); 98 | } 99 | 100 | @Override 101 | public List getWorkloadOptionalArguments() { 102 | return Arrays.asList( 103 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 104 | "--num_reads " + appConfig.numKeysToRead, 105 | "--num_writes " + appConfig.numKeysToWrite, 106 | "--num_threads_read " + appConfig.numReaderThreads, 107 | "--num_threads_write " + appConfig.numWriterThreads); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/SQLAppBase.java: -------------------------------------------------------------------------------- 1 | package com.yugabyte.sample.apps; 2 | 3 | import org.apache.log4j.Logger; 4 | 5 | import java.sql.Connection; 6 | import java.sql.ResultSet; 7 | import java.sql.Statement; 8 | 9 | public class SQLAppBase extends AppBase { 10 | 11 | private Logger LOG = Logger.getLogger(SQLAppBase.class); 12 | 13 | private long initialRowCount = 0; 14 | 15 | protected long getRowCount() throws Exception { 16 | long count = 0; 17 | String table = getTableName(); 18 | Connection connection = getPostgresConnection(); 19 | try { 20 | Statement statement = connection.createStatement(); 21 | String query = "SELECT COUNT(*) FROM " + table; 22 | ResultSet rs = statement.executeQuery(query); 23 | if (rs.next()) { 24 | count = rs.getLong(1); 25 | LOG.info("Row count in table " + table + ": " + count); 26 | } else { 27 | LOG.error("No result received!"); 28 | } 29 | } finally { 30 | if (connection != null) { 31 | connection.close(); 32 | } 33 | } 34 | return count; 35 | } 36 | 37 | @Override 38 | public void verifyTotalRowsWritten() throws Exception { 39 | if (appConfig.numKeysToWrite >= 0) { 40 | String table = getTableName(); 41 | LOG.info("Verifying the inserts on table " + table + " (" + initialRowCount + " rows initially) ..."); 42 | LOG.info("appConfig.numKeysToWrite: " + appConfig.numKeysToWrite + ", numKeysWritten: " + numKeysWritten); 43 | long actual = getRowCount(); 44 | long expected = numKeysWritten.get(); 45 | if (actual != (expected + initialRowCount)) { 46 | LOG.fatal("New rows count does not match! Expected: " + (expected + initialRowCount) + ", actual: " + actual); 47 | } else { 48 | LOG.info("Table row count verified successfully"); 49 | } 50 | } 51 | } 52 | 53 | @Override 54 | public void recordExistingRowCount() throws Exception { 55 | LOG.info("Recording the row count in table " + getTableName() + " if it exists ..."); 56 | initialRowCount = getRowCount(); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/SqlConsistentHashing.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugabyteDB, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.sql.Connection; 17 | import java.sql.PreparedStatement; 18 | import java.sql.Statement; 19 | 20 | import java.util.concurrent.ThreadLocalRandom; 21 | 22 | import org.apache.log4j.Logger; 23 | 24 | /* 25 | * Consistent hashing is useful when you have a dynamic set of nodes and 26 | * you need to send a key-value request to one of the nodes. Consistent 27 | * hashing is great at load balancing without moving too many keys when 28 | * nodes are added or removed. 29 | * 30 | * This app maintains a list of hashes one for each "virtual" node and 31 | * supports two operations: 32 | * a. Config change: Add or remove a node. 33 | * b. Get node: Get the node for a given key. 34 | * 35 | * Config Change Operation: 36 | * 1. At coin flip, choose whether to add or remove a node. 37 | * 2. If adding a node, add a node with a random hash. 38 | * 3. If removing a node, remove a random node. 39 | * 40 | * Get Node Operation: 41 | * 1. Pick a random key. 42 | * 2. Find the node with the smallest hash greater than the key. 43 | * If no such node exists, return the smallest hash node. 44 | */ 45 | public class SqlConsistentHashing extends AppBase { 46 | private static final Logger LOG = Logger.getLogger(SqlConsistentHashing.class); 47 | 48 | // Static initialization of this app's config. 49 | static { 50 | // Use 10 Get Node threads and 10 Config Change threads. 51 | appConfig.readIOPSPercentage = -1; 52 | appConfig.numReaderThreads = 10; 53 | appConfig.numWriterThreads = 10; 54 | // Disable number of keys. 55 | appConfig.numKeysToRead = -1; 56 | appConfig.numKeysToWrite = -1; 57 | // Run the app for 1 minute. 58 | appConfig.runTimeSeconds = 60; 59 | // Report restart read requests metric by default. 60 | appConfig.restartReadsReported = true; 61 | // Avoid load balancing errors. 62 | appConfig.loadBalance = false; 63 | appConfig.disableYBLoadBalancingPolicy = true; 64 | } 65 | 66 | // The default table name to create and use for ops. 67 | private static final String DEFAULT_TABLE_NAME = "consistent_hashing"; 68 | 69 | // Initial number of nodes. 70 | private static final int INITIAL_NODES = 100000; 71 | 72 | // Cache connection and statements. 73 | private Connection connection = null; 74 | private PreparedStatement preparedAddNode = null; 75 | private PreparedStatement preparedRemoveNode = null; 76 | private PreparedStatement preparedGetNode = null; 77 | 78 | public Connection getConnection() { 79 | if (connection == null) { 80 | try { 81 | connection = getPostgresConnection(); 82 | } catch (Exception e) { 83 | LOG.fatal("Failed to create a connection ", e); 84 | } 85 | } 86 | return connection; 87 | } 88 | 89 | public PreparedStatement prepareAddNode() { 90 | if (preparedAddNode == null) { 91 | try { 92 | preparedAddNode = getConnection().prepareStatement( 93 | String.format("INSERT INTO %s (node_hash) VALUES (?)", getTableName())); 94 | } catch (Exception e) { 95 | LOG.fatal("Failed to prepare add node statement ", e); 96 | } 97 | } 98 | return preparedAddNode; 99 | } 100 | 101 | public PreparedStatement prepareRemoveNode() { 102 | if (preparedRemoveNode == null) { 103 | try { 104 | preparedRemoveNode = getConnection().prepareStatement( 105 | String.format("DELETE FROM %s WHERE node_hash =" + 106 | " (SELECT node_hash FROM %s ORDER BY RANDOM() LIMIT 1)", 107 | getTableName(), getTableName())); 108 | } catch (Exception e) { 109 | LOG.fatal("Failed to prepare remove node statement ", e); 110 | } 111 | } 112 | return preparedRemoveNode; 113 | } 114 | 115 | public PreparedStatement prepareGetNode() { 116 | if (preparedGetNode == null) { 117 | try { 118 | preparedGetNode = getConnection().prepareStatement( 119 | String.format("SELECT COALESCE(" + 120 | " (SELECT MIN(node_hash) FROM %s WHERE node_hash > ?)," + 121 | " (SELECT MIN(node_hash) FROM %s)" + 122 | ")", 123 | getTableName(), getTableName())); 124 | } catch (Exception e) { 125 | LOG.fatal("Failed to prepare get node statement ", e); 126 | } 127 | } 128 | return preparedGetNode; 129 | } 130 | 131 | @Override 132 | public void createTablesIfNeeded(TableOp tableOp) throws Exception { 133 | Connection connection = getConnection(); 134 | // Every run should start cleanly. 135 | connection.createStatement().execute( 136 | String.format("DROP TABLE IF EXISTS %s", getTableName())); 137 | LOG.info("Dropping any table(s) left from previous runs if any"); 138 | connection.createStatement().execute(String.format( 139 | "CREATE TABLE %s (node_hash INT) SPLIT INTO 24 TABLETS", 140 | getTableName())); 141 | LOG.info("Created table " + getTableName()); 142 | connection.createStatement().execute(String.format( 143 | "INSERT INTO %s" + 144 | " SELECT (RANDOM() * 1000000000)::INT" + 145 | " FROM generate_series(1, %d)", 146 | getTableName(), INITIAL_NODES)); 147 | LOG.info("Inserted " + INITIAL_NODES + " nodes into " + getTableName()); 148 | } 149 | 150 | @Override 151 | public String getTableName() { 152 | String tableName = appConfig.tableName != null ? 153 | appConfig.tableName : DEFAULT_TABLE_NAME; 154 | return tableName.toLowerCase(); 155 | } 156 | 157 | @Override 158 | public long doRead() { 159 | int key = ThreadLocalRandom.current().nextInt(); 160 | try { 161 | PreparedStatement preparedGetNode = prepareGetNode(); 162 | preparedGetNode.setInt(1, key); 163 | preparedGetNode.executeQuery(); 164 | return 1; 165 | } catch (Exception e) { 166 | // Suppress this error for readability. 167 | if (!e.getMessage().contains("Restart read required")) { 168 | LOG.error("Error retrieving node uuid", e); 169 | } 170 | return 0; 171 | } 172 | } 173 | 174 | @Override 175 | public long doWrite(int threadIdx) { 176 | int coinFlip = ThreadLocalRandom.current().nextInt(2); 177 | if (coinFlip == 0) { 178 | return addNode(); 179 | } else { 180 | return removeNode(); 181 | } 182 | } 183 | 184 | public long addNode() { 185 | try { 186 | int nodeHash = ThreadLocalRandom.current().nextInt(); 187 | PreparedStatement preparedAddNode = prepareAddNode(); 188 | preparedAddNode.setInt(1, nodeHash); 189 | preparedAddNode.executeUpdate(); 190 | return 1; 191 | } catch (Exception e) { 192 | // Suppress this error for readability. 193 | if (!e.getMessage().contains("Restart read required")) { 194 | LOG.error("Error adding a node " + e); 195 | } 196 | return 0; 197 | } 198 | } 199 | 200 | public long removeNode() { 201 | try { 202 | PreparedStatement preparedRemoveNode = prepareRemoveNode(); 203 | preparedRemoveNode.executeUpdate(); 204 | return 1; 205 | } catch (Exception e) { 206 | // Suppress this error for readability. 207 | if (!e.getMessage().contains("Restart read required")) { 208 | LOG.error("Error removing a node " + e); 209 | } 210 | return 0; 211 | } 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/SqlInserts.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | package com.yugabyte.sample.apps; 14 | 15 | import java.sql.*; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | 19 | import org.apache.log4j.Logger; 20 | 21 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 22 | 23 | /** 24 | * This workload writes and reads some random string keys from a postgresql table. 25 | */ 26 | public class SqlInserts extends SQLAppBase { 27 | private static final Logger LOG = Logger.getLogger(SqlInserts.class); 28 | 29 | // Static initialization of this workload's config. These are good defaults for getting a decent 30 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 31 | // depending on the machine and what resources it has to spare. 32 | static { 33 | // Disable the read-write percentage. 34 | appConfig.readIOPSPercentage = -1; 35 | // Set the read and write threads to 2 each. 36 | appConfig.numReaderThreads = 2; 37 | appConfig.numWriterThreads = 2; 38 | // The number of keys to read. 39 | appConfig.numKeysToRead = NUM_KEYS_TO_READ_FOR_YSQL_AND_YCQL; 40 | // The number of keys to write. This is the combined total number of inserts and updates. 41 | appConfig.numKeysToWrite = NUM_KEYS_TO_WRITE_FOR_YSQL_AND_YCQL; 42 | // The number of unique keys to write. This determines the number of inserts (as opposed to 43 | // updates). 44 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS_FOR_YSQL_AND_YCQL; 45 | } 46 | 47 | // The default table name to create and use for CRUD ops. 48 | private static final String DEFAULT_TABLE_NAME = "PostgresqlKeyValue"; 49 | 50 | // The shared prepared select statement for fetching the data. 51 | private volatile Connection selConnection = null; 52 | private volatile PreparedStatement preparedSelect = null; 53 | 54 | // The shared prepared insert statement for inserting the data. 55 | private volatile Connection insConnection = null; 56 | private volatile PreparedStatement preparedInsert = null; 57 | 58 | // Lock for initializing prepared statement objects. 59 | private static final Object prepareInitLock = new Object(); 60 | 61 | public SqlInserts() { 62 | buffer = new byte[appConfig.valueSize]; 63 | } 64 | 65 | /** 66 | * Drop the table created by this app. 67 | */ 68 | @Override 69 | public void dropTable() throws Exception { 70 | try (Connection connection = getPostgresConnection()) { 71 | connection.createStatement().execute("DROP TABLE IF EXISTS " + getTableName()); 72 | LOG.info(String.format("Dropped table: %s", getTableName())); 73 | } 74 | } 75 | 76 | @Override 77 | public void createTablesIfNeeded(TableOp tableOp) throws Exception { 78 | try (Connection connection = getPostgresConnection()) { 79 | 80 | // (Re)Create the table (every run should start cleanly with an empty table). 81 | if (tableOp.equals(TableOp.DropTable)) { 82 | connection.createStatement().execute( 83 | String.format("DROP TABLE IF EXISTS %s", getTableName())); 84 | LOG.info("Dropping any table(s) left from previous runs if any"); 85 | } 86 | connection.createStatement().execute( 87 | String.format("CREATE TABLE IF NOT EXISTS %s (k text PRIMARY KEY, v text)", getTableName())); 88 | LOG.info(String.format("Created table: %s", getTableName())); 89 | if (tableOp.equals(TableOp.TruncateTable)) { 90 | connection.createStatement().execute( 91 | String.format("TRUNCATE TABLE %s", getTableName())); 92 | LOG.info(String.format("Truncated table: %s", getTableName())); 93 | } 94 | } 95 | } 96 | 97 | public String getTableName() { 98 | String tableName = appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 99 | return tableName.toLowerCase(); 100 | } 101 | 102 | private PreparedStatement getPreparedSelect() throws Exception { 103 | if (preparedSelect == null) { 104 | close(selConnection); 105 | selConnection = getPostgresConnection(); 106 | preparedSelect = selConnection.prepareStatement( 107 | String.format("SELECT k, v FROM %s WHERE k = ?;", getTableName())); 108 | } 109 | return preparedSelect; 110 | } 111 | 112 | @Override 113 | public long doRead() { 114 | Key key = getSimpleLoadGenerator().getKeyToRead(); 115 | if (key == null) { 116 | // There are no keys to read yet. 117 | return 0; 118 | } 119 | 120 | try { 121 | PreparedStatement statement = getPreparedSelect(); 122 | statement.setString(1, key.asString()); 123 | try (ResultSet rs = statement.executeQuery()) { 124 | if (!rs.next()) { 125 | LOG.error("Read key: " + key.asString() + " expected 1 row in result, got 0"); 126 | return 0; 127 | } 128 | 129 | if (!key.asString().equals(rs.getString("k"))) { 130 | LOG.error("Read key: " + key.asString() + ", got " + rs.getString("k")); 131 | } 132 | LOG.debug("Read key: " + key.toString()); 133 | 134 | key.verify(rs.getString("v")); 135 | 136 | if (rs.next()) { 137 | LOG.error("Read key: " + key.asString() + " expected 1 row in result, got more"); 138 | return 0; 139 | } 140 | } 141 | } catch (Exception e) { 142 | LOG.info("Failed reading value: " + key.getValueStr(), e); 143 | close(preparedSelect); 144 | preparedSelect = null; 145 | return 0; 146 | } 147 | return 1; 148 | } 149 | 150 | private PreparedStatement getPreparedInsert() throws Exception { 151 | if (preparedInsert == null) { 152 | close(insConnection); 153 | insConnection = getPostgresConnection(); 154 | insConnection.createStatement().execute("set yb_enable_upsert_mode = true"); 155 | preparedInsert = insConnection.prepareStatement( 156 | String.format("INSERT INTO %s (k, v) VALUES (?, ?);", getTableName())); 157 | } 158 | return preparedInsert; 159 | } 160 | 161 | @Override 162 | public long doWrite(int threadIdx) { 163 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 164 | if (key == null) { 165 | return 0; 166 | } 167 | 168 | int result = 0; 169 | try { 170 | PreparedStatement statement = getPreparedInsert(); 171 | // Prefix hashcode to ensure generated keys are random and not sequential. 172 | statement.setString(1, key.asString()); 173 | statement.setString(2, key.getValueStr()); 174 | result = statement.executeUpdate(); 175 | LOG.debug("Wrote key: " + key.asString() + ", " + key.getValueStr() + ", return code: " + 176 | result); 177 | getSimpleLoadGenerator().recordWriteSuccess(key); 178 | } catch (Exception e) { 179 | getSimpleLoadGenerator().recordWriteFailure(key); 180 | LOG.info("Failed writing key: " + key.asString(), e); 181 | close(preparedInsert); 182 | preparedInsert = null; 183 | } 184 | return result; 185 | } 186 | 187 | @Override 188 | public List getWorkloadDescription() { 189 | return Arrays.asList( 190 | "Sample key-value app built on PostgreSQL with concurrent readers and writers. The app inserts unique string keys", 191 | "each with a string value to a postgres table with an index on the value column.", 192 | "There are multiple readers and writers that update these keys and read them", 193 | "for a specified number of operations,default value for read ops is "+AppBase.appConfig.numKeysToRead+" and write ops is "+AppBase.appConfig.numKeysToWrite+", with the readers query the keys by the associated values that are", 194 | "indexed. Note that the number of reads and writes to perform can be specified as", 195 | "a parameter, user can run read/write(both) operations indefinitely by passing -1 to --num_reads or --num_writes or both."); 196 | } 197 | 198 | @Override 199 | public List getWorkloadOptionalArguments() { 200 | return Arrays.asList( 201 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 202 | "--num_reads " + appConfig.numKeysToRead, 203 | "--num_writes " + appConfig.numKeysToWrite, 204 | "--num_threads_read " + appConfig.numReaderThreads, 205 | "--num_threads_write " + appConfig.numWriterThreads, 206 | "--load_balance " + appConfig.loadBalance, 207 | "--topology_keys " + appConfig.topologyKeys, 208 | "--debug_driver " + appConfig.enableDriverDebug); 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/SqlSecondaryIndex.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | package com.yugabyte.sample.apps; 14 | 15 | import java.sql.Connection; 16 | import java.sql.PreparedStatement; 17 | import java.sql.ResultSet; 18 | import java.util.Arrays; 19 | import java.util.List; 20 | 21 | import org.apache.log4j.Logger; 22 | 23 | import com.yugabyte.sample.apps.AppBase.TableOp; 24 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 25 | 26 | /** 27 | * This workload writes and reads some random string keys from a postgresql table. 28 | */ 29 | public class SqlSecondaryIndex extends AppBase { 30 | private static final Logger LOG = Logger.getLogger(SqlSecondaryIndex.class); 31 | 32 | // Static initialization of this workload's config. These are good defaults for getting a decent 33 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 34 | // depending on the machine and what resources it has to spare. 35 | static { 36 | // Disable the read-write percentage. 37 | appConfig.readIOPSPercentage = -1; 38 | // Set the read and write threads to 1 each. 39 | appConfig.numReaderThreads = 1; 40 | appConfig.numWriterThreads = 1; 41 | // The number of keys to read. 42 | appConfig.numKeysToRead = -1; 43 | // The number of keys to write. This is the combined total number of inserts and updates. 44 | appConfig.numKeysToWrite = -1; 45 | // The number of unique keys to write. This determines the number of inserts (as opposed to 46 | // updates). 47 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS; 48 | } 49 | 50 | // The default table name to create and use for CRUD ops. 51 | private static final String DEFAULT_TABLE_NAME = "SqlSecondaryIndex"; 52 | 53 | // The shared prepared select statement for fetching the data. 54 | private volatile PreparedStatement preparedSelect = null; 55 | 56 | // The shared prepared insert statement for inserting the data. 57 | private volatile PreparedStatement preparedInsert = null; 58 | 59 | // Lock for initializing prepared statement objects. 60 | private static final Object prepareInitLock = new Object(); 61 | 62 | public SqlSecondaryIndex() { 63 | buffer = new byte[appConfig.valueSize]; 64 | } 65 | 66 | /** 67 | * Drop the table created by this app. 68 | */ 69 | @Override 70 | public void dropTable() throws Exception { 71 | try (Connection connection = getPostgresConnection()) { 72 | connection.createStatement().execute("DROP TABLE IF EXISTS " + getTableName()); 73 | LOG.info(String.format("Dropped table: %s", getTableName())); 74 | } 75 | } 76 | 77 | @Override 78 | public void createTablesIfNeeded(TableOp tableOp) throws Exception { 79 | try (Connection connection = getPostgresConnection()) { 80 | 81 | // (Re)Create the table (every run should start cleanly with an empty table). 82 | if (tableOp.equals(TableOp.DropTable)) { 83 | connection.createStatement().execute( 84 | String.format("DROP TABLE IF EXISTS %s", getTableName())); 85 | LOG.info("Dropping table(s) left from previous runs if any"); 86 | } 87 | connection.createStatement().executeUpdate( 88 | String.format("CREATE TABLE IF NOT EXISTS %s (k text PRIMARY KEY, v text);", getTableName())); 89 | LOG.info(String.format("Created table: %s", getTableName())); 90 | 91 | if (tableOp.equals(TableOp.TruncateTable)) { 92 | connection.createStatement().execute( 93 | String.format("TRUNCATE TABLE %s", getTableName())); 94 | LOG.info(String.format("Truncated table: %s", getTableName())); 95 | } 96 | 97 | // Create an index on the table. 98 | connection.createStatement().executeUpdate( 99 | String.format("CREATE INDEX IF NOT EXISTS %s_index ON %s(v);", 100 | getTableName(), getTableName())); 101 | LOG.info(String.format("Created index on table: %s", getTableName())); 102 | } 103 | } 104 | 105 | public String getTableName() { 106 | String tableName = appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 107 | return tableName.toLowerCase(); 108 | } 109 | 110 | private PreparedStatement getPreparedSelect() throws Exception { 111 | if (preparedSelect == null) { 112 | preparedSelect = getPostgresConnection().prepareStatement( 113 | String.format("SELECT k, v FROM %s WHERE v = ?;", getTableName())); 114 | } 115 | return preparedSelect; 116 | } 117 | 118 | @Override 119 | public long doRead() { 120 | Key key = getSimpleLoadGenerator().getKeyToRead(); 121 | if (key == null) { 122 | // There are no keys to read yet. 123 | return 0; 124 | } 125 | 126 | try { 127 | PreparedStatement statement = getPreparedSelect(); 128 | statement.setString(1, key.getValueStr()); 129 | try (ResultSet rs = statement.executeQuery()) { 130 | if (!rs.next()) { 131 | LOG.error("Read key: " + key.asString() + " expected 1 row in result, got 0"); 132 | return 0; 133 | } 134 | 135 | if (!key.asString().equals(rs.getString("k"))) { 136 | LOG.error("Read key: " + key.asString() + ", got " + rs.getString("k")); 137 | } 138 | LOG.debug("Read key: " + key.toString()); 139 | 140 | key.verify(rs.getString("v")); 141 | 142 | if (rs.next()) { 143 | LOG.error("Read key: " + key.asString() + " expected 1 row in result, got more"); 144 | return 0; 145 | } 146 | } 147 | } catch (Exception e) { 148 | LOG.info("Failed reading value: " + key.getValueStr(), e); 149 | preparedSelect = null; 150 | return 0; 151 | } 152 | return 1; 153 | } 154 | 155 | private PreparedStatement getPreparedInsert() throws Exception { 156 | if (preparedInsert == null) { 157 | preparedInsert = getPostgresConnection().prepareStatement( 158 | String.format("INSERT INTO %s (k, v) VALUES (?, ?);", getTableName())); 159 | } 160 | return preparedInsert; 161 | } 162 | 163 | @Override 164 | public long doWrite(int threadIdx) { 165 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 166 | if (key == null) { 167 | return 0; 168 | } 169 | 170 | int result = 0; 171 | try { 172 | PreparedStatement statement = getPreparedInsert(); 173 | // Prefix hashcode to ensure generated keys are random and not sequential. 174 | statement.setString(1, key.asString()); 175 | statement.setString(2, key.getValueStr()); 176 | result = statement.executeUpdate(); 177 | LOG.debug("Wrote key: " + key.asString() + ", " + key.getValueStr() + ", return code: " + 178 | result); 179 | getSimpleLoadGenerator().recordWriteSuccess(key); 180 | } catch (Exception e) { 181 | getSimpleLoadGenerator().recordWriteFailure(key); 182 | LOG.info("Failed writing key: " + key.asString(), e); 183 | preparedInsert = null; 184 | } 185 | return result; 186 | } 187 | 188 | @Override 189 | public List getWorkloadDescription() { 190 | return Arrays.asList( 191 | "Sample key-value app built on postgresql. The app writes out unique string keys", 192 | "each with a string value to a postgres table with an index on the value column.", 193 | "There are multiple readers and writers that update these keys and read them", 194 | "indefinitely, with the readers query the keys by the associated values that are", 195 | "indexed. Note that the number of reads and writes to perform can be specified as", 196 | "a parameter."); 197 | } 198 | 199 | @Override 200 | public List getWorkloadOptionalArguments() { 201 | return Arrays.asList( 202 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 203 | "--num_reads " + appConfig.numKeysToRead, 204 | "--num_writes " + appConfig.numKeysToWrite, 205 | "--num_threads_read " + appConfig.numReaderThreads, 206 | "--num_threads_write " + appConfig.numWriterThreads, 207 | "--load_balance " + appConfig.loadBalance, 208 | "--topology_keys " + appConfig.topologyKeys, 209 | "--debug_driver " + appConfig.enableDriverDebug); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/SqlSnapshotTxns.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | package com.yugabyte.sample.apps; 14 | 15 | import java.sql.Connection; 16 | import java.sql.PreparedStatement; 17 | import java.sql.ResultSet; 18 | import java.util.Arrays; 19 | import java.util.List; 20 | 21 | import org.apache.log4j.Logger; 22 | 23 | import com.yugabyte.sample.apps.AppBase.TableOp; 24 | import com.yugabyte.sample.common.SimpleLoadGenerator.Key; 25 | 26 | import static java.sql.Connection.TRANSACTION_REPEATABLE_READ; 27 | 28 | /** 29 | * This workload writes and reads some random string keys from a postgresql table. 30 | */ 31 | public class SqlSnapshotTxns extends AppBase { 32 | private static final Logger LOG = Logger.getLogger(SqlSnapshotTxns.class); 33 | 34 | // Static initialization of this workload's config. These are good defaults for getting a decent 35 | // read dominated workload on a reasonably powered machine. Exact IOPS will of course vary 36 | // depending on the machine and what resources it has to spare. 37 | static { 38 | // Disable the read-write percentage. 39 | appConfig.readIOPSPercentage = -1; 40 | // Set the read and write threads to 1 each. 41 | appConfig.numReaderThreads = 1; 42 | appConfig.numWriterThreads = 1; 43 | // The number of keys to read. 44 | appConfig.numKeysToRead = -1; 45 | // The number of keys to write. This is the combined total number of inserts and updates. 46 | appConfig.numKeysToWrite = -1; 47 | // The number of unique keys to write. This determines the number of inserts (as opposed to 48 | // updates). 49 | appConfig.numUniqueKeysToWrite = NUM_UNIQUE_KEYS; 50 | } 51 | 52 | // The default table name to create and use for CRUD ops. 53 | private static final String DEFAULT_TABLE_NAME = "SqlSnapshotTxns"; 54 | 55 | // The shared prepared select statement for fetching the data. 56 | private volatile PreparedStatement preparedSelect = null; 57 | 58 | // The shared prepared insert statement for inserting the data. 59 | private volatile PreparedStatement preparedInsert = null; 60 | 61 | // Lock for initializing prepared statement objects. 62 | private static final Object prepareInitLock = new Object(); 63 | 64 | public SqlSnapshotTxns() { 65 | buffer = new byte[appConfig.valueSize]; 66 | } 67 | 68 | /** 69 | * Drop the table created by this app. 70 | */ 71 | @Override 72 | public void dropTable() throws Exception { 73 | try (Connection connection = getPostgresConnection()) { 74 | connection.createStatement().execute("DROP TABLE IF EXISTS " + getTableName()); 75 | LOG.info(String.format("Dropped table: %s", getTableName())); 76 | } 77 | } 78 | 79 | @Override 80 | public void createTablesIfNeeded(TableOp tableOp) throws Exception { 81 | try (Connection connection = getPostgresConnection()) { 82 | connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); 83 | connection.setAutoCommit(false); 84 | 85 | // (Re)Create the table (every run should start cleanly with an empty table). 86 | // connection.createStatement().execute( 87 | // String.format("DROP TABLE IF EXISTS %s", getTableName())); 88 | LOG.info("Dropping table(s) left from previous runs if any"); 89 | connection.createStatement().executeUpdate( 90 | String.format("CREATE TABLE IF NOT EXISTS %s (k text PRIMARY KEY, v text);", getTableName())); 91 | LOG.info(String.format("Created table: %s", getTableName())); 92 | } 93 | } 94 | 95 | public String getTableName() { 96 | String tableName = appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 97 | return tableName.toLowerCase(); 98 | } 99 | 100 | private PreparedStatement getPreparedSelect() throws Exception { 101 | if (preparedSelect == null) { 102 | preparedSelect = getPostgresConnection().prepareStatement( 103 | String.format("SELECT k, v FROM %s WHERE k = ?;", getTableName())); 104 | } 105 | return preparedSelect; 106 | } 107 | 108 | @Override 109 | public long doRead() { 110 | Key key = getSimpleLoadGenerator().getKeyToRead(); 111 | if (key == null) { 112 | // There are no keys to read yet. 113 | return 0; 114 | } 115 | 116 | try { 117 | PreparedStatement statement = getPreparedSelect(); 118 | statement.setString(1, key.asString()); 119 | try (ResultSet rs = statement.executeQuery()) { 120 | if (!rs.next()) { 121 | LOG.error("Read key: " + key.asString() + " expected 1 row in result, got 0"); 122 | return 0; 123 | } 124 | 125 | if (!key.asString().equals(rs.getString("k"))) { 126 | LOG.error("Read key: " + key.asString() + ", got " + rs.getString("k")); 127 | } 128 | LOG.debug("Read key: " + key.toString()); 129 | 130 | key.verify(rs.getString("v")); 131 | 132 | if (rs.next()) { 133 | LOG.error("Read key: " + key.asString() + " expected 1 row in result, got more"); 134 | return 0; 135 | } 136 | } 137 | } catch (Exception e) { 138 | LOG.info("Failed reading value: " + key.getValueStr(), e); 139 | preparedSelect = null; 140 | return 0; 141 | } 142 | return 1; 143 | } 144 | 145 | private PreparedStatement getPreparedInsert() throws Exception { 146 | String stmt = "BEGIN TRANSACTION;" + 147 | String.format("INSERT INTO %s (k, v) VALUES (?, ?);", getTableName()) + 148 | String.format("INSERT INTO %s (k, v) VALUES (?, ?);", getTableName()) + 149 | "COMMIT;"; 150 | if (preparedInsert == null) { 151 | Connection connection = getPostgresConnection(); 152 | connection.createStatement().execute("set yb_enable_upsert_mode = true"); 153 | preparedInsert = connection.prepareStatement(stmt); 154 | } 155 | return preparedInsert; 156 | } 157 | 158 | @Override 159 | public long doWrite(int threadIdx) { 160 | Key key = getSimpleLoadGenerator().getKeyToWrite(); 161 | if (key == null) { 162 | return 0; 163 | } 164 | 165 | int result = 0; 166 | try { 167 | PreparedStatement statement = getPreparedInsert(); 168 | // Prefix hashcode to ensure generated keys are random and not sequential. 169 | statement.setString(1, key.asString()); 170 | statement.setString(2, key.getValueStr()); 171 | statement.setString(3, key.asString() + "-copy"); 172 | statement.setString(4, key.getValueStr()); 173 | result = statement.executeUpdate(); 174 | LOG.debug("Wrote key: " + key.asString() + ", " + key.getValueStr() + ", return code: " + 175 | result); 176 | getSimpleLoadGenerator().recordWriteSuccess(key); 177 | return 1; 178 | } catch (Exception e) { 179 | getSimpleLoadGenerator().recordWriteFailure(key); 180 | LOG.info("Failed writing key: " + key.asString(), e); 181 | preparedInsert = null; 182 | } 183 | return 0; 184 | } 185 | 186 | @Override 187 | public List getWorkloadDescription() { 188 | return Arrays.asList( 189 | "Sample key-value app built on postgresql. The app writes out unique string keys", 190 | "each with a string value to a postgres table with an index on the value column.", 191 | "There are multiple readers and writers that update these keys and read them", 192 | "indefinitely, with the readers query the keys by the associated values that are", 193 | "indexed. Note that the number of reads and writes to perform can be specified as", 194 | "a parameter."); 195 | } 196 | 197 | @Override 198 | public List getWorkloadOptionalArguments() { 199 | return Arrays.asList( 200 | "--num_unique_keys " + appConfig.numUniqueKeysToWrite, 201 | "--num_reads " + appConfig.numKeysToRead, 202 | "--num_writes " + appConfig.numKeysToWrite, 203 | "--num_threads_read " + appConfig.numReaderThreads, 204 | "--num_threads_write " + appConfig.numWriterThreads, 205 | "--load_balance " + appConfig.loadBalance, 206 | "--topology_keys " + appConfig.topologyKeys, 207 | "--debug_driver " + appConfig.enableDriverDebug); 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/SqlWarehouseStock.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugabyteDB, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.apps; 15 | 16 | import java.sql.Connection; 17 | import java.sql.PreparedStatement; 18 | import java.sql.ResultSet; 19 | 20 | import java.util.concurrent.atomic.AtomicLong; 21 | import java.util.concurrent.ThreadLocalRandom; 22 | 23 | import org.apache.log4j.Logger; 24 | 25 | /* 26 | * E-commerce is another important usecase for OLTP databases. TPC-C is an 27 | * example benchmark that simulates a simple e-commerce workload. 28 | * 29 | * Simulate restocking items in a warehouse. Customers 30 | * continously place orders that deplete the stock of items. The warehouse 31 | * restocks items whose stock has fallen below 10 items by adding a 32 | * 100 items. The warehouse has 100 items of each type initially. 33 | * There are 1000 types of items. 34 | * 35 | * The restocking operation does a full table scan to find the items that 36 | * have a low stock. This leads to read restart errors. This app helps 37 | * understand whether the new clockbound clock helps improve the 38 | * performance of this workload. 39 | * 40 | * Database Configuration: 41 | * configure with wallclock and compare the metrics with 42 | * a clockbound clock configuration. 43 | * 44 | * Setup: 45 | * 1. Create a warehouse_stock TABLE with columns (item_id INT, stock INT). 46 | * 2. Insert 100 items with item_id 0 to 999 initialized to 100 stock. 47 | * 48 | * Workload: 49 | * There are two operations in this workload. 50 | * a. NewOrder: Decrement the stock of a random item. 51 | * b. Restock: Restocks items whose stock has fallen below 10 items by 52 | * adding 100 items. 53 | * 54 | * NewOrder Operation: 55 | * 1. Pick a random item_id. 56 | * 2. Decrement the stock of the item only when there is enough stock. 57 | * 58 | * Restock Operation: 59 | * 1. Scan the table, restock by 100 when below 10. 60 | */ 61 | public class SqlWarehouseStock extends AppBase { 62 | private static final Logger LOG = Logger.getLogger(SqlWarehouseStock.class); 63 | 64 | // Static initialization of this app's config. 65 | static { 66 | // Use 1 Restock thread and 100 NewOrder threads. 67 | appConfig.readIOPSPercentage = -1; 68 | appConfig.numReaderThreads = 1; 69 | appConfig.numWriterThreads = 100; 70 | // Disable number of keys. 71 | appConfig.numKeysToRead = -1; 72 | appConfig.numKeysToWrite = -1; 73 | // Run the app for 1 minute. 74 | appConfig.runTimeSeconds = 60; 75 | // Report restart read requests metric by default. 76 | appConfig.restartReadsReported = true; 77 | // Avoid load balancing errors. 78 | appConfig.loadBalance = false; 79 | appConfig.disableYBLoadBalancingPolicy = true; 80 | } 81 | 82 | // The default table name to create and use for ops. 83 | private static final String DEFAULT_TABLE_NAME = "warehouse_stock"; 84 | 85 | // The number of items in the warehouse. 86 | private static final int NUM_ITEMS = 1000; 87 | 88 | // The stock level below which restocking is needed. 89 | private static final int RESTOCK_THRESHOLD = 10; 90 | 91 | // The amount to restock by. 92 | private static final int RESTOCK_AMOUNT = 100; 93 | 94 | // Initial stock. 95 | private static final int INITIAL_STOCK = 100; 96 | 97 | // Shared counter to store the number of restocks required. 98 | private static final AtomicLong numRestocksRequired = new AtomicLong(0); 99 | 100 | // Shared counter to store the number of stale reads. 101 | private static final AtomicLong numStaleReads = new AtomicLong(0); 102 | 103 | // Cache connection and statements. 104 | private Connection connection = null; 105 | private PreparedStatement preparedRestock = null; 106 | private PreparedStatement preparedNewOrder = null; 107 | 108 | public Connection getConnection() { 109 | if (connection == null) { 110 | try { 111 | connection = getPostgresConnection(); 112 | } catch (Exception e) { 113 | LOG.fatal("Failed to create a connection ", e); 114 | } 115 | } 116 | return connection; 117 | } 118 | 119 | public PreparedStatement getPreparedRestock() { 120 | if (preparedRestock == null) { 121 | try { 122 | preparedRestock = getConnection().prepareStatement( 123 | String.format("UPDATE %s SET stock = stock + %d WHERE stock < %d", 124 | getTableName(), RESTOCK_AMOUNT, RESTOCK_THRESHOLD)); 125 | } catch (Exception e) { 126 | LOG.fatal("Failed to prepare statement: UPDATE " + getTableName(), e); 127 | } 128 | } 129 | return preparedRestock; 130 | } 131 | 132 | public PreparedStatement getPreparedNewOrder() { 133 | if (preparedNewOrder == null) { 134 | try { 135 | preparedNewOrder = getConnection().prepareStatement( 136 | String.format("UPDATE %s SET stock = stock - 1" + 137 | " WHERE item_id = ? AND stock > 0" + 138 | " RETURNING stock", 139 | getTableName())); 140 | } catch (Exception e) { 141 | LOG.fatal("Failed to prepare statement: UPDATE " + getTableName(), e); 142 | } 143 | } 144 | return preparedNewOrder; 145 | } 146 | 147 | @Override 148 | public void createTablesIfNeeded(TableOp tableOp) throws Exception { 149 | Connection connection = getConnection(); 150 | // Every run should start cleanly. 151 | connection.createStatement().execute( 152 | String.format("DROP TABLE IF EXISTS %s", getTableName())); 153 | LOG.info("Dropping any table(s) left from previous runs if any"); 154 | connection.createStatement().execute(String.format( 155 | "CREATE TABLE %s (item_id INT PRIMARY KEY, stock INT)" + 156 | " SPLIT INTO 24 TABLETS", 157 | getTableName())); 158 | LOG.info(String.format("Created table: %s", getTableName())); 159 | int numRows = connection.createStatement().executeUpdate(String.format( 160 | "INSERT INTO %s SELECT GENERATE_SERIES(0, %d-1), %d", 161 | getTableName(), NUM_ITEMS, INITIAL_STOCK)); 162 | LOG.info(String.format( 163 | "Inserted %d rows into %s", numRows, getTableName())); 164 | } 165 | 166 | @Override 167 | public String getTableName() { 168 | String tableName = appConfig.tableName != null ? 169 | appConfig.tableName : DEFAULT_TABLE_NAME; 170 | return tableName.toLowerCase(); 171 | } 172 | 173 | // Executes the Restock operation. 174 | @Override 175 | public long doRead() { 176 | try { 177 | long restocksRequired = numRestocksRequired.get(); 178 | PreparedStatement preparedRestock = getPreparedRestock(); 179 | int numRestocked = preparedRestock.executeUpdate(); 180 | if (numRestocked < restocksRequired) { 181 | numStaleReads.incrementAndGet(); 182 | } 183 | numRestocksRequired.addAndGet(-numRestocked); 184 | return 1; 185 | } catch (Exception e) { 186 | // Suppress this error for readability. 187 | if (!e.getMessage().contains("Restart read required")) { 188 | LOG.error("Error restocking ", e); 189 | } 190 | return 0; 191 | } 192 | } 193 | 194 | // Executes a NewOrder operation. 195 | @Override 196 | public long doWrite(int threadIdx) { 197 | try { 198 | int itemId = ThreadLocalRandom.current().nextInt(NUM_ITEMS); 199 | PreparedStatement preparedNewOrder = getPreparedNewOrder(); 200 | preparedNewOrder.setInt(1, itemId); 201 | ResultSet rs = preparedNewOrder.executeQuery(); 202 | if (!rs.next()) { 203 | // No rows updated, return 0. 204 | return 0; 205 | } 206 | int stock = rs.getInt(1); 207 | if (stock < RESTOCK_THRESHOLD) { 208 | numRestocksRequired.incrementAndGet(); 209 | } 210 | return 1; 211 | } catch (Exception e) { 212 | // Suppress this error for readability. 213 | if (!e.getMessage().contains("Restart read required")) { 214 | LOG.error("Error creating a new order ", e); 215 | } 216 | return 0; 217 | } 218 | } 219 | 220 | /* 221 | * Appends the number of stale reads to the metrics output. 222 | */ 223 | @Override 224 | public void appendMessage(StringBuilder sb) { 225 | sb.append("Stale reads: ").append(numStaleReads.get()).append(" total reads | "); 226 | super.appendMessage(sb); 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/apps/anomalies/SqlQueryLatencyIncrease.java: -------------------------------------------------------------------------------- 1 | package com.yugabyte.sample.apps.anomalies; 2 | 3 | import com.yugabyte.sample.apps.SQLAppBase; 4 | import java.sql.Connection; 5 | import java.sql.PreparedStatement; 6 | import java.sql.Statement; 7 | import org.apache.log4j.Logger; 8 | 9 | public class SqlQueryLatencyIncrease extends SQLAppBase { 10 | 11 | private static final Logger LOG = Logger.getLogger(SqlQueryLatencyIncrease.class); 12 | 13 | private volatile String selectSeqScan; 14 | private volatile Connection selConnection = null; 15 | private volatile Statement statement = null; 16 | private volatile String selectIndex = null; 17 | private long seqScanStartTime = 0; 18 | private long seqScanEndTime = 0; 19 | 20 | private long lastReadLogTime = 0; 21 | 22 | private static final String DEFAULT_TABLE_NAME = "PostgresqlKeyValue"; 23 | 24 | public SqlQueryLatencyIncrease() { 25 | } 26 | 27 | @Override 28 | public void dropTable() throws Exception { 29 | try (Connection connection = getPostgresConnection()) { 30 | connection.createStatement().execute("DROP TABLE IF EXISTS " + getTableName()); 31 | LOG.info(String.format("Dropped table: %s", getTableName())); 32 | } 33 | } 34 | 35 | @Override 36 | public String getTableName() { 37 | String tableName = appConfig.tableName != null ? appConfig.tableName : DEFAULT_TABLE_NAME; 38 | return tableName.toLowerCase(); 39 | } 40 | 41 | @Override 42 | public void createTablesIfNeeded(TableOp tableOp) throws Exception { 43 | try (Connection connection = getPostgresConnection()) { 44 | Statement s = connection.createStatement(); 45 | s.addBatch( 46 | String.format( 47 | "CREATE TABLE IF NOT EXISTS %s (k int PRIMARY KEY, v1 int, v2 int) SPLIT INTO 3 TABLETS;", 48 | getTableName())); 49 | 50 | s.addBatch( 51 | String.format( 52 | "CREATE INDEX IF NOT EXISTS %s_v1_v2 ON %s(v1,v2);", getTableName(), getTableName())); 53 | s.executeBatch(); 54 | LOG.info("Created table and index " + getTableName()); 55 | 56 | if (tableOp.equals(TableOp.TruncateTable)) { 57 | Statement t = connection.createStatement(); 58 | t.execute(String.format("TRUNCATE TABLE %s;", getTableName())); 59 | LOG.info("Truncated table " + getTableName()); 60 | } 61 | 62 | s = connection.createStatement(); 63 | long rowsToInsert = appConfig.numKeysToWrite; 64 | long maxValue = Math.min(rowsToInsert / 100, 100); 65 | String createIndexStatement = 66 | String.format( 67 | "INSERT INTO %s SELECT i, i / " + maxValue + ", i %% " + maxValue + " FROM" 68 | + " generate_series(1, " + rowsToInsert + ") AS i WHERE (SELECT COUNT(*) from %s) = 0;", 69 | getTableName(), getTableName()); 70 | LOG.info("Inserting data: " + createIndexStatement); 71 | s.execute(createIndexStatement); 72 | 73 | LOG.info("Inserted " + rowsToInsert + " rows into table " + getTableName()); 74 | 75 | s = connection.createStatement(); 76 | s.execute(String.format("ANALYZE %s;", getTableName())); 77 | LOG.info("Analyze. Executing"); 78 | } 79 | } 80 | 81 | private String getSelect() throws Exception { 82 | if (selectIndex == null || selectSeqScan == null) { 83 | LOG.info("Preparing SELECT statements"); 84 | close(selConnection); 85 | selConnection = getPostgresConnection(); 86 | statement = selConnection.createStatement(); 87 | 88 | String query = String.format("select * from %s where v1 = :v1 and v2 = :v2;", getTableName()); 89 | String indexHint = String.format("/*+IndexScan(%s %s_v1_v2)*/", getTableName(), getTableName()); 90 | String seqScan = String.format("/*+SeqScan(%s)*/", getTableName()); 91 | 92 | LOG.info("selIndex " + indexHint + query); 93 | LOG.info("selSeqScan " + seqScan + query); 94 | 95 | selectIndex = indexHint + query; 96 | selectSeqScan = seqScan + query; 97 | 98 | setupSkew(); 99 | } 100 | 101 | String select = selectIndex; 102 | long currentTimeSec = (System.currentTimeMillis() - workloadStartTime) / 1000; 103 | if (currentTimeSec > seqScanStartTime && currentTimeSec < seqScanEndTime) { 104 | if (System.currentTimeMillis() - 3000 > lastReadLogTime) { 105 | // Let's notify once per 3 seconds 106 | LOG.info( 107 | "Read Skew Iteration: " + (currentTimeSec - seqScanStartTime) 108 | + " / " + (seqScanEndTime - seqScanStartTime) + " sec done."); 109 | lastReadLogTime = System.currentTimeMillis(); 110 | } 111 | select = selectSeqScan; 112 | } 113 | return select; 114 | } 115 | 116 | protected void setupSkew() { 117 | boolean timeBased = appConfig.runTimeSeconds > 0; 118 | 119 | if (timeBased) { 120 | // 6/8 normal traffic + 1/8 high latency traffic + 1/8 normal traffic 121 | seqScanStartTime = appConfig.runTimeSeconds / 8 * 6; 122 | seqScanEndTime = appConfig.runTimeSeconds / 8 * 7; 123 | 124 | LOG.info( 125 | "Setting up read skew appConfig.runTimeSeconds = " 126 | + appConfig.runTimeSeconds 127 | + " configuration.getNumReaderThreads() = " 128 | + configuration.getNumReaderThreads() 129 | + " seqScanStartTime = " 130 | + seqScanStartTime 131 | + " seqScanEndTime = " 132 | + seqScanEndTime); 133 | } else { 134 | throw new IllegalArgumentException("This workload only support time based run." 135 | + " Please specify --run_time parameter"); 136 | } 137 | } 138 | 139 | @Override 140 | public long doRead() { 141 | int rowsToInsert = (int) appConfig.numKeysToWrite; 142 | long maxValue = Math.min(rowsToInsert / 100, 100); 143 | long key = random.nextInt(rowsToInsert); 144 | long v1 = key / maxValue; 145 | long v2 = key % maxValue; 146 | try { 147 | String select = getSelect(); 148 | statement.execute(select 149 | .replace(":v1", String.valueOf(v1)) 150 | .replace(":v2", String.valueOf(v2))); 151 | } catch (Exception e) { 152 | LOG.info("Failed reading value " + v1 + ":" + v2, e); 153 | if (statement != null) { 154 | try { 155 | statement.close(); 156 | } catch (Throwable t) { 157 | LOG.warn("encountered exception closing statement: " + t); 158 | } 159 | } 160 | statement = null; 161 | selectSeqScan = null; 162 | selectIndex = null; 163 | return 0; 164 | } 165 | return 1; 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/IOPSThread.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common; 15 | 16 | import java.util.concurrent.locks.ReentrantLock; 17 | 18 | import org.apache.log4j.Logger; 19 | 20 | import com.yugabyte.sample.apps.AppBase; 21 | 22 | /** 23 | * A class that encapsulates a single IO thread. The thread has an index (which is an integer), 24 | * models an OLTP app and an IO type (read or write). It performs the required IO as long as 25 | * the app has not completed all its IO. 26 | */ 27 | public class IOPSThread extends Thread { 28 | private static final Logger LOG = Logger.getLogger(IOPSThread.class); 29 | 30 | // The thread id. 31 | protected int threadIdx; 32 | 33 | /** 34 | * The IO types supported by this class. 35 | */ 36 | public static enum IOType { 37 | Write, 38 | Read, 39 | } 40 | // The io type this thread performs. 41 | IOType ioType; 42 | 43 | // The app that is being run. 44 | protected AppBase app; 45 | 46 | private int numExceptions = 0; 47 | 48 | private volatile boolean ioThreadFailed = false; 49 | 50 | private final boolean printAllExceptions; 51 | 52 | // Flag to disable concurrency, i.e. execute ops in lock step. 53 | private final boolean concurrencyDisabled; 54 | 55 | // Lock to disable concurrency. 56 | // Construct a fair lock to prevent one thread from hogging the lock. 57 | private static final ReentrantLock lock = new ReentrantLock(true); 58 | 59 | // Throttle reads or writes in this thread. 60 | private final Throttler throttler; 61 | 62 | public IOPSThread(int threadIdx, AppBase app, IOType ioType, boolean printAllExceptions, 63 | boolean concurrencyDisabled, double maxThroughput) { 64 | this.threadIdx = threadIdx; 65 | this.app = app; 66 | this.ioType = ioType; 67 | this.printAllExceptions = printAllExceptions; 68 | this.concurrencyDisabled = concurrencyDisabled; 69 | if (maxThroughput > 0) { 70 | LOG.info("Throttling " + (ioType == IOType.Read ? "read" : "write") + 71 | " ops to " + maxThroughput + " ops/sec."); 72 | this.throttler = new Throttler(maxThroughput); 73 | } else { 74 | this.throttler = null; 75 | } 76 | } 77 | 78 | public int getNumExceptions() { 79 | return numExceptions; 80 | } 81 | 82 | public boolean hasFailed() { 83 | return ioThreadFailed; 84 | } 85 | 86 | public long numOps() { 87 | return app.numOps(); 88 | } 89 | 90 | /** 91 | * Cleanly shuts down the IOPSThread. 92 | */ 93 | public void stopThread() { 94 | this.app.stopApp(); 95 | } 96 | 97 | /** 98 | * Method that performs the desired type of IO in the IOPS thread. 99 | */ 100 | @Override 101 | public void run() { 102 | try { 103 | LOG.debug("Starting " + ioType.toString() + " IOPS thread #" + threadIdx); 104 | int numConsecutiveExceptions = 0; 105 | while (!app.hasFinished()) { 106 | if (concurrencyDisabled) { 107 | // Wait for the previous thread to execute its step. 108 | lock.lock(); 109 | } 110 | try { 111 | if (throttler != null) { 112 | throttler.traceOp(); 113 | } 114 | switch (ioType) { 115 | case Write: app.performWrite(threadIdx); break; 116 | case Read: app.performRead(); break; 117 | } 118 | numConsecutiveExceptions = 0; 119 | } catch (RuntimeException e) { 120 | numExceptions++; 121 | if (numConsecutiveExceptions++ % 10 == 0 || printAllExceptions) { 122 | app.reportException(e); 123 | } 124 | // Reset state only for redis workload. CQL workloads will hit 'InvalidQueryException' 125 | // with prepared statements if reset and the same statement is re-executed. 126 | if (!app.getRedisServerInUse().isEmpty()) { 127 | LOG.warn("Resetting clients for redis: " + app.getRedisServerInUse()); 128 | app.resetClients(); 129 | } 130 | 131 | if (numConsecutiveExceptions > 500) { 132 | LOG.error("Had more than " + numConsecutiveExceptions 133 | + " consecutive exceptions. Exiting.", e); 134 | ioThreadFailed = true; 135 | return; 136 | } 137 | try { 138 | Thread.sleep(1000); 139 | } catch (InterruptedException ie) { 140 | LOG.error("Sleep interrupted.", ie); 141 | ioThreadFailed = true; 142 | return; 143 | } 144 | } finally { 145 | if (concurrencyDisabled) { 146 | // Signal the next thread in the wait queue to resume. 147 | lock.unlock(); 148 | } 149 | if (throttler != null) { 150 | // Sleep only after releasing the lock above. 151 | throttler.throttleOp(); 152 | } 153 | } 154 | } 155 | } finally { 156 | LOG.debug("IOPS thread #" + threadIdx + " finished"); 157 | app.terminate(); 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/LogUtil.java: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) YugaByte, Inc. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | // in compliance with the License. You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software distributed under the License 10 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | // or implied. See the License for the specific language governing permissions and limitations 12 | // under the License. 13 | // 14 | package com.yugabyte.sample.common; 15 | 16 | import org.apache.log4j.ConsoleAppender; 17 | import org.apache.log4j.Level; 18 | import org.apache.log4j.Logger; 19 | import org.apache.log4j.PatternLayout; 20 | 21 | public class LogUtil { 22 | public static void configureLogLevel(boolean verbose) { 23 | // First remove all appenders. 24 | Logger.getLogger("com.yugabyte.sample").removeAppender("YBConsoleLogger"); 25 | Logger.getRootLogger().removeAppender("YBConsoleLogger");; 26 | 27 | // Create the console appender. 28 | ConsoleAppender console = new ConsoleAppender(); 29 | console.setName("YBConsoleLogger"); 30 | String PATTERN = "%d [%p|%c|%C{1}] %m%n"; 31 | console.setLayout(new PatternLayout(PATTERN)); 32 | console.setThreshold(verbose ? Level.DEBUG : Level.INFO); 33 | console.activateOptions(); 34 | 35 | // Set the desired logging level. 36 | if (verbose) { 37 | // If verbose, make everything DEBUG log level and output to console. 38 | Logger.getRootLogger().addAppender(console); 39 | Logger.getRootLogger().setLevel(Level.DEBUG); 40 | } else { 41 | // If not verbose, allow YB sample app and driver INFO logs go to console. 42 | Logger.getLogger("com.yugabyte.sample").addAppender(console); 43 | Logger.getLogger("com.yugabyte.driver").addAppender(console); 44 | Logger.getLogger("com.datastax.driver").addAppender(console); 45 | Logger.getRootLogger().setLevel(Level.WARN); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/SimpleLoadGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common; 15 | 16 | import java.security.MessageDigest; 17 | import java.util.HashSet; 18 | import java.util.Random; 19 | import java.util.Set; 20 | import java.util.concurrent.ThreadLocalRandom; 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | import org.apache.commons.codec.binary.Hex; 24 | import org.apache.log4j.Logger; 25 | 26 | public class SimpleLoadGenerator { 27 | private static final Logger LOG = Logger.getLogger(SimpleLoadGenerator.class); 28 | 29 | public static class Key { 30 | // The underlying key is an integer. 31 | Long key; 32 | // The randomized loadtester prefix. 33 | String keyPrefix = (CmdLineOpts.loadTesterUUID != null) 34 | ? CmdLineOpts.loadTesterUUID.toString() 35 | : "key"; 36 | 37 | public Key(long key, String keyPrefix) { 38 | this.key = new Long(key); 39 | if (keyPrefix != null) { 40 | this.keyPrefix = keyPrefix; 41 | } 42 | } 43 | 44 | public long asNumber() { 45 | return key; 46 | } 47 | 48 | public String asString() { return keyPrefix + ":" + key.toString(); } 49 | 50 | public String getKeyWithHashPrefix() throws Exception { 51 | String k = asString(); 52 | MessageDigest md = MessageDigest.getInstance("MD5"); 53 | md.update(k.getBytes()); 54 | return Hex.encodeHexString(md.digest()) + ":" + k; 55 | } 56 | 57 | public String getValueStr() { 58 | return ("val:" + key.toString()); 59 | } 60 | 61 | public String getValueStr(int idx, int size) { 62 | StringBuilder sb = new StringBuilder(); 63 | sb.append("val"); 64 | sb.append(idx); 65 | sb.append(":"); 66 | sb.append(key.toString()); 67 | for (int i = sb.length(); i < size; ++i) { 68 | sb.append("_"); 69 | } 70 | return sb.toString(); 71 | } 72 | 73 | public void verify(String value) { 74 | if (value == null || !value.equals(getValueStr())) { 75 | LOG.fatal("Value mismatch for key: " + key.toString() + 76 | ", expected: " + getValueStr() + 77 | ", got: " + value); 78 | } 79 | } 80 | 81 | @Override 82 | public String toString() { 83 | return "Key: " + key + ", value: " + getValueStr(); 84 | } 85 | } 86 | 87 | // The key to start from. 88 | final long startKey; 89 | // The key to write till. 90 | final long endKey; 91 | // The max key that was successfully written consecutively. 92 | AtomicLong maxWrittenKey; 93 | // The max key that has been generated and handed out so far. 94 | AtomicLong maxGeneratedKey; 95 | // Set of keys that failed to write. 96 | final Set failedKeys; 97 | // Keys that have been written above maxWrittenKey. 98 | final Set writtenKeys; 99 | // A background thread to track keys written and increment maxWrittenKey. 100 | Thread writtenKeysTracker; 101 | // The prefix for the key. 102 | String keyPrefix; 103 | // Random number generator. 104 | Random random = new Random(); 105 | 106 | public SimpleLoadGenerator(long startKey, final long endKey, 107 | long maxWrittenKey) { 108 | this.startKey = startKey; 109 | this.endKey = endKey; 110 | this.maxWrittenKey = new AtomicLong(maxWrittenKey); 111 | this.maxGeneratedKey = new AtomicLong(maxWrittenKey); 112 | failedKeys = new HashSet(); 113 | writtenKeys = new HashSet(); 114 | writtenKeysTracker = new Thread("Written Keys Tracker") { 115 | @Override 116 | public void run() { 117 | do { 118 | long key = SimpleLoadGenerator.this.maxWrittenKey.get() + 1; 119 | synchronized (this) { 120 | if (failedKeys.contains(key) || writtenKeys.remove(key)) { 121 | SimpleLoadGenerator.this.maxWrittenKey.set(key); 122 | if (key == endKey - 1) { 123 | // We've inserted all requested keys, no need to track 124 | // maxWrittenKey/writtenKeys anymore. 125 | break; 126 | } 127 | } else { 128 | try { 129 | wait(); 130 | } catch (InterruptedException e) { 131 | // Ignore 132 | } 133 | }; 134 | } 135 | } while (true); 136 | } 137 | }; 138 | // Make tracker a daemon thread so that it will not block the load tester from exiting 139 | // when done. 140 | writtenKeysTracker.setDaemon(true); 141 | writtenKeysTracker.start(); 142 | } 143 | 144 | public void setKeyPrefix(String prefix) { 145 | keyPrefix = prefix; 146 | } 147 | 148 | public void recordWriteSuccess(Key key) { 149 | if (key.asNumber() > maxWrittenKey.get()) { 150 | synchronized (writtenKeysTracker) { 151 | writtenKeys.add(key.asNumber()); 152 | writtenKeysTracker.notify(); 153 | } 154 | } 155 | } 156 | 157 | public void recordWriteFailure(Key key) { 158 | synchronized (writtenKeysTracker) { 159 | if (key != null) { 160 | failedKeys.add(key.asNumber()); 161 | writtenKeysTracker.notify(); 162 | } 163 | } 164 | } 165 | 166 | // Always returns a non-null key. 167 | public Key getKeyToWrite() { 168 | Key retKey = null; 169 | do { 170 | long maxKey = maxWrittenKey.get(); 171 | // Return a random key to update if we have already written all keys. 172 | if (maxKey != -1 && maxKey == endKey - 1) { 173 | retKey = generateKey(ThreadLocalRandom.current().nextLong(maxKey)); 174 | } else { 175 | retKey = generateKey(maxGeneratedKey.incrementAndGet()); 176 | } 177 | 178 | if (retKey == null) { 179 | try { 180 | Thread.sleep(1 /* millisecs */); 181 | } catch (InterruptedException e) { /* Ignore */ } 182 | } 183 | } while (retKey == null); 184 | 185 | return retKey; 186 | } 187 | 188 | public Key getKeyToRead() { 189 | long maxKey = maxWrittenKey.get(); 190 | if (maxKey < 0) { 191 | return null; 192 | } else if (maxKey == 0) { 193 | return generateKey(0); 194 | } 195 | do { 196 | long key = ThreadLocalRandom.current().nextLong(maxKey); 197 | if (!failedKeys.contains(key)) 198 | return generateKey(key); 199 | } while (true); 200 | } 201 | 202 | public long getMaxWrittenKey() { 203 | return maxWrittenKey.get(); 204 | } 205 | 206 | public long getMaxGeneratedKey() { 207 | return maxGeneratedKey.get(); 208 | } 209 | 210 | public Key generateKey(long key) { 211 | return new Key(key, keyPrefix); 212 | } 213 | 214 | public boolean stillLoading() { 215 | return maxGeneratedKey.get() < endKey - 1; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/Throttler.java: -------------------------------------------------------------------------------- 1 | package com.yugabyte.sample.common; 2 | 3 | import org.apache.commons.math3.distribution.PoissonDistribution; 4 | 5 | // Throttles the IO operations to a certain throughput. 6 | // The wait time is sampled from a Poisson distribution to introduce 7 | // randomness and prevent every thread from resuming at the same time. 8 | public class Throttler { 9 | private final PoissonDistribution poissonDistribution; 10 | private long startTime; 11 | 12 | public Throttler(double maxThroughput) { 13 | double throttleDelay = 1000.0 / maxThroughput; 14 | this.poissonDistribution = new PoissonDistribution(throttleDelay); 15 | } 16 | 17 | // Begin throttling an operation. 18 | public void traceOp() { 19 | startTime = System.currentTimeMillis(); 20 | } 21 | 22 | // Operation done. Wait until the next operation can start. 23 | public void throttleOp() { 24 | long opDelay = poissonDistribution.sample(); 25 | long endTime = System.currentTimeMillis(); 26 | long waitTime = opDelay - (endTime - startTime); 27 | if (waitTime > 0) { 28 | try { 29 | Thread.sleep(waitTime); 30 | } catch (InterruptedException e) { 31 | e.printStackTrace(); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/TimeseriesLoadGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common; 15 | 16 | import java.util.Random; 17 | import java.util.concurrent.atomic.AtomicLong; 18 | 19 | import org.apache.log4j.Logger; 20 | 21 | public class TimeseriesLoadGenerator { 22 | private static final Logger LOG = Logger.getLogger(TimeseriesLoadGenerator.class); 23 | 24 | Random random = new Random(); 25 | // The data source id. 26 | String id; 27 | // The timestamp at which the data emit started. 28 | long dataEmitStartTs = -1; 29 | // State variable tracking the last timestamp emitted by this source (assumed to be the same 30 | // across all the nodes). -1 indicates no data point has been emitted. 31 | long lastEmittedTs = -1; 32 | // The time interval for generating data points. One data point is generated every 33 | // dataEmitRateMs milliseconds. 34 | long dataEmitRateMs; 35 | // The table level ttl used to calculate how many data points should be returned. 36 | long tableTTLMillis = -1; 37 | // If true, verification is enabled. Anytime the load generator detects that the load is not 38 | // keeping up with the DB writes, it turns verification off. 39 | Boolean verificationEnabled = true; 40 | // The write lag tracked as a variable. 41 | AtomicLong maxWriteLag = new AtomicLong(0); 42 | 43 | public TimeseriesLoadGenerator(int idx, long dataEmitRateMs, long tableTTLMillis) { 44 | // Add a unique uuid for each load tester. 45 | 46 | this.id = ((CmdLineOpts.loadTesterUUID != null) 47 | ? idx + "-" + CmdLineOpts.loadTesterUUID 48 | : "" + idx); 49 | this.dataEmitRateMs = dataEmitRateMs; 50 | this.tableTTLMillis = tableTTLMillis; 51 | } 52 | 53 | public String getId() { 54 | return id; 55 | } 56 | 57 | public boolean isVerificationEnabled() { 58 | return verificationEnabled; 59 | } 60 | 61 | public long getWriteLag() { 62 | return maxWriteLag.get(); 63 | } 64 | 65 | /** 66 | * Returns the epoch time when the next data point should be emitted. Returns -1 if no data 67 | * point needs to be emitted in order to satisfy dataEmitRateMs. 68 | */ 69 | public long getDataEmitTs() { 70 | long ts = System.currentTimeMillis(); 71 | // Check if too little time has elapsed since the last data point was emitted. 72 | if (ts - lastEmittedTs < dataEmitRateMs) { 73 | return -1; 74 | } 75 | // Return the data point at the time boundary needed. 76 | if (lastEmittedTs == -1) { 77 | return ts - (ts % dataEmitRateMs); 78 | } 79 | // Check if we have skipped intervals, in which case we need to turn verification off. 80 | if ((ts - lastEmittedTs) / dataEmitRateMs > 1) { 81 | turnVerificationOff(ts); 82 | } 83 | return lastEmittedTs + dataEmitRateMs; 84 | } 85 | 86 | /** 87 | * @return true if this generator has emitted any data so far. 88 | */ 89 | public boolean getHasEmittedData() { 90 | return (lastEmittedTs > -1); 91 | } 92 | 93 | /** 94 | * Callback from the user of this generator object to track the latest emitted data point. This 95 | * should be called after getting the next timestamp by calling getDataEmitTs() and persisting 96 | * the value in the db. 97 | * @param ts timestamp to set. 98 | */ 99 | public synchronized void setLastEmittedTs(long ts) { 100 | // Set the time when we started emitting data. 101 | if (dataEmitStartTs == -1) { 102 | dataEmitStartTs = ts; 103 | } 104 | if (lastEmittedTs < ts) { 105 | lastEmittedTs = ts; 106 | } 107 | } 108 | 109 | /** 110 | * @return the timestamp when the latest data point was emitted. 111 | */ 112 | public synchronized long getLastEmittedTs() { 113 | return lastEmittedTs; 114 | } 115 | 116 | /** 117 | * Returns the number of data points we expect to have written between the start and finish time. 118 | * Note that the start and end time are assumed to be exclusive - so any data point at those times 119 | * is not included in the calculation. 120 | * @param startTime the start time for the query. 121 | * @param endTime the end time for the query. 122 | * @return the number of data points between start and end. 123 | */ 124 | public int getExpectedNumDataPoints(long startTime, long endTime) { 125 | // If verification is disabled, nothing to do. 126 | if (!verificationEnabled) { 127 | return -1; 128 | } 129 | // If we are too far in the future, we are not able to catch up. Disable verification. 130 | long ts = System.currentTimeMillis(); 131 | if (ts - tableTTLMillis >= endTime) { 132 | turnVerificationOff(ts); 133 | return -1; 134 | } 135 | // Pick the more recent time between when we started emitting data and the start of our read 136 | // time interval, as there should be no data points returned before that. 137 | long effectiveStartTime = (startTime < dataEmitStartTs) ? dataEmitStartTs : startTime; 138 | if (endTime - effectiveStartTime > tableTTLMillis) { 139 | effectiveStartTime = endTime - tableTTLMillis; 140 | } 141 | long startTimeEmitBoundary = (effectiveStartTime / dataEmitRateMs + 1) * dataEmitRateMs; 142 | long endTimeEmitBoundary = endTime - endTime % dataEmitRateMs; 143 | long deltaT = endTimeEmitBoundary - startTimeEmitBoundary; 144 | int expectedRows = (int)(deltaT / dataEmitRateMs) + 1; 145 | LOG.debug("startTime: " + startTime + ", effectiveStartTime: " + effectiveStartTime + 146 | ", endTime: " + endTime + ", startTimeEmitBoundary : " + startTimeEmitBoundary + 147 | ", endTimeEmitBoundary: " + endTimeEmitBoundary + ", expectedRows: " + expectedRows); 148 | return expectedRows; 149 | } 150 | 151 | private void turnVerificationOff(long ts) { 152 | if (verificationEnabled) { 153 | synchronized (verificationEnabled) { 154 | verificationEnabled = false; 155 | } 156 | } 157 | maxWriteLag.set(ts - lastEmittedTs); 158 | } 159 | 160 | public String printDebugInfo(long startTime, long endTime) { 161 | // Pick the more recent time between when we started emitting data and the start of our read 162 | // time interval, as there should be no data points returned before that. 163 | long effectiveStartTime = (startTime < dataEmitStartTs) ? dataEmitStartTs : startTime; 164 | if (endTime - effectiveStartTime > tableTTLMillis) { 165 | effectiveStartTime = endTime - tableTTLMillis; 166 | } 167 | long startTimeEmitBoundary = (effectiveStartTime / dataEmitRateMs + 1) * dataEmitRateMs; 168 | long endTimeEmitBoundary = endTime - endTime % dataEmitRateMs; 169 | long deltaT = endTimeEmitBoundary - startTimeEmitBoundary; 170 | int expectedRows = (int)(deltaT / dataEmitRateMs) + 1; 171 | return "startTime: " + startTime + ", effectiveStartTime: " + effectiveStartTime + 172 | ", endTime: " + endTime + ", startTimeEmitBoundary : " + startTimeEmitBoundary + 173 | ", endTimeEmitBoundary: " + endTimeEmitBoundary + ", expectedRows: " + expectedRows; 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/JsonStatsMetric.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | import com.google.gson.JsonObject; 17 | 18 | public class JsonStatsMetric { 19 | private StatsTracker latency; 20 | private ThroughputStats throughput; 21 | private final String name; 22 | 23 | public JsonStatsMetric(String name) { 24 | latency = new StatsTracker(); 25 | throughput = new ThroughputStats(); 26 | this.name = name; 27 | } 28 | 29 | public synchronized void observe(Observation o) { 30 | throughput.observe(o); 31 | latency.observe(o.getLatencyMillis()); 32 | } 33 | 34 | public synchronized JsonObject getJson() { 35 | JsonObject json = new JsonObject(); 36 | json.add("latency", latency.getJson()); 37 | json.add("throughput", throughput.getJson()); 38 | return json; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/Metric.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | import com.google.gson.JsonObject; 17 | 18 | public class Metric { 19 | private final boolean outputJsonMetrics; 20 | 21 | private final ReadableStatsMetric readableStatsMetric; 22 | private final JsonStatsMetric jsonStatsMetric; 23 | 24 | public Metric(String name, boolean outputJsonMetrics) { 25 | this.outputJsonMetrics = outputJsonMetrics; 26 | 27 | readableStatsMetric = new ReadableStatsMetric(name); 28 | jsonStatsMetric = new JsonStatsMetric(name); 29 | } 30 | 31 | public synchronized void observe(Observation o) { 32 | readableStatsMetric.accumulate(o.getCount(), o.getLatencyNanos()); 33 | if (outputJsonMetrics) jsonStatsMetric.observe(o); 34 | } 35 | 36 | public String getReadableMetricsAndReset() { 37 | return readableStatsMetric.getMetricsAndReset(); 38 | } 39 | 40 | public JsonObject getJsonMetrics() { 41 | return jsonStatsMetric.getJson(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/MetricsTracker.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | import java.util.Map; 17 | import java.util.concurrent.ConcurrentHashMap; 18 | 19 | import com.google.gson.JsonObject; 20 | import org.apache.log4j.Logger; 21 | 22 | public class MetricsTracker extends Thread { 23 | private static final Logger LOG = Logger.getLogger(MetricsTracker.class); 24 | private final boolean outputJsonMetrics; 25 | 26 | // Interface to print custom messages. 27 | public interface StatusMessageAppender { 28 | String appenderName(); 29 | void appendMessage(StringBuilder sb); 30 | } 31 | 32 | // The type of metrics supported. 33 | public enum MetricName { 34 | Read, 35 | Write, 36 | } 37 | // Map to store all the metrics objects. 38 | Map metrics = new ConcurrentHashMap(); 39 | // State variable to make sure this thread is started exactly once. 40 | boolean hasStarted = false; 41 | // Map of custom appenders. 42 | Map appenders = new ConcurrentHashMap(); 43 | 44 | public MetricsTracker(boolean outputJsonMetrics) { 45 | this.setDaemon(true); 46 | this.outputJsonMetrics = outputJsonMetrics; 47 | } 48 | 49 | public void registerStatusMessageAppender(StatusMessageAppender appender) { 50 | appenders.put(appender.appenderName(), appender); 51 | } 52 | 53 | public synchronized void createMetric(MetricName metricName) { 54 | if (!metrics.containsKey(metricName)) { 55 | metrics.put(metricName, new Metric(metricName.name(), outputJsonMetrics)); 56 | } 57 | } 58 | 59 | public Metric getMetric(MetricName metricName) { 60 | return metrics.get(metricName); 61 | } 62 | 63 | public void getReadableMetricsAndReset(StringBuilder sb) { 64 | for (MetricName metricName : MetricName.values()) { 65 | sb.append(String.format("%s | ", metrics.get(metricName).getReadableMetricsAndReset())); 66 | } 67 | } 68 | 69 | @Override 70 | public synchronized void start() { 71 | if (!hasStarted) { 72 | hasStarted = true; 73 | super.start(); 74 | } 75 | } 76 | 77 | @Override 78 | public void run() { 79 | while (true) { 80 | try { 81 | Thread.sleep(5000); 82 | StringBuilder sb = new StringBuilder(); 83 | getReadableMetricsAndReset(sb); 84 | for (StatusMessageAppender appender : appenders.values()) { 85 | appender.appendMessage(sb); 86 | } 87 | LOG.info(sb.toString()); 88 | 89 | if (this.outputJsonMetrics) { 90 | JsonObject json = new JsonObject(); 91 | for (MetricName metricName : MetricName.values()) { 92 | json.add(metricName.name(), metrics.get(metricName).getJsonMetrics()); 93 | } 94 | LOG.info(String.format("%s", json.toString())); 95 | } 96 | } catch (InterruptedException e) {} 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/Observation.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | import java.util.concurrent.TimeUnit; 17 | 18 | public class Observation { 19 | private long count; 20 | private long startTsNanos; 21 | private long endTsNanos; 22 | 23 | public Observation(long count, long startTsNanos, long endTsNanos) { 24 | this.count = count; 25 | this.startTsNanos = startTsNanos; 26 | this.endTsNanos = endTsNanos; 27 | } 28 | 29 | public long getLatencyNanos() { return endTsNanos - startTsNanos; } 30 | 31 | public double getLatencyMillis() { 32 | return ((double) getLatencyNanos()) / ((double) TimeUnit.MILLISECONDS.toNanos(1)); 33 | } 34 | 35 | public long getCount() { return count; } 36 | 37 | public long getStartTsNanos() { return startTsNanos; } 38 | 39 | public long getEndTsNanos() { return endTsNanos; } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/PromMetrics.java: -------------------------------------------------------------------------------- 1 | package com.yugabyte.sample.common.metrics; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.net.HttpURLConnection; 7 | import java.net.InetSocketAddress; 8 | import java.net.URL; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.regex.Matcher; 12 | import java.util.regex.Pattern; 13 | import java.security.cert.X509Certificate; 14 | 15 | import javax.net.ssl.HostnameVerifier; 16 | import javax.net.ssl.HttpsURLConnection; 17 | import javax.net.ssl.SSLContext; 18 | import javax.net.ssl.TrustManager; 19 | import javax.net.ssl.X509TrustManager; 20 | 21 | import org.apache.log4j.Logger; 22 | 23 | // Utility class to hit yb-tserver-ip:9000/prometheus-metrics and 24 | // fetch counters for the relevant metrics. 25 | public class PromMetrics { 26 | private final List promContactPoints; 27 | 28 | private static final Logger LOG = Logger.getLogger(PromMetrics.class); 29 | 30 | /* 31 | * Initializes with T-Server nodes to be contacted for the metrics. 32 | */ 33 | public PromMetrics(List nodes) throws IOException { 34 | promContactPoints = new ArrayList<>(); 35 | for (InetSocketAddress node : nodes) { 36 | promContactPoints.add(String.format( 37 | "http://%s:9000/prometheus-metrics", node.getHostString())); 38 | } 39 | disableSSLVerification(); 40 | } 41 | 42 | /* 43 | * Disable SSL since prometheus-metrics are not exposed with a valid 44 | * certificate. 45 | * 46 | * TODO: Figure out how to do this in a more secure way. 47 | */ 48 | public static void disableSSLVerification() throws IOException { 49 | try { 50 | TrustManager[] trustAllCerts = new TrustManager[]{ 51 | new X509TrustManager() { 52 | public X509Certificate[] getAcceptedIssuers() { 53 | return null; 54 | } 55 | public void checkClientTrusted(X509Certificate[] certs, String authType) { 56 | } 57 | public void checkServerTrusted(X509Certificate[] certs, String authType) { 58 | } 59 | } 60 | }; 61 | 62 | SSLContext sc = SSLContext.getInstance("TLS"); 63 | sc.init(null, trustAllCerts, new java.security.SecureRandom()); 64 | HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); 65 | 66 | HostnameVerifier allHostsValid = (hostname, session) -> true; 67 | HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); 68 | } catch (Exception e) { 69 | throw new IOException("Failed to disable SSL verification", e); 70 | } 71 | } 72 | 73 | /* 74 | * Fetches the counter for the given metric and table name 75 | * and accumulates across all the T-Servers. 76 | */ 77 | public long getCounter(String metricName, String tableName) { 78 | long counter = 0; 79 | for (String promContactPoint : promContactPoints) { 80 | long fetchedCounter = fetchPromCounter(metricName, tableName, promContactPoint); 81 | if (fetchedCounter > 0) { 82 | counter += fetchedCounter; 83 | } 84 | } 85 | 86 | return counter; 87 | } 88 | 89 | /* 90 | * Fetches the metric counter for one T-Server. 91 | */ 92 | private long fetchPromCounter(String metricName, String tableName, String promContactPoint) { 93 | HttpURLConnection connection = null; 94 | 95 | try { 96 | URL url = new URL(promContactPoint); 97 | connection = (HttpURLConnection) url.openConnection(); 98 | connection.setRequestMethod("GET"); 99 | 100 | int responseCode = connection.getResponseCode(); 101 | if (responseCode == HttpURLConnection.HTTP_OK) { 102 | BufferedReader reader = new BufferedReader( 103 | new InputStreamReader(connection.getInputStream())); 104 | StringBuilder response = new StringBuilder(); 105 | String line; 106 | 107 | while ((line = reader.readLine()) != null) { 108 | response.append(line).append("\n"); 109 | } 110 | reader.close(); 111 | 112 | // Example: match restart_read_requests{table_name="usertable", ...} 10 1234567890 113 | // ^ 114 | // | 115 | // counter 116 | Pattern pattern = Pattern.compile( 117 | "^" + metricName + "\\{[^}]*table_name=\"" + tableName + 118 | "\"[^}]*\\}\\s+(\\d+)\\s+(\\d+)", Pattern.MULTILINE); 119 | Matcher matcher = pattern.matcher(response.toString()); 120 | 121 | if (matcher.find()) { 122 | long counter = Long.parseLong(matcher.group(1)); 123 | if (matcher.find()) { 124 | // Only one match is expected. 125 | LOG.fatal("Found multiple matches for metric " + 126 | metricName + " for table " + tableName); 127 | } 128 | return counter; 129 | } 130 | 131 | LOG.error("Failed to find metric " + metricName + " for table " + tableName); 132 | } else if (responseCode == HttpURLConnection.HTTP_MOVED_TEMP || responseCode == HttpURLConnection.HTTP_MOVED_PERM) { 133 | String newUrl = connection.getHeaderField("Location"); 134 | LOG.info("Redirecting to " + newUrl); 135 | return fetchPromCounter(metricName, tableName, newUrl); 136 | } else { 137 | LOG.error("Failed to fetch metrics: HTTP response code " + responseCode); 138 | } 139 | } catch(IOException e) { 140 | LOG.error("Failed to fetch metrics", e); 141 | } finally { 142 | if (connection != null) { 143 | connection.disconnect(); 144 | } 145 | } 146 | 147 | return -1; 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/ReadableStatsMetric.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | import org.apache.log4j.Logger; 17 | 18 | public class ReadableStatsMetric { 19 | private static final Logger LOG = Logger.getLogger(ReadableStatsMetric.class); 20 | String name; 21 | private final Object lock = new Object(); 22 | protected long curOpCount = 0; 23 | protected long curOpLatencyNanos = 0; 24 | protected long totalOpCount = 0; 25 | private long lastSnapshotNanos; 26 | 27 | public ReadableStatsMetric(String name) { 28 | this.name = name; 29 | lastSnapshotNanos = System.nanoTime(); 30 | } 31 | 32 | /** 33 | * Accumulate metrics with operations processed as one batch. 34 | * @param numOps number of ops processed as one batch 35 | * @param batchLatencyNanos whole batch latency 36 | */ 37 | public synchronized void accumulate(long numOps, long batchLatencyNanos) { 38 | curOpCount += numOps; 39 | curOpLatencyNanos += batchLatencyNanos * numOps; 40 | totalOpCount += numOps; 41 | } 42 | 43 | public synchronized String getMetricsAndReset() { 44 | long currNanos = System.nanoTime(); 45 | long elapsedNanos = currNanos - lastSnapshotNanos; 46 | LOG.debug("currentOpLatency: " + curOpLatencyNanos + ", currentOpCount: " + curOpCount); 47 | double ops_per_sec = 48 | (elapsedNanos == 0) ? 0 : (curOpCount * 1000000000 * 1.0 / elapsedNanos); 49 | double latency = (curOpCount == 0) ? 0 : (curOpLatencyNanos / 1000000 * 1.0 / curOpCount); 50 | curOpCount = 0; 51 | curOpLatencyNanos = 0; 52 | lastSnapshotNanos = currNanos; 53 | return String.format("%s: %.2f ops/sec (%.2f ms/op), %d total ops", 54 | name, ops_per_sec, latency, totalOpCount); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/StatsTracker.java: -------------------------------------------------------------------------------- 1 | package com.yugabyte.sample.common.metrics; 2 | 3 | import com.google.gson.JsonObject; 4 | import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; 5 | 6 | public class StatsTracker { 7 | private DescriptiveStatistics stats; 8 | 9 | public StatsTracker() { 10 | this.stats = new DescriptiveStatistics(); 11 | } 12 | 13 | public synchronized void observe(double value) { 14 | stats.addValue(value); 15 | } 16 | 17 | public synchronized JsonObject getJson() { 18 | JsonObject json = new JsonObject(); 19 | json.addProperty("mean", stats.getMean()); 20 | json.addProperty("variance", stats.getVariance()); 21 | json.addProperty("sampleSize", stats.getN()); 22 | json.addProperty("min", stats.getMin()); 23 | json.addProperty("max", stats.getMax()); 24 | json.addProperty("p99", stats.getPercentile(99)); 25 | return json; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/ThroughputObserver.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | public class ThroughputObserver { 17 | private final long startTsNanos; // inclusive 18 | private final long endTsNanos; // exclusive 19 | private long ops; 20 | 21 | ThroughputObserver(long startTsNanos, long endTsNanos) { 22 | this.startTsNanos = startTsNanos; 23 | this.endTsNanos = endTsNanos; 24 | this.ops = 0; 25 | } 26 | 27 | long observe(Observation o) { 28 | long toAdd; 29 | if (o.getEndTsNanos() < startTsNanos || endTsNanos <= o.getStartTsNanos()) { 30 | toAdd = 0; 31 | } else { 32 | toAdd = o.getCount(); 33 | } 34 | synchronized(this) { 35 | ops += toAdd; 36 | } 37 | 38 | return toAdd; 39 | } 40 | 41 | synchronized long getOps() { return ops; } 42 | 43 | long getStartTsNanos() { return startTsNanos; } 44 | 45 | long getEndTsNanos() { return endTsNanos; } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/yugabyte/sample/common/metrics/ThroughputStats.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) YugaByte, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations 11 | // under the License. 12 | // 13 | 14 | package com.yugabyte.sample.common.metrics; 15 | 16 | import java.util.concurrent.TimeUnit; 17 | 18 | import com.google.gson.JsonObject; 19 | 20 | public class ThroughputStats { 21 | private ThroughputObserver activeObserver; 22 | 23 | private StatsTracker throughput; 24 | 25 | private static final long NANOS_PER_THROUGHPUT_OBSERVATION = TimeUnit.SECONDS.toNanos(1); 26 | 27 | private ThroughputObserver getObserver(long startTsNanos) { 28 | return new ThroughputObserver(startTsNanos, startTsNanos + NANOS_PER_THROUGHPUT_OBSERVATION); 29 | } 30 | 31 | public ThroughputStats() { 32 | this.activeObserver = null; 33 | this.throughput = new StatsTracker(); 34 | } 35 | 36 | public void observe(Observation o) { 37 | if (activeObserver == null) { 38 | activeObserver = getObserver(o.getStartTsNanos()); 39 | } 40 | long observedSoFar = activeObserver.observe(o); 41 | while (observedSoFar < o.getCount() && activeObserver.getStartTsNanos() < o.getEndTsNanos()) { 42 | throughput.observe(activeObserver.getOps()); 43 | long oldEndTsNanos = activeObserver.getEndTsNanos(); 44 | activeObserver = getObserver(oldEndTsNanos); 45 | observedSoFar += activeObserver.observe(o); 46 | } 47 | assert (observedSoFar == o.getCount()); 48 | } 49 | 50 | public synchronized JsonObject getJson() { 51 | return throughput.getJson(); 52 | } 53 | } 54 | --------------------------------------------------------------------------------