├── pytpcc ├── results.json ├── __init__.py ├── drivers │ ├── __init__.py │ ├── csvdriver.py │ ├── abstractdriver.py │ ├── sqlitedriver.py │ └── postgresqldriver.py ├── runtime │ ├── __init__.py │ ├── executor.py │ └── loader.py ├── util │ ├── __init__.py │ ├── nurand.py │ ├── scaleparameters.py │ ├── rand.py │ └── results.py ├── CONFIG_EXAMPLE ├── MONGODB_EXAMPLE ├── README_v1.1 ├── message.py ├── constants.py ├── worker.py ├── tpcc.sql ├── tpcc_postgresql.sql ├── verify.js ├── tpcc_jsonb.sql ├── coordinator.py └── tpcc.py ├── py_tpcc.egg-info ├── not-zip-safe ├── dependency_links.txt ├── top_level.txt ├── entry_points.txt ├── SOURCES.txt └── PKG-INFO ├── setup.cfg ├── vldb2019 ├── paper.pdf ├── poster.pdf ├── posterHiRes.pdf └── results │ ├── dbSizes.csv │ └── dbSizes.json ├── .gitignore ├── .github └── pull_request_template.md ├── setup.py ├── README ├── .vscode └── launch.json └── README.md /pytpcc/results.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pytpcc/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /pytpcc/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /py_tpcc.egg-info/not-zip-safe: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /py_tpcc.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /py_tpcc.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | pytpcc 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = dev 3 | tag_svn_revision = true 4 | -------------------------------------------------------------------------------- /py_tpcc.egg-info/entry_points.txt: -------------------------------------------------------------------------------- 1 | 2 | # -*- Entry points: -*- 3 | -------------------------------------------------------------------------------- /vldb2019/paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb-labs/py-tpcc/HEAD/vldb2019/paper.pdf -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | *.config 4 | *.csv 5 | *.out 6 | *.log 7 | *.log.* 8 | temp* 9 | -------------------------------------------------------------------------------- /pytpcc/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __all__ = ["executor", "loader"] 4 | -------------------------------------------------------------------------------- /vldb2019/poster.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb-labs/py-tpcc/HEAD/vldb2019/poster.pdf -------------------------------------------------------------------------------- /vldb2019/posterHiRes.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb-labs/py-tpcc/HEAD/vldb2019/posterHiRes.pdf -------------------------------------------------------------------------------- /pytpcc/util/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __all__ = ["scaleparameters", "rand", "nurand", "results"] -------------------------------------------------------------------------------- /py_tpcc.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | setup.cfg 2 | setup.py 3 | py_tpcc.egg-info/PKG-INFO 4 | py_tpcc.egg-info/SOURCES.txt 5 | py_tpcc.egg-info/dependency_links.txt 6 | py_tpcc.egg-info/entry_points.txt 7 | py_tpcc.egg-info/not-zip-safe 8 | py_tpcc.egg-info/top_level.txt 9 | pytpcc/__init__.py -------------------------------------------------------------------------------- /py_tpcc.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.0 2 | Name: py-tpcc 3 | Version: 0.0dev 4 | Summary: Python implementation of the TPC-C benchmark 5 | Home-page: http://www.cs.brown.edu/~pavlo/ 6 | Author: Andy Pavlo 7 | Author-email: pavlo@cs.brown.edu 8 | License: BSD 9 | Description: UNKNOWN 10 | Platform: UNKNOWN 11 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Thanks for submitting a PR to the py-tpcc repo. Please include the following fields (if relevant) prior to submitting your PR. 2 | 3 | **Jira Ticket:** < Ticket Number > 4 | 5 | **Whats Changed:** 6 | High level explanation of changes 7 | 8 | **Patch testing results:** 9 | If applicable, link a patch test showing code changes running successfully 10 | 11 | **Related PRs:** 12 | If applicable, link related PRs 13 | -------------------------------------------------------------------------------- /pytpcc/CONFIG_EXAMPLE: -------------------------------------------------------------------------------- 1 | # HypertableDriver Configuration File 2 | # Created 2011-05-02 01:43:37.859545 3 | [hypertable] 4 | 5 | # hostname 6 | host = localhost 7 | 8 | # namespace name 9 | namespace = tpcc 10 | 11 | # port 12 | port = 38080 13 | 14 | #clientnodes splited by spaces 15 | clients =u1 u2 192.168.3.21 16 | #directories of the code on the client node 17 | path =./code/tpcc/py-tpcc/mtpcc 18 | -------------------------------------------------------------------------------- /pytpcc/MONGODB_EXAMPLE: -------------------------------------------------------------------------------- 1 | # MongodbDriver Configuration File 2 | # generate example with default values via `tpcc.py --print-config mongodb` 3 | [mongodb] 4 | # The mongodb connection string or full URI or mongodb+srv string 5 | uri = mongodb://localhost:27017 6 | causal_consistency = True 7 | findandmodify = True 8 | name = tpcc 9 | secondary_reads = True 10 | denormalize = True 11 | retry_writes = True 12 | # user = username 13 | # passwd = passwd 14 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import sys, os 3 | 4 | version = '0.0' 5 | 6 | setup(name='py-tpcc', 7 | version=version, 8 | description="Python implementation of the TPC-C benchmark", 9 | long_description="""\ 10 | """, 11 | classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers 12 | keywords='', 13 | author='Andy Pavlo', 14 | author_email='pavlo@cs.brown.edu', 15 | url='http://www.cs.brown.edu/~pavlo/', 16 | license='BSD', 17 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 18 | include_package_data=True, 19 | zip_safe=False, 20 | install_requires=[ 21 | # -*- Extra requirements: -*- 22 | ], 23 | entry_points=""" 24 | # -*- Entry points: -*- 25 | """, 26 | ) 27 | -------------------------------------------------------------------------------- /pytpcc/README_v1.1: -------------------------------------------------------------------------------- 1 | 1. 3 newly added files message.py worker.py coordinator.py. Copy them into the old pytpcc directory. 2 | 2. Coordinator is the main part. Use it like the old tpcc.py. e.g., python coordinator.py --config hypertable.config --clientprocs 5 hypertable 3 | 3. Old argument --clients is replaced with --clientprocs, which specifies how many worker processes you want to run on each client node. 4 | 4. All clientnodes(name or ip) must be specified in the configure file. 5 | 5. The directory of the code on the client side should be specified in the configure file,too. The default address is user's home address, which is default using ssh. 6 | It should remain the same for each client node, which is not a problem for now. 7 | 6. Execnet python module should be installed on each client. Here is how to install it. http://codespeak.net/execnet/install.html 8 | 9 | -------------------------------------------------------------------------------- /vldb2019/results/dbSizes.csv: -------------------------------------------------------------------------------- 1 | _id,collections,objects,avgObjSize,dataSize,storageSize,indexes,indexSize,fsUsedSize,fsTotalSize,initial,schema,scale 2 | tpcc_denorm100,8,2.0015349e+07,667.3280676744632,12.439493250101805,11.133750915527344,18,0.7842521667480469,293.30438232421875,1161.84375,true,denorm,100 3 | tpcc_denorm300,8,5.9812523e+07,668.9843211429152,37.26560631301254,28.309009552001953,18,2.2015228271484375,293.30438232421875,1161.84375,true,denorm,300 4 | tpcc_denorm50,8,1.0057136e+07,664.7093958956108,6.225959206931293,5.279247283935547,18,0.386474609375,293.30438232421875,1161.84375,true,denorm,50 5 | tpcc_norm100,9,5.0032327e+07,306.51489873737034,14.282440435141325,8.775493621826172,21,2.2393455505371094,293.30438232421875,1161.84375,true,normal,100 6 | tpcc_norm300,9,1.71357778e+08,293.5935497074431,46.85440874937922,38.29002380371094,21,8.207008361816406,453.9142837524414,1161.84375,true,normal,300 7 | tpcc_norm50,9,2.5077701e+07,306.14241879668316,7.150087544694543,4.682960510253906,21,1.2502708435058594,293.30438232421875,1161.84375,true,normal,50 8 | -------------------------------------------------------------------------------- /vldb2019/results/dbSizes.json: -------------------------------------------------------------------------------- 1 | {"_id":"tpcc_denorm100","collections":8.0,"objects":2.0015349e+07,"avgObjSize":667.3280676744632,"dataSize":12.439493250101805,"storageSize":11.133750915527344,"indexes":18.0,"indexSize":0.7842521667480469,"fsUsedSize":293.30438232421875,"fsTotalSize":1161.84375,"initial":true,"schema":"denorm","scale":100.0} 2 | {"_id":"tpcc_denorm300","collections":8.0,"objects":5.9812523e+07,"avgObjSize":668.9843211429152,"dataSize":37.26560631301254,"storageSize":28.309009552001953,"indexes":18.0,"indexSize":2.2015228271484375,"fsUsedSize":293.30438232421875,"fsTotalSize":1161.84375,"initial":true,"schema":"denorm","scale":300.0} 3 | {"_id":"tpcc_denorm50","collections":8.0,"objects":1.0057136e+07,"avgObjSize":664.7093958956108,"dataSize":6.225959206931293,"storageSize":5.279247283935547,"indexes":18.0,"indexSize":0.386474609375,"fsUsedSize":293.30438232421875,"fsTotalSize":1161.84375,"initial":true,"schema":"denorm","scale":50.0} 4 | {"_id":"tpcc_norm100","collections":9.0,"objects":5.0032327e+07,"avgObjSize":306.51489873737034,"dataSize":14.282440435141325,"storageSize":8.775493621826172,"indexes":21.0,"indexSize":2.2393455505371094,"fsUsedSize":293.30438232421875,"fsTotalSize":1161.84375,"initial":true,"schema":"normal","scale":100.0} 5 | {"_id":"tpcc_norm300","collections":9.0,"objects":1.71357778e+08,"avgObjSize":293.5935497074431,"dataSize":46.85440874937922,"storageSize":38.29002380371094,"indexes":21.0,"indexSize":8.207008361816406,"fsUsedSize":453.9142837524414,"fsTotalSize":1161.84375,"initial":true,"scale":300.0,"schema":"normal"} 6 | {"_id":"tpcc_norm50","collections":9.0,"objects":2.5077701e+07,"avgObjSize":306.14241879668316,"dataSize":7.150087544694543,"storageSize":4.682960510253906,"indexes":21.0,"indexSize":1.2502708435058594,"fsUsedSize":293.30438232421875,"fsTotalSize":1161.84375,"initial":true,"schema":"normal","scale":50.0} 7 | -------------------------------------------------------------------------------- /pytpcc/message.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------- 4 | # Copyright (C) 2011 5 | # Andy Pavlo & Yang Lu 6 | # http:##www.cs.brown.edu/~pavlo/ 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining 9 | # a copy of this software and associated documentation files (the 10 | # "Software"), to deal in the Software without restriction, including 11 | # without limitation the rights to use, copy, modify, merge, publish, 12 | # distribute, sublicense, and/or sell copies of the Software, and to 13 | # permit persons to whom the Software is furnished to do so, subject to 14 | # the following conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be 17 | # included in all copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 22 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | # OTHER DEALINGS IN THE SOFTWARE. 26 | # ----------------------------------------------------------------------- 27 | 28 | import sys 29 | import os 30 | import string 31 | import datetime 32 | import logging 33 | import re 34 | import argparse 35 | import glob 36 | import time 37 | from pprint import pprint,pformat 38 | 39 | from util import * 40 | from runtime import * 41 | import drivers 42 | 43 | EMPTY = 0 44 | CMD_LOAD = 1 45 | CMD_EXECUTE = 2 46 | CMD_STOP = 3 47 | LOAD_COMPLETED = 4 48 | EXECUTE_COMPLETED = 5 49 | 50 | class Message: 51 | def __init__(self,header=EMPTY,data=None): 52 | self.header=header 53 | self.data=data 54 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | + ----------------------------------------------- + 2 | + Python TPC-C + 3 | + ----------------------------------------------- + 4 | 5 | 6 | The basic idea is that you will need to create a new driver file that 7 | implements the functions defined in "abstractdriver.py". One function will 8 | load in the tuples into your database for a given table. Then there are five 9 | separate functions that execute the given transaction based on a set of input 10 | parameters. All the work for generating the tuples and the input parameters 11 | for the transactions has been done for you. 12 | 13 | Here's what you need to do to get started: 14 | 15 | (1) Download the source code from Github: 16 | 17 | https://github.com/apavlo/py-tpcc/tree/master/pytpcc 18 | 19 | (2) Create a new file in the 'drivers' directory for your system that follows 20 | the proper naming convention. For example, if your system is 'MongoDB', then 21 | your new file will be called 'mongodbdriver.py' and that file will contain a 22 | new class called 'MongodbDriver' (note the capitalization). 23 | 24 | (3) Inside your class you will need to implement the required functions of 25 | defined in AbstractDriver. There is documentation on what these need to do 26 | also available on Github: 27 | 28 | https://github.com/apavlo/py-tpcc/wiki 29 | 30 | (3) Try running your system. I would start by defining the configuration file 31 | that gets returned with by the 'makeDefaultConfig' function in your driver and 32 | then implement the data loading part first, since that will guide how you 33 | actually execute the transactions. Using 'MongoDB' as an example again, you 34 | can print out the driver's configuration dict to a file: 35 | 36 | $ python ./tpcc.py --print-config mongodb > mongodb.config 37 | 38 | Make any changes you need to 'mongodb.config' (e.g., passwords, hostnames). 39 | Then test the loader: 40 | 41 | $ python ./tpcc.py --no-execute --config=mongodb.config mongodb 42 | 43 | You can use the CSV driver if you want to see what the data or transaction 44 | input parameters will look like. The following command will dump out just the 45 | input to the driver's functions to files in /tmp/tpcc-* 46 | 47 | $ python ./tpcc.py csv 48 | 49 | You can also look at my SqliteDriver implementation to get an idea of what 50 | your transaction implementation functions need to do: 51 | 52 | https://github.com/apavlo/py-tpcc/blob/master/pytpcc/drivers/sqlitedriver.py 53 | -------------------------------------------------------------------------------- /pytpcc/util/nurand.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Original Java Version: 8 | # Copyright (C) 2008 9 | # Evan Jones 10 | # Massachusetts Institute of Technology 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining 13 | # a copy of this software and associated documentation files (the 14 | # "Software"), to deal in the Software without restriction, including 15 | # without limitation the rights to use, copy, modify, merge, publish, 16 | # distribute, sublicense, and/or sell copies of the Software, and to 17 | # permit persons to whom the Software is furnished to do so, subject to 18 | # the following conditions: 19 | # 20 | # The above copyright notice and this permission notice shall be 21 | # included in all copies or substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 26 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 27 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 28 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | # OTHER DEALINGS IN THE SOFTWARE. 30 | # ----------------------------------------------------------------------- 31 | 32 | from .import rand 33 | 34 | def makeForLoad(): 35 | """Create random NURand constants, appropriate for loading the database.""" 36 | cLast = rand.number(0, 255) 37 | cId = rand.number(0, 1023) 38 | orderLineItemId = rand.number(0, 8191) 39 | return NURandC(cLast, cId, orderLineItemId) 40 | 41 | def validCRun(cRun, cLoad): 42 | """Returns true if the cRun value is valid for running. See TPC-C 2.1.6.1 (page 20)""" 43 | cDelta = abs(cRun - cLoad) 44 | return 65 <= cDelta and cDelta <= 119 and cDelta != 96 and cDelta != 112 45 | 46 | def makeForRun(loadC): 47 | """Create random NURand constants for running TPC-C. TPC-C 2.1.6.1. (page 20) specifies the valid range for these constants.""" 48 | cRun = rand.number(0, 255) 49 | while validCRun(cRun, loadC.cLast) == False: 50 | cRun = rand.number(0, 255) 51 | assert validCRun(cRun, loadC.cLast) 52 | 53 | cId = rand.number(0, 1023) 54 | orderLineItemId = rand.number(0, 8191) 55 | return NURandC(cRun, cId, orderLineItemId) 56 | 57 | class NURandC: 58 | def __init__(self, cLast, cId, orderLineItemId): 59 | self.cLast = cLast 60 | self.cId = cId 61 | self.orderLineItemId = orderLineItemId 62 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Debug MongoDB driver", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "program": "tpcc.py", 12 | "console": "integratedTerminal", 13 | //"args": "${command:pickArgs}" 14 | "args": "--reset --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --config=mongodb.config mongodb --stop-on-error", 15 | "cwd": "${workspaceFolder}/pytpcc/", 16 | "python": "/home/linuka/python_envs/py-tpcc-env/bin/python" 17 | }, 18 | { 19 | "name": "Debug Postgresql JSONB driver", 20 | "type": "debugpy", 21 | "request": "launch", 22 | "program": "tpcc.py", 23 | "console": "integratedTerminal", 24 | "args": "--reset --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb", 25 | "cwd": "${workspaceFolder}/pytpcc/", 26 | "python": "/home/linuka/python_envs/py-tpcc-env/bin/python" 27 | }, 28 | { 29 | // To debug doDelivery, doOrderStatus, doPayment, doStockLevel, getNumberWH 30 | "name": "Debug Postgresql JSONB driver No RESET No LOAD", 31 | "type": "debugpy", 32 | "request": "launch", 33 | "program": "tpcc.py", 34 | "console": "integratedTerminal", 35 | "args": "--no-load --clients=4 --duration=10 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb", 36 | "cwd": "${workspaceFolder}/pytpcc/", 37 | "python": "/home/linuka/python_envs/py-tpcc-env/bin/python" 38 | }, 39 | { 40 | "name": "Debug Postgresql JSONB driver Perf", 41 | "type": "debugpy", 42 | "request": "launch", 43 | "program": "tpcc.py", 44 | "console": "integratedTerminal", 45 | "args": "--reset --clients=1 --duration=1 --scalefactor=20 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb", 46 | "cwd": "${workspaceFolder}/pytpcc/", 47 | "python": "/home/linuka/python_envs/py-tpcc-env/bin/python" 48 | }, 49 | { 50 | "name": "Debug Postgresql driver", 51 | "type": "debugpy", 52 | "request": "launch", 53 | "program": "tpcc.py", 54 | "console": "integratedTerminal", 55 | //"args": "${command:pickArgs}" 56 | "args": "--reset --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --config=postgresql.config postgresql", 57 | "cwd": "${workspaceFolder}/pytpcc/", 58 | "python": "/home/linuka/python_envs/py-tpcc-env/bin/python" 59 | }, 60 | { 61 | "name": "Debug Postgresql driver No RESET No LOAD", 62 | "type": "debugpy", 63 | "request": "launch", 64 | "program": "tpcc.py", 65 | "console": "integratedTerminal", 66 | //"args": "${command:pickArgs}" 67 | "args": "--no-load --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --config=postgresql.config postgresql", 68 | "cwd": "${workspaceFolder}/pytpcc/", 69 | "python": "/home/linuka/python_envs/py-tpcc-env/bin/python" 70 | } 71 | 72 | ] 73 | } -------------------------------------------------------------------------------- /pytpcc/util/scaleparameters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # ----------------------------------------------------------------------- 4 | # Copyright (C) 2011 5 | # Andy Pavlo 6 | # http://www.cs.brown.edu/~pavlo/ 7 | # 8 | # Original Java Version: 9 | # Copyright (C) 2008 10 | # Evan Jones 11 | # Massachusetts Institute of Technology 12 | # 13 | # Permission is hereby granted, free of charge, to any person obtaining 14 | # a copy of this software and associated documentation files (the 15 | # "Software"), to deal in the Software without restriction, including 16 | # without limitation the rights to use, copy, modify, merge, publish, 17 | # distribute, sublicense, and/or sell copies of the Software, and to 18 | # permit persons to whom the Software is furnished to do so, subject to 19 | # the following conditions: 20 | # 21 | # The above copyright notice and this permission notice shall be 22 | # included in all copies or substantial portions of the Software. 23 | # 24 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 27 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 28 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 29 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 30 | # OTHER DEALINGS IN THE SOFTWARE. 31 | # ----------------------------------------------------------------------- 32 | 33 | import constants 34 | 35 | def makeDefault(warehouses): 36 | return ScaleParameters(constants.NUM_ITEMS, \ 37 | warehouses, \ 38 | constants.DISTRICTS_PER_WAREHOUSE, \ 39 | constants.CUSTOMERS_PER_DISTRICT, \ 40 | constants.INITIAL_NEW_ORDERS_PER_DISTRICT) 41 | ## DEF 42 | 43 | def makeWithScaleFactor(warehouses, scaleFactor): 44 | assert scaleFactor >= 1.0 45 | 46 | items = int(constants.NUM_ITEMS/scaleFactor) 47 | if items <= 0: items = 1 48 | districts = int(max(constants.DISTRICTS_PER_WAREHOUSE, 1)) 49 | customers = int(max(constants.CUSTOMERS_PER_DISTRICT/scaleFactor, 1)) 50 | newOrders = int(max(constants.INITIAL_NEW_ORDERS_PER_DISTRICT/scaleFactor, 0)) 51 | 52 | return ScaleParameters(items, warehouses, districts, customers, newOrders) 53 | ## DEF 54 | 55 | class ScaleParameters: 56 | 57 | def __init__(self, items, warehouses, districtsPerWarehouse, customersPerDistrict, newOrdersPerDistrict): 58 | assert 1 <= items and items <= constants.NUM_ITEMS 59 | self.items = items 60 | assert warehouses > 0 61 | self.warehouses = warehouses 62 | self.starting_warehouse = 1 63 | assert 1 <= districtsPerWarehouse and districtsPerWarehouse <= constants.DISTRICTS_PER_WAREHOUSE 64 | self.districtsPerWarehouse = districtsPerWarehouse 65 | assert 1 <= customersPerDistrict and customersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT 66 | self.customersPerDistrict = customersPerDistrict 67 | assert 0 <= newOrdersPerDistrict and newOrdersPerDistrict <= constants.CUSTOMERS_PER_DISTRICT 68 | assert newOrdersPerDistrict <= constants.INITIAL_NEW_ORDERS_PER_DISTRICT 69 | self.newOrdersPerDistrict = newOrdersPerDistrict 70 | self.ending_warehouse = (self.warehouses + self.starting_warehouse - 1) 71 | ## DEF 72 | 73 | def __str__(self): 74 | out = "%d items\n" % self.items 75 | out += "%d warehouses\n" % self.warehouses 76 | out += "%d districts/warehouse\n" % self.districtsPerWarehouse 77 | out += "%d customers/district\n" % self.customersPerDistrict 78 | out += "%d initial new orders/district" % self.newOrdersPerDistrict 79 | return out 80 | ## DEF 81 | 82 | ## CLASS 83 | -------------------------------------------------------------------------------- /pytpcc/drivers/csvdriver.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining 8 | # a copy of this software and associated documentation files (the 9 | # "Software"), to deal in the Software without restriction, including 10 | # without limitation the rights to use, copy, modify, merge, publish, 11 | # distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so, subject to 13 | # the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 21 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | # OTHER DEALINGS IN THE SOFTWARE. 25 | # ----------------------------------------------------------------------- 26 | 27 | import os 28 | import csv 29 | from datetime import datetime 30 | from pprint import pprint,pformat 31 | 32 | from abstractdriver import * 33 | 34 | ## ============================================== 35 | ## CSVDriver 36 | ## ============================================== 37 | class CsvDriver(AbstractDriver): 38 | DEFAULT_CONFIG = { 39 | "table_directory": ("The path to the directory to store the table CSV files", "/tmp/tpcc-tables" ), 40 | "txn_directory": ("The path to the directory to store the txn CSV files", "/tmp/tpcc-txns" ), 41 | } 42 | 43 | def __init__(self, ddl): 44 | super(CsvDriver, self).__init__("csv", ddl) 45 | self.table_directory = None 46 | self.table_outputs = { } 47 | self.txn_directory = None 48 | self.txn_outputs = { } 49 | self.txn_params = { } 50 | ## DEF 51 | 52 | def makeDefaultConfig(self): 53 | return CsvDriver.DEFAULT_CONFIG 54 | ## DEF 55 | 56 | def loadConfig(self, config): 57 | for key in CsvDriver.DEFAULT_CONFIG.keys(): 58 | assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) 59 | 60 | self.table_directory = config["table_directory"] 61 | assert self.table_directory 62 | if not os.path.exists(self.table_directory): os.makedirs(self.table_directory) 63 | 64 | self.txn_directory = config["txn_directory"] 65 | assert self.txn_directory 66 | if not os.path.exists(self.txn_directory): os.makedirs(self.txn_directory) 67 | ## DEF 68 | 69 | def loadTuples(self, tableName, tuples): 70 | if not tableName in self.table_outputs: 71 | path = os.path.join(self.table_directory, "%s.csv" % tableName) 72 | self.table_outputs[tableName] = csv.writer(open(path, 'wb'), quoting=csv.QUOTE_ALL) 73 | ## IF 74 | self.table_outputs[tableName].writerows(tuples) 75 | ## DEF 76 | 77 | def executeTransaction(self, txn, params): 78 | if not txn in self.txn_outputs: 79 | path = os.path.join(self.txn_directory, "%s.csv" % txn) 80 | self.txn_outputs[txn] = csv.writer(open(path, 'wb'), quoting=csv.QUOTE_ALL) 81 | self.txn_params[txn] = params.keys()[:] 82 | self.txn_outputs[txn].writerow(["Timestamp"] + self.txn_params[txn]) 83 | ## IF 84 | row = [datetime.now()] + [params[k] for k in self.txn_params[txn]] 85 | self.txn_outputs[txn].writerow(row) 86 | ## DEF 87 | ## CLASS 88 | 89 | 90 | -------------------------------------------------------------------------------- /pytpcc/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Original Java Version: 8 | # Copyright (C) 2008 9 | # Evan Jones 10 | # Massachusetts Institute of Technology 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining 13 | # a copy of this software and associated documentation files (the 14 | # "Software"), to deal in the Software without restriction, including 15 | # without limitation the rights to use, copy, modify, merge, publish, 16 | # distribute, sublicense, and/or sell copies of the Software, and to 17 | # permit persons to whom the Software is furnished to do so, subject to 18 | # the following conditions: 19 | # 20 | # The above copyright notice and this permission notice shall be 21 | # included in all copies or substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 26 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 27 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 28 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | # OTHER DEALINGS IN THE SOFTWARE. 30 | # ----------------------------------------------------------------------- 31 | 32 | MONEY_DECIMALS = 2 33 | 34 | # Item constants 35 | NUM_ITEMS = 100000 36 | MIN_IM = 1 37 | MAX_IM = 10000 38 | MIN_PRICE = 1.00 39 | MAX_PRICE = 100.00 40 | MIN_I_NAME = 14 41 | MAX_I_NAME = 24 42 | MIN_I_DATA = 26 43 | MAX_I_DATA = 50 44 | 45 | # Warehouse constants 46 | MIN_TAX = 0 47 | MAX_TAX = 0.2000 48 | TAX_DECIMALS = 4 49 | INITIAL_W_YTD = 300000.00 50 | MIN_NAME = 6 51 | MAX_NAME = 10 52 | MIN_STREET = 10 53 | MAX_STREET = 20 54 | MIN_CITY = 10 55 | MAX_CITY = 20 56 | STATE = 2 57 | ZIP_LENGTH = 9 58 | ZIP_SUFFIX = "11111" 59 | 60 | # Stock constants 61 | MIN_QUANTITY = 10 62 | MAX_QUANTITY = 100 63 | DIST = 24 64 | STOCK_PER_WAREHOUSE = 100000 65 | 66 | # District constants 67 | DISTRICTS_PER_WAREHOUSE = 10 68 | INITIAL_D_YTD = 30000.00 # different from Warehouse 69 | INITIAL_NEXT_O_ID = 3001 70 | 71 | # Customer constants 72 | CUSTOMERS_PER_DISTRICT = 3000 73 | INITIAL_CREDIT_LIM = 50000.00 74 | MIN_DISCOUNT = 0.0000 75 | MAX_DISCOUNT = 0.5000 76 | DISCOUNT_DECIMALS = 4 77 | INITIAL_BALANCE = -10.00 78 | INITIAL_YTD_PAYMENT = 10.00 79 | INITIAL_PAYMENT_CNT = 1 80 | INITIAL_DELIVERY_CNT = 0 81 | MIN_FIRST = 6 82 | MAX_FIRST = 10 83 | MIDDLE = "OE" 84 | PHONE = 16 85 | MIN_C_DATA = 300 86 | MAX_C_DATA = 500 87 | GOOD_CREDIT = "GC" 88 | BAD_CREDIT = "BC" 89 | 90 | # Order constants 91 | MIN_CARRIER_ID = 1 92 | MAX_CARRIER_ID = 10 93 | # HACK: This is not strictly correct, but it works 94 | NULL_CARRIER_ID = 0 95 | # o_id < than this value, carrier != null, >= -> carrier == null 96 | NULL_CARRIER_LOWER_BOUND = 2101 97 | MIN_OL_CNT = 5 98 | MAX_OL_CNT = 15 99 | INITIAL_ALL_LOCAL = 1 100 | INITIAL_ORDERS_PER_DISTRICT = 3000 101 | 102 | # Used to generate new order transactions 103 | MAX_OL_QUANTITY = 10 104 | 105 | # Order line constants 106 | INITIAL_QUANTITY = 5 107 | MIN_AMOUNT = 0.01 108 | 109 | # History constants 110 | MIN_DATA = 12 111 | MAX_DATA = 24 112 | INITIAL_AMOUNT = 10.00 113 | 114 | # New order constants 115 | INITIAL_NEW_ORDERS_PER_DISTRICT = 900 116 | 117 | # TPC-C 2.4.3.4 (page 31) says this must be displayed when new order rolls back. 118 | INVALID_ITEM_MESSAGE = "Item number is not valid" 119 | 120 | # Used to generate stock level transactions 121 | MIN_STOCK_LEVEL_THRESHOLD = 10 122 | MAX_STOCK_LEVEL_THRESHOLD = 20 123 | 124 | # Used to generate payment transactions 125 | MIN_PAYMENT = 1.0 126 | MAX_PAYMENT = 5000.0 127 | 128 | # Indicates "brand" items and stock in i_data and s_data. 129 | ORIGINAL_STRING = "ORIGINAL" 130 | 131 | # Table Names 132 | TABLENAME_ITEM = "ITEM" 133 | TABLENAME_WAREHOUSE = "WAREHOUSE" 134 | TABLENAME_DISTRICT = "DISTRICT" 135 | TABLENAME_CUSTOMER = "CUSTOMER" 136 | TABLENAME_STOCK = "STOCK" 137 | TABLENAME_ORDERS = "ORDERS" 138 | TABLENAME_NEW_ORDER = "NEW_ORDER" 139 | TABLENAME_ORDER_LINE = "ORDER_LINE" 140 | TABLENAME_HISTORY = "HISTORY" 141 | 142 | ALL_TABLES = [ 143 | TABLENAME_ITEM, 144 | TABLENAME_WAREHOUSE, 145 | TABLENAME_DISTRICT, 146 | TABLENAME_CUSTOMER, 147 | TABLENAME_STOCK, 148 | TABLENAME_ORDERS, 149 | TABLENAME_NEW_ORDER, 150 | TABLENAME_ORDER_LINE, 151 | TABLENAME_HISTORY, 152 | ] 153 | 154 | # Transaction Types 155 | def enum(*sequential, **named): 156 | enums = dict(map(lambda x: (x, x), sequential)) 157 | # dict(zip(sequential, range(len(sequential))), **named) 158 | return type('Enum', (), enums) 159 | TransactionTypes = enum( 160 | "DELIVERY", 161 | "NEW_ORDER", 162 | "ORDER_STATUS", 163 | "PAYMENT", 164 | "STOCK_LEVEL", 165 | ) 166 | -------------------------------------------------------------------------------- /pytpcc/util/rand.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Original Java Version: 8 | # Copyright (C) 2008 9 | # Evan Jones 10 | # Massachusetts Institute of Technology 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining 13 | # a copy of this software and associated documentation files (the 14 | # "Software"), to deal in the Software without restriction, including 15 | # without limitation the rights to use, copy, modify, merge, publish, 16 | # distribute, sublicense, and/or sell copies of the Software, and to 17 | # permit persons to whom the Software is furnished to do so, subject to 18 | # the following conditions: 19 | # 20 | # The above copyright notice and this permission notice shall be 21 | # included in all copies or substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 26 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 27 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 28 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | # OTHER DEALINGS IN THE SOFTWARE. 30 | # ----------------------------------------------------------------------- 31 | 32 | import random 33 | from . import nurand 34 | 35 | SYLLABLES = [ "BAR", "OUGHT", "ABLE", "PRI", "PRES", "ESE", "ANTI", "CALLY", "ATION", "EING" ] 36 | 37 | nurandVar = None # NURand 38 | def setNURand(nu): 39 | global nurandVar 40 | nurandVar = nu 41 | ## DEF 42 | 43 | def NURand(a, x, y): 44 | """A non-uniform random number, as defined by TPC-C 2.1.6. (page 20).""" 45 | global nurandVar 46 | assert x <= y 47 | if nurandVar is None: 48 | setNURand(nurand.makeForLoad()) 49 | 50 | if a == 255: 51 | c = nurandVar.cLast 52 | elif a == 1023: 53 | c = nurandVar.cId 54 | elif a == 8191: 55 | c = nurandVar.orderLineItemId 56 | else: 57 | raise Exception("a = " + a + " is not a supported value") 58 | 59 | return (((number(0, a) | number(x, y)) + c) % (y - x + 1)) + x 60 | ## DEF 61 | 62 | def number(minimum, maximum): 63 | value = random.randint(minimum, maximum) 64 | assert minimum <= value and value <= maximum 65 | return value 66 | ## DEF 67 | 68 | def numberExcluding(minimum, maximum, excluding): 69 | """An in the range [minimum, maximum], excluding excluding.""" 70 | assert minimum < maximum 71 | assert minimum <= excluding and excluding <= maximum 72 | 73 | ## Generate 1 less number than the range 74 | num = number(minimum, maximum-1) 75 | 76 | ## Adjust the numbers to remove excluding 77 | if num >= excluding: num += 1 78 | assert minimum <= num and num <= maximum and num != excluding 79 | return num 80 | ## DEF 81 | 82 | def fixedPoint(decimal_places, minimum, maximum): 83 | assert decimal_places > 0 84 | assert minimum < maximum 85 | 86 | multiplier = 1 87 | for i in range(0, decimal_places): 88 | multiplier *= 10 89 | 90 | int_min = int(minimum * multiplier + 0.5) 91 | int_max = int(maximum * multiplier + 0.5) 92 | 93 | return float(number(int_min, int_max) / float(multiplier)) 94 | ## DEF 95 | 96 | def selectUniqueIds(numUnique, minimum, maximum): 97 | rows = set() 98 | for i in range(0, numUnique): 99 | index = None 100 | while index == None or index in rows: 101 | index = number(minimum, maximum) 102 | ## WHILE 103 | rows.add(index) 104 | ## FOR 105 | assert len(rows) == numUnique 106 | return rows 107 | ## DEF 108 | 109 | def astring(minimum_length, maximum_length): 110 | """A random alphabetic string with length in range [minimum_length, maximum_length].""" 111 | return randomString(minimum_length, maximum_length, 'a', 26) 112 | ## DEF 113 | 114 | def nstring(minimum_length, maximum_length): 115 | """A random numeric string with length in range [minimum_length, maximum_length].""" 116 | return randomString(minimum_length, maximum_length, '0', 10) 117 | ## DEF 118 | 119 | def randomString(minimum_length, maximum_length, base, numCharacters): 120 | length = number(minimum_length, maximum_length) 121 | baseByte = ord(base) 122 | string = "" 123 | for i in range(length): 124 | string += chr(baseByte + number(0, numCharacters-1)) 125 | return string 126 | ## DEF 127 | 128 | def makeLastName(number): 129 | """A last name as defined by TPC-C 4.3.2.3. Not actually random.""" 130 | global SYLLABLES 131 | assert 0 <= number and number <= 999 132 | indicies = [ number//100, (number//10)%10, number%10 ] 133 | return "".join(map(lambda x: SYLLABLES[x], indicies)) 134 | ## DEF 135 | 136 | def makeRandomLastName(maxCID): 137 | """A non-uniform random last name, as defined by TPC-C 4.3.2.3. The name will be limited to maxCID.""" 138 | min_cid = 999 139 | if (maxCID - 1) < min_cid: min_cid = maxCID - 1 140 | return makeLastName(NURand(255, 0, min_cid)) 141 | ## DEF 142 | -------------------------------------------------------------------------------- /pytpcc/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------- 4 | # Copyright (C) 2011 5 | # Andy Pavlo & Yang Lu 6 | # http:##www.cs.brown.edu/~pavlo/ 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining 9 | # a copy of this software and associated documentation files (the 10 | # "Software"), to deal in the Software without restriction, including 11 | # without limitation the rights to use, copy, modify, merge, publish, 12 | # distribute, sublicense, and/or sell copies of the Software, and to 13 | # permit persons to whom the Software is furnished to do so, subject to 14 | # the following conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be 17 | # included in all copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 22 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | # OTHER DEALINGS IN THE SOFTWARE. 26 | # ----------------------------------------------------------------------- 27 | 28 | import sys 29 | import os 30 | import string 31 | import datetime 32 | import logging 33 | import re 34 | import argparse 35 | import glob 36 | import time 37 | import message 38 | import pickle 39 | import traceback 40 | from pprint import pprint,pformat 41 | 42 | from util import * 43 | from runtime import * 44 | import drivers 45 | 46 | ## ============================================== 47 | ## createDriverClass 48 | ## ============================================== 49 | def createDriverClass(name): 50 | full_name = "%sDriver" % name.title() 51 | mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) 52 | klass = getattr(mod, full_name) 53 | return klass 54 | ## DEF 55 | 56 | 57 | ## ============================================== 58 | ## loaderFunc 59 | ## ============================================== 60 | def loaderFunc(driverClass, scaleParameters, args, config, w_ids, debug): 61 | driver = driverClass(args['ddl']) 62 | assert driver != None 63 | logging.debug("Starting client execution: %s [warehouses=%d]" % (driver, len(w_ids))) 64 | 65 | config['load'] = True 66 | config['execute'] = False 67 | config['reset'] = False 68 | driver.loadConfig(config) 69 | 70 | try: 71 | loadItems = (1 in w_ids) 72 | l = loader.Loader(driver, scaleParameters, w_ids, loadItems) 73 | driver.loadStart() 74 | l.execute() 75 | driver.loadFinish() 76 | except KeyboardInterrupt: 77 | return -1 78 | except (Exception, AssertionError) as ex: 79 | logging.warn("Failed to load data: %s" % (ex)) 80 | #if debug: 81 | traceback.print_exc(file=sys.stdout) 82 | raise 83 | 84 | ## DEF 85 | 86 | 87 | ## ============================================== 88 | ## executorFunc 89 | ## ============================================== 90 | def executorFunc(driverClass, scaleParameters, args, config, debug): 91 | driver = driverClass(args['ddl']) 92 | assert driver != None 93 | logging.debug("Starting client execution: %s" % driver) 94 | 95 | config['execute'] = True 96 | config['reset'] = False 97 | driver.loadConfig(config) 98 | 99 | e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error'], sameWH=args['samewh']) 100 | driver.executeStart() 101 | results = e.execute(args['duration']) 102 | driver.executeFinish() 103 | 104 | return results 105 | ## DEF 106 | 107 | ## MAIN 108 | if __name__=='__channelexec__': 109 | driverClass=None 110 | for item in channel: 111 | command=pickle.loads(item) 112 | if command.header==message.CMD_LOAD: 113 | scaleParameters=command.data[0] 114 | args=command.data[1] 115 | config=command.data[2] 116 | w_ids=command.data[3] 117 | 118 | ## Create a handle to the target client driver at the client side 119 | driverClass = createDriverClass(args['system']) 120 | assert driverClass != None, "Failed to find '%s' class" % args['system'] 121 | driver = driverClass(args['ddl']) 122 | assert driver != None, "Failed to create '%s' driver" % args['system'] 123 | 124 | loaderFunc(driverClass,scaleParameters,args,config,w_ids,True) 125 | m=message.Message(header=message.LOAD_COMPLETED) 126 | channel.send(pickle.dumps(m,-1)) 127 | elif command.header==message.CMD_EXECUTE: 128 | scaleParameters=command.data[0] 129 | args=command.data[1] 130 | config=command.data[2] 131 | 132 | ## Create a handle to the target client driver at the client side 133 | if driverClass==None: 134 | driverClass = createDriverClass(args['system']) 135 | assert driverClass != None, "Failed to find '%s' class" % args['system'] 136 | driver = driverClass(args['ddl']) 137 | assert driver != None, "Failed to create '%s' driver" % args['system'] 138 | 139 | results=executorFunc(driverClass,scaleParameters,args,config,True) 140 | m=message.Message(header=message.EXECUTE_COMPLETED,data=results) 141 | channel.send(pickle.dumps(m,-1)) 142 | 143 | elif command.header==message.CMD_STOP: 144 | pass 145 | else: 146 | pass 147 | 148 | -------------------------------------------------------------------------------- /pytpcc/tpcc.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE WAREHOUSE ( 2 | W_ID SMALLINT DEFAULT '0' NOT NULL, 3 | W_NAME VARCHAR(16) DEFAULT NULL, 4 | W_STREET_1 VARCHAR(32) DEFAULT NULL, 5 | W_STREET_2 VARCHAR(32) DEFAULT NULL, 6 | W_CITY VARCHAR(32) DEFAULT NULL, 7 | W_STATE VARCHAR(2) DEFAULT NULL, 8 | W_ZIP VARCHAR(9) DEFAULT NULL, 9 | W_TAX FLOAT DEFAULT NULL, 10 | W_YTD FLOAT DEFAULT NULL, 11 | CONSTRAINT W_PK_ARRAY PRIMARY KEY (W_ID) 12 | ); 13 | 14 | CREATE TABLE DISTRICT ( 15 | D_ID TINYINT DEFAULT '0' NOT NULL, 16 | D_W_ID SMALLINT DEFAULT '0' NOT NULL REFERENCES WAREHOUSE (W_ID), 17 | D_NAME VARCHAR(16) DEFAULT NULL, 18 | D_STREET_1 VARCHAR(32) DEFAULT NULL, 19 | D_STREET_2 VARCHAR(32) DEFAULT NULL, 20 | D_CITY VARCHAR(32) DEFAULT NULL, 21 | D_STATE VARCHAR(2) DEFAULT NULL, 22 | D_ZIP VARCHAR(9) DEFAULT NULL, 23 | D_TAX FLOAT DEFAULT NULL, 24 | D_YTD FLOAT DEFAULT NULL, 25 | D_NEXT_O_ID INT DEFAULT NULL, 26 | PRIMARY KEY (D_W_ID,D_ID) 27 | ); 28 | 29 | CREATE TABLE ITEM ( 30 | I_ID INTEGER DEFAULT '0' NOT NULL, 31 | I_IM_ID INTEGER DEFAULT NULL, 32 | I_NAME VARCHAR(32) DEFAULT NULL, 33 | I_PRICE FLOAT DEFAULT NULL, 34 | I_DATA VARCHAR(64) DEFAULT NULL, 35 | CONSTRAINT I_PK_ARRAY PRIMARY KEY (I_ID) 36 | ); 37 | 38 | CREATE TABLE CUSTOMER ( 39 | C_ID INTEGER DEFAULT '0' NOT NULL, 40 | C_D_ID TINYINT DEFAULT '0' NOT NULL, 41 | C_W_ID SMALLINT DEFAULT '0' NOT NULL, 42 | C_FIRST VARCHAR(32) DEFAULT NULL, 43 | C_MIDDLE VARCHAR(2) DEFAULT NULL, 44 | C_LAST VARCHAR(32) DEFAULT NULL, 45 | C_STREET_1 VARCHAR(32) DEFAULT NULL, 46 | C_STREET_2 VARCHAR(32) DEFAULT NULL, 47 | C_CITY VARCHAR(32) DEFAULT NULL, 48 | C_STATE VARCHAR(2) DEFAULT NULL, 49 | C_ZIP VARCHAR(9) DEFAULT NULL, 50 | C_PHONE VARCHAR(32) DEFAULT NULL, 51 | C_SINCE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 52 | C_CREDIT VARCHAR(2) DEFAULT NULL, 53 | C_CREDIT_LIM FLOAT DEFAULT NULL, 54 | C_DISCOUNT FLOAT DEFAULT NULL, 55 | C_BALANCE FLOAT DEFAULT NULL, 56 | C_YTD_PAYMENT FLOAT DEFAULT NULL, 57 | C_PAYMENT_CNT INTEGER DEFAULT NULL, 58 | C_DELIVERY_CNT INTEGER DEFAULT NULL, 59 | C_DATA VARCHAR(500), 60 | PRIMARY KEY (C_W_ID,C_D_ID,C_ID), 61 | UNIQUE (C_W_ID,C_D_ID,C_LAST,C_FIRST), 62 | CONSTRAINT C_FKEY_D FOREIGN KEY (C_D_ID, C_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) 63 | ); 64 | CREATE INDEX IDX_CUSTOMER ON CUSTOMER (C_W_ID,C_D_ID,C_LAST); 65 | 66 | CREATE TABLE HISTORY ( 67 | H_C_ID INTEGER DEFAULT NULL, 68 | H_C_D_ID TINYINT DEFAULT NULL, 69 | H_C_W_ID SMALLINT DEFAULT NULL, 70 | H_D_ID TINYINT DEFAULT NULL, 71 | H_W_ID SMALLINT DEFAULT '0' NOT NULL, 72 | H_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 73 | H_AMOUNT FLOAT DEFAULT NULL, 74 | H_DATA VARCHAR(32) DEFAULT NULL, 75 | CONSTRAINT H_FKEY_C FOREIGN KEY (H_C_ID, H_C_D_ID, H_C_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID), 76 | CONSTRAINT H_FKEY_D FOREIGN KEY (H_D_ID, H_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) 77 | ); 78 | 79 | CREATE TABLE STOCK ( 80 | S_I_ID INTEGER DEFAULT '0' NOT NULL REFERENCES ITEM (I_ID), 81 | S_W_ID SMALLINT DEFAULT '0 ' NOT NULL REFERENCES WAREHOUSE (W_ID), 82 | S_QUANTITY INTEGER DEFAULT '0' NOT NULL, 83 | S_DIST_01 VARCHAR(32) DEFAULT NULL, 84 | S_DIST_02 VARCHAR(32) DEFAULT NULL, 85 | S_DIST_03 VARCHAR(32) DEFAULT NULL, 86 | S_DIST_04 VARCHAR(32) DEFAULT NULL, 87 | S_DIST_05 VARCHAR(32) DEFAULT NULL, 88 | S_DIST_06 VARCHAR(32) DEFAULT NULL, 89 | S_DIST_07 VARCHAR(32) DEFAULT NULL, 90 | S_DIST_08 VARCHAR(32) DEFAULT NULL, 91 | S_DIST_09 VARCHAR(32) DEFAULT NULL, 92 | S_DIST_10 VARCHAR(32) DEFAULT NULL, 93 | S_YTD INTEGER DEFAULT NULL, 94 | S_ORDER_CNT INTEGER DEFAULT NULL, 95 | S_REMOTE_CNT INTEGER DEFAULT NULL, 96 | S_DATA VARCHAR(64) DEFAULT NULL, 97 | PRIMARY KEY (S_W_ID,S_I_ID) 98 | ); 99 | 100 | CREATE TABLE ORDERS ( 101 | O_ID INTEGER DEFAULT '0' NOT NULL, 102 | O_C_ID INTEGER DEFAULT NULL, 103 | O_D_ID TINYINT DEFAULT '0' NOT NULL, 104 | O_W_ID SMALLINT DEFAULT '0' NOT NULL, 105 | O_ENTRY_D TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 106 | O_CARRIER_ID INTEGER DEFAULT NULL, 107 | O_OL_CNT INTEGER DEFAULT NULL, 108 | O_ALL_LOCAL INTEGER DEFAULT NULL, 109 | PRIMARY KEY (O_W_ID,O_D_ID,O_ID), 110 | UNIQUE (O_W_ID,O_D_ID,O_C_ID,O_ID), 111 | CONSTRAINT O_FKEY_C FOREIGN KEY (O_C_ID, O_D_ID, O_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID) 112 | ); 113 | CREATE INDEX IDX_ORDERS ON ORDERS (O_W_ID,O_D_ID,O_C_ID); 114 | 115 | CREATE TABLE NEW_ORDER ( 116 | NO_O_ID INTEGER DEFAULT '0' NOT NULL, 117 | NO_D_ID TINYINT DEFAULT '0' NOT NULL, 118 | NO_W_ID SMALLINT DEFAULT '0' NOT NULL, 119 | CONSTRAINT NO_PK_TREE PRIMARY KEY (NO_D_ID,NO_W_ID,NO_O_ID), 120 | CONSTRAINT NO_FKEY_O FOREIGN KEY (NO_O_ID, NO_D_ID, NO_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID) 121 | ); 122 | 123 | CREATE TABLE ORDER_LINE ( 124 | OL_O_ID INTEGER DEFAULT '0' NOT NULL, 125 | OL_D_ID TINYINT DEFAULT '0' NOT NULL, 126 | OL_W_ID SMALLINT DEFAULT '0' NOT NULL, 127 | OL_NUMBER INTEGER DEFAULT '0' NOT NULL, 128 | OL_I_ID INTEGER DEFAULT NULL, 129 | OL_SUPPLY_W_ID SMALLINT DEFAULT NULL, 130 | OL_DELIVERY_D TIMESTAMP DEFAULT NULL, 131 | OL_QUANTITY INTEGER DEFAULT NULL, 132 | OL_AMOUNT FLOAT DEFAULT NULL, 133 | OL_DIST_INFO VARCHAR(32) DEFAULT NULL, 134 | PRIMARY KEY (OL_W_ID,OL_D_ID,OL_O_ID,OL_NUMBER), 135 | CONSTRAINT OL_FKEY_O FOREIGN KEY (OL_O_ID, OL_D_ID, OL_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID), 136 | CONSTRAINT OL_FKEY_S FOREIGN KEY (OL_I_ID, OL_SUPPLY_W_ID) REFERENCES STOCK (S_I_ID, S_W_ID) 137 | ); 138 | --CREATE INDEX IDX_ORDER_LINE_3COL ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); 139 | --CREATE INDEX IDX_ORDER_LINE_2COL ON ORDER_LINE (OL_W_ID,OL_D_ID); 140 | CREATE INDEX IDX_ORDER_LINE_TREE ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); 141 | -------------------------------------------------------------------------------- /pytpcc/tpcc_postgresql.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE WAREHOUSE ( 2 | W_ID SMALLINT DEFAULT '0' NOT NULL, 3 | W_NAME VARCHAR(16) DEFAULT NULL, 4 | W_STREET_1 VARCHAR(32) DEFAULT NULL, 5 | W_STREET_2 VARCHAR(32) DEFAULT NULL, 6 | W_CITY VARCHAR(32) DEFAULT NULL, 7 | W_STATE VARCHAR(2) DEFAULT NULL, 8 | W_ZIP VARCHAR(9) DEFAULT NULL, 9 | W_TAX FLOAT DEFAULT NULL, 10 | W_YTD FLOAT DEFAULT NULL, 11 | CONSTRAINT W_PK_ARRAY PRIMARY KEY (W_ID) 12 | ); 13 | 14 | CREATE TABLE DISTRICT ( 15 | D_ID SMALLINT DEFAULT '0' NOT NULL, 16 | D_W_ID SMALLINT DEFAULT '0' NOT NULL REFERENCES WAREHOUSE (W_ID), 17 | D_NAME VARCHAR(16) DEFAULT NULL, 18 | D_STREET_1 VARCHAR(32) DEFAULT NULL, 19 | D_STREET_2 VARCHAR(32) DEFAULT NULL, 20 | D_CITY VARCHAR(32) DEFAULT NULL, 21 | D_STATE VARCHAR(2) DEFAULT NULL, 22 | D_ZIP VARCHAR(9) DEFAULT NULL, 23 | D_TAX FLOAT DEFAULT NULL, 24 | D_YTD FLOAT DEFAULT NULL, 25 | D_NEXT_O_ID INT DEFAULT NULL, 26 | PRIMARY KEY (D_W_ID,D_ID) 27 | ); 28 | 29 | CREATE TABLE ITEM ( 30 | I_ID INTEGER DEFAULT '0' NOT NULL, 31 | I_IM_ID INTEGER DEFAULT NULL, 32 | I_NAME VARCHAR(32) DEFAULT NULL, 33 | I_PRICE FLOAT DEFAULT NULL, 34 | I_DATA VARCHAR(64) DEFAULT NULL, 35 | CONSTRAINT I_PK_ARRAY PRIMARY KEY (I_ID) 36 | ); 37 | 38 | CREATE TABLE CUSTOMER ( 39 | C_ID INTEGER DEFAULT '0' NOT NULL, 40 | C_D_ID SMALLINT DEFAULT '0' NOT NULL, 41 | C_W_ID SMALLINT DEFAULT '0' NOT NULL, 42 | C_FIRST VARCHAR(32) DEFAULT NULL, 43 | C_MIDDLE VARCHAR(2) DEFAULT NULL, 44 | C_LAST VARCHAR(32) DEFAULT NULL, 45 | C_STREET_1 VARCHAR(32) DEFAULT NULL, 46 | C_STREET_2 VARCHAR(32) DEFAULT NULL, 47 | C_CITY VARCHAR(32) DEFAULT NULL, 48 | C_STATE VARCHAR(2) DEFAULT NULL, 49 | C_ZIP VARCHAR(9) DEFAULT NULL, 50 | C_PHONE VARCHAR(32) DEFAULT NULL, 51 | C_SINCE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 52 | C_CREDIT VARCHAR(2) DEFAULT NULL, 53 | C_CREDIT_LIM FLOAT DEFAULT NULL, 54 | C_DISCOUNT FLOAT DEFAULT NULL, 55 | C_BALANCE FLOAT DEFAULT NULL, 56 | C_YTD_PAYMENT FLOAT DEFAULT NULL, 57 | C_PAYMENT_CNT INTEGER DEFAULT NULL, 58 | C_DELIVERY_CNT INTEGER DEFAULT NULL, 59 | C_DATA VARCHAR(500), 60 | PRIMARY KEY (C_W_ID,C_D_ID,C_ID), 61 | UNIQUE (C_W_ID,C_D_ID,C_LAST,C_FIRST), 62 | CONSTRAINT C_FKEY_D FOREIGN KEY (C_D_ID, C_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) 63 | ); 64 | CREATE INDEX IDX_CUSTOMER ON CUSTOMER (C_W_ID,C_D_ID,C_LAST); 65 | 66 | CREATE TABLE HISTORY ( 67 | H_C_ID INTEGER DEFAULT NULL, 68 | H_C_D_ID SMALLINT DEFAULT NULL, 69 | H_C_W_ID SMALLINT DEFAULT NULL, 70 | H_D_ID SMALLINT DEFAULT NULL, 71 | H_W_ID SMALLINT DEFAULT '0' NOT NULL, 72 | H_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 73 | H_AMOUNT FLOAT DEFAULT NULL, 74 | H_DATA VARCHAR(32) DEFAULT NULL, 75 | CONSTRAINT H_FKEY_C FOREIGN KEY (H_C_ID, H_C_D_ID, H_C_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID), 76 | CONSTRAINT H_FKEY_D FOREIGN KEY (H_D_ID, H_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) 77 | ); 78 | 79 | CREATE TABLE STOCK ( 80 | S_I_ID INTEGER DEFAULT '0' NOT NULL REFERENCES ITEM (I_ID), 81 | S_W_ID SMALLINT DEFAULT '0 ' NOT NULL REFERENCES WAREHOUSE (W_ID), 82 | S_QUANTITY INTEGER DEFAULT '0' NOT NULL, 83 | S_DIST_01 VARCHAR(32) DEFAULT NULL, 84 | S_DIST_02 VARCHAR(32) DEFAULT NULL, 85 | S_DIST_03 VARCHAR(32) DEFAULT NULL, 86 | S_DIST_04 VARCHAR(32) DEFAULT NULL, 87 | S_DIST_05 VARCHAR(32) DEFAULT NULL, 88 | S_DIST_06 VARCHAR(32) DEFAULT NULL, 89 | S_DIST_07 VARCHAR(32) DEFAULT NULL, 90 | S_DIST_08 VARCHAR(32) DEFAULT NULL, 91 | S_DIST_09 VARCHAR(32) DEFAULT NULL, 92 | S_DIST_10 VARCHAR(32) DEFAULT NULL, 93 | S_YTD INTEGER DEFAULT NULL, 94 | S_ORDER_CNT INTEGER DEFAULT NULL, 95 | S_REMOTE_CNT INTEGER DEFAULT NULL, 96 | S_DATA VARCHAR(64) DEFAULT NULL, 97 | PRIMARY KEY (S_W_ID,S_I_ID) 98 | ); 99 | 100 | CREATE TABLE ORDERS ( 101 | O_ID INTEGER DEFAULT '0' NOT NULL, 102 | O_C_ID INTEGER DEFAULT NULL, 103 | O_D_ID SMALLINT DEFAULT '0' NOT NULL, 104 | O_W_ID SMALLINT DEFAULT '0' NOT NULL, 105 | O_ENTRY_D TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 106 | O_CARRIER_ID INTEGER DEFAULT NULL, 107 | O_OL_CNT INTEGER DEFAULT NULL, 108 | O_ALL_LOCAL INTEGER DEFAULT NULL, 109 | PRIMARY KEY (O_W_ID,O_D_ID,O_ID), 110 | UNIQUE (O_W_ID,O_D_ID,O_C_ID,O_ID), 111 | CONSTRAINT O_FKEY_C FOREIGN KEY (O_C_ID, O_D_ID, O_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID) 112 | ); 113 | CREATE INDEX IDX_ORDERS ON ORDERS (O_W_ID,O_D_ID,O_C_ID); 114 | 115 | CREATE TABLE NEW_ORDER ( 116 | NO_O_ID INTEGER DEFAULT '0' NOT NULL, 117 | NO_D_ID SMALLINT DEFAULT '0' NOT NULL, 118 | NO_W_ID SMALLINT DEFAULT '0' NOT NULL, 119 | CONSTRAINT NO_PK_TREE PRIMARY KEY (NO_D_ID,NO_W_ID,NO_O_ID), 120 | CONSTRAINT NO_FKEY_O FOREIGN KEY (NO_O_ID, NO_D_ID, NO_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID) 121 | ); 122 | 123 | CREATE TABLE ORDER_LINE ( 124 | OL_O_ID INTEGER DEFAULT '0' NOT NULL, 125 | OL_D_ID SMALLINT DEFAULT '0' NOT NULL, 126 | OL_W_ID SMALLINT DEFAULT '0' NOT NULL, 127 | OL_NUMBER INTEGER DEFAULT '0' NOT NULL, 128 | OL_I_ID INTEGER DEFAULT NULL, 129 | OL_SUPPLY_W_ID SMALLINT DEFAULT NULL, 130 | OL_DELIVERY_D TIMESTAMP DEFAULT NULL, 131 | OL_QUANTITY INTEGER DEFAULT NULL, 132 | OL_AMOUNT FLOAT DEFAULT NULL, 133 | OL_DIST_INFO VARCHAR(32) DEFAULT NULL, 134 | PRIMARY KEY (OL_W_ID,OL_D_ID,OL_O_ID,OL_NUMBER), 135 | CONSTRAINT OL_FKEY_O FOREIGN KEY (OL_O_ID, OL_D_ID, OL_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID), 136 | CONSTRAINT OL_FKEY_S FOREIGN KEY (OL_I_ID, OL_SUPPLY_W_ID) REFERENCES STOCK (S_I_ID, S_W_ID) 137 | ); 138 | --CREATE INDEX IDX_ORDER_LINE_3COL ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); 139 | --CREATE INDEX IDX_ORDER_LINE_2COL ON ORDER_LINE (OL_W_ID,OL_D_ID); 140 | CREATE INDEX IDX_ORDER_LINE_TREE ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## TPC-C in Python for MongoDB 2 | Approved in July of 1992, TPC Benchmark C is an on-line transaction processing (OLTP) benchmark. TPC-C is more complex than previous OLTP benchmarks such as TPC-A because of its multiple transaction types, more complex database and overall execution structure. TPC-C involves a mix of five concurrent transactions of different types and complexity either executed on-line or queued for deferred execution. The database is comprised of nine types of tables with a wide range of record and population sizes. TPC-C is measured in transactions per minute (tpmC). While the benchmark portrays the activity of a wholesale supplier, TPC-C is not limited to the activity of any particular business segment, but, rather represents any industry that must manage, sell, or distribute a product or service. 3 | 4 | To learn more about TPC-C, please see the [TPC-C](https://www.tpc.org/tpcc/) documentation. 5 | 6 | This repo is an experimental variant of Python TPC-C implementation based on the original [here](http://github.com/apavlo/py-tpcc). 7 | 8 | The structure of the repo is: 9 | 10 | 1. **pytpcc** - the code for pytpcc with driver (DB) specific code in **drivers** subdirectory. 11 | 2. **vldb2019** - 2019 VLDB paper, poster and results generated from this code 12 | * [VLDB Paper](vldb2019/paper.pdf) 13 | * [VLDB Poster](vldb2019/poster.pdf) 14 | * [Result directory](vldb2019/results) 15 | 16 | All the tests were run using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas?jmp=VLDB2019). 17 | Use code `VLDB2019` to get $150 credit to get started with MongoDB Atlas. 18 | 19 | ## Sharded MongoDB Driver 20 | 21 | 1. Create ana activate a python env. 22 | 23 | ```bash 24 | mkdir ~/python_envs 25 | cd ~/python_envs 26 | ~/python_envs$ python -m venv py-tpcc-env 27 | source ~/python_envs/py-tpcc-env/bin/activate 28 | ``` 29 | 2. Install pymongo 30 | 31 | ```bash 32 | pip install pymongo 33 | ``` 34 | 35 | 3. Print your config. 36 | 37 | ```bash 38 | cd ~/py-tpcc/pytpcc 39 | ~/py-tpcc/pytpcc$ python ./tpcc.py --print-config mongodb > mongodb.config 40 | ``` 41 | 42 | 4. Edit the configuration for Postgres in the mongodb.config. 43 | * Change shards to the number of `shards` 44 | * Change the mongodb connection `uri` string 45 | * Change the database `name` 46 | 47 | ```bash 48 | # MongodbDriver Configuration File 49 | # Created 2025-10-08 14:18:24.378446 50 | [mongodb] 51 | 52 | # The mongodb connection string or URI 53 | uri = mongodb://user:pass@10.2.1.119:27017/admin?ssl=true&tlsAllowInvalidHostnames=true&tlsAllowInvalidCertificates=true 54 | 55 | # Database name 56 | name = tpcc 57 | 58 | # If true, data will be denormalized using MongoDB schema design best practices 59 | denormalize = True 60 | 61 | # If true, transactions will not be used (benchmarking only) 62 | notransactions = 63 | 64 | # If true, all things to update will be fetched via findAndModify 65 | findandmodify = True 66 | 67 | # If true, aggregation queries will be used 68 | agg = 69 | 70 | # If true, we will allow secondary reads 71 | secondary_reads = True 72 | 73 | # If true, we will enable retryable writes 74 | retry_writes = True 75 | 76 | # If true, we will perform causal reads 77 | causal_consistency = True 78 | 79 | # If true, we will have use only one 'unsharded' items collection 80 | no_global_items = 81 | 82 | # If > 0 then sharded 83 | shards = 3 84 | ``` 85 | 86 | 4. Run pytpcc using --warehouses=XXX 87 | 88 | * Reset the database and load the data 89 | ```bash 90 | python ./tpcc.py --reset --no-execute --clients=100 --duration=10 --warehouses=21 --config=mongodb.config mongodb --stop-on-error 91 | ``` 92 | 93 | * Only load the data 94 | ```bash 95 | python ./tpcc.py --no-execute --clients=100 --duration=10 --warehouses=21 --config=mongodb.config mongodb --stop-on-error 96 | ``` 97 | 98 | * Execute the tests without loading data. 99 | ```bash 100 | python ./tpcc.py --no-load --clients=100 --duration=10 --warehouses=21 --config=mongodb.config mongodb --stop-on-error 101 | ``` 102 | 103 | * Execute the tests with loading 104 | ```bash 105 | python ./tpcc.py --clients=100 --duration=10 --warehouses=21 --config=mongodb.config mongodb --stop-on-error 106 | ``` 107 | 108 | ## Postgres JSONB Driver 109 | 110 | This branch contains a Postgres JSONB Driver. 111 | 112 | Steps to run the PostgreSQL JSONB Driver 113 | 114 | 1. Start Postgres. 115 | 116 | ```bash 117 | sudo systemctl start postgresql 118 | ``` 119 | 120 | 2. Create ana activate a python env. 121 | 122 | ```bash 123 | mkdir ~/python_envs 124 | cd ~/python_envs 125 | ~/python_envs$ python -m venv py-tpcc-env 126 | source ~/python_envs/py-tpcc-env/bin/activate 127 | ``` 128 | 129 | 3. Print your config. 130 | 131 | ```bash 132 | cd ~/py-tpcc/pytpcc 133 | ~/py-tpcc/pytpcc$ python ./tpcc.py --print-config postgresqljsonb > postgresqljsonb.config 134 | ``` 135 | 136 | 3. Edit the configuraiton for Postgres in the postgresqljsonb.config. Add a password. 137 | 138 | ```bash 139 | # PostgresqljsonbDriver Configuration File 140 | # Created 2025-03-18 23:00:45.340852 141 | [postgresqljsonb] 142 | 143 | # The name of the PostgreSQL database 144 | database = tpcc 145 | 146 | # The host address of the PostgreSQL server 147 | host = localhost 148 | 149 | # The port number of the PostgreSQL server 150 | port = 5432 151 | 152 | # The username to connect to the PostgreSQL database 153 | user = postgres 154 | 155 | # The password to connect to the PostgreSQL database 156 | password = 157 | ``` 158 | 159 | 4. Run the PostgreSQL JSONB driver tests with resetting the database. 160 | 161 | ```bash 162 | ~/py-tpcc/pytpcc$ python ./tpcc.py --reset --clients=1 --duration=1 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb --stop-on-error 163 | ``` 164 | 165 | 5. Run the PostgreSQL JSONB driver tests with no load phase to use the data that is already loaded in the Postgres database. 166 | 167 | ```bash 168 | ~/py-tpcc/pytpcc$ python ./tpcc.py --no-load --clients=1 --duration=1 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb --stop-on-error 169 | ``` 170 | 171 | 6. If you need to connect to Postgres and check the database size 172 | 173 | ```bash 174 | psql -U postgres # and type the password 175 | postgres=# \l+ 176 | 177 | # For any SQL command first use the database 178 | \c tpcc; 179 | ``` 180 | -------------------------------------------------------------------------------- /pytpcc/drivers/abstractdriver.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining 8 | # a copy of this software and associated documentation files (the 9 | # "Software"), to deal in the Software without restriction, including 10 | # without limitation the rights to use, copy, modify, merge, publish, 11 | # distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so, subject to 13 | # the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 21 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | # OTHER DEALINGS IN THE SOFTWARE. 25 | # ----------------------------------------------------------------------- 26 | 27 | from datetime import datetime 28 | 29 | import constants 30 | 31 | ## ============================================== 32 | ## AbstractDriver 33 | ## ============================================== 34 | class AbstractDriver(object): 35 | def __init__(self, name, ddl): 36 | self.name = name 37 | self.driver_name = "%sDriver" % self.name.title() 38 | self.ddl = ddl 39 | 40 | def __str__(self): 41 | return self.driver_name 42 | 43 | def makeDefaultConfig(self): 44 | """This function needs to be implemented by all sub-classes. 45 | It should return the items that need to be in your implementation's configuration file. 46 | Each item in the list is a triplet containing: ( , , ) 47 | """ 48 | raise NotImplementedError("%s does not implement makeDefaultConfig" % (self.driver_name)) 49 | 50 | def loadConfig(self, config): 51 | """Initialize the driver using the given configuration dict""" 52 | raise NotImplementedError("%s does not implement loadConfig" % (self.driver_name)) 53 | 54 | def formatConfig(self, config): 55 | """Return a formatted version of the config dict that can be used with the --config command line argument""" 56 | ret = "# %s Configuration File\n" % (self.driver_name) 57 | ret += "# Created %s\n" % (datetime.now()) 58 | ret += "[%s]" % self.name 59 | 60 | for name in config.keys(): 61 | desc, default = config[name] 62 | if not default: 63 | default = "" 64 | ret += "\n\n# %s\n%-20s = %s" % (desc, name, default) 65 | return ret 66 | 67 | def loadStart(self): 68 | """Optional callback to indicate to the driver that the data loading phase is about to begin.""" 69 | return None 70 | 71 | def loadFinish(self): 72 | """Optional callback to indicate to the driver that the data loading phase is finished.""" 73 | return None 74 | 75 | def loadFinishItem(self): 76 | """Optional callback to indicate to the driver that the ITEM data has been passed to the driver.""" 77 | return None 78 | 79 | def loadFinishWarehouse(self, w_id): 80 | """Optional callback to indicate to the driver that the data for the given warehouse is finished.""" 81 | return None 82 | 83 | def loadFinishDistrict(self, w_id, d_id): 84 | """Optional callback to indicate to the driver that the data for the given district is finished.""" 85 | return None 86 | 87 | def loadTuples(self, tableName, tuples): 88 | """Load a list of tuples into the target table""" 89 | raise NotImplementedError("%s does not implement loadTuples" % (self.driver_name)) 90 | 91 | def executeStart(self): 92 | """Optional callback before the execution phase starts""" 93 | return None 94 | 95 | def executeFinish(self): 96 | """Callback after the execution phase finishes""" 97 | return None 98 | 99 | def executeTransaction(self, txn, params): 100 | """Execute a transaction based on the given name""" 101 | 102 | if constants.TransactionTypes.DELIVERY == txn: 103 | result = self.doDelivery(params) 104 | elif constants.TransactionTypes.NEW_ORDER == txn: 105 | result = self.doNewOrder(params) 106 | elif constants.TransactionTypes.ORDER_STATUS == txn: 107 | result = self.doOrderStatus(params) 108 | elif constants.TransactionTypes.PAYMENT == txn: 109 | result = self.doPayment(params) 110 | elif constants.TransactionTypes.STOCK_LEVEL == txn: 111 | result = self.doStockLevel(params) 112 | else: 113 | assert False, "Unexpected TransactionType: " + txn 114 | return result 115 | 116 | def doDelivery(self, params): 117 | """Execute DELIVERY Transaction 118 | Parameters Dict: 119 | w_id 120 | o_carrier_id 121 | ol_delivery_d 122 | """ 123 | raise NotImplementedError("%s does not implement doDelivery" % (self.driver_name)) 124 | 125 | def doNewOrder(self, params): 126 | """Execute NEW_ORDER Transaction 127 | Parameters Dict: 128 | w_id 129 | d_id 130 | c_id 131 | o_entry_d 132 | i_ids 133 | i_w_ids 134 | i_qtys 135 | """ 136 | raise NotImplementedError("%s does not implement doNewOrder" % (self.driver_name)) 137 | 138 | def doOrderStatus(self, params): 139 | """Execute ORDER_STATUS Transaction 140 | Parameters Dict: 141 | w_id 142 | d_id 143 | c_id 144 | c_last 145 | """ 146 | raise NotImplementedError("%s does not implement doOrderStatus" % (self.driver_name)) 147 | 148 | def doPayment(self, params): 149 | """Execute PAYMENT Transaction 150 | Parameters Dict: 151 | w_id 152 | d_id 153 | h_amount 154 | c_w_id 155 | c_d_id 156 | c_id 157 | c_last 158 | h_date 159 | """ 160 | raise NotImplementedError("%s does not implement doPayment" % (self.driver_name)) 161 | 162 | def doStockLevel(self, params): 163 | """Execute STOCK_LEVEL Transaction 164 | Parameters Dict: 165 | w_id 166 | d_id 167 | threshold 168 | """ 169 | raise NotImplementedError("%s does not implement doStockLevel" % (self.driver_name)) 170 | ## CLASS 171 | -------------------------------------------------------------------------------- /pytpcc/verify.js: -------------------------------------------------------------------------------- 1 | function checkConsistency(rule, collection, agg, sample=false) { 2 | agg.push({$count:"c"}); 3 | if (sample) { 4 | var c=db.WAREHOUSE.count(); 5 | var start=Math.round(Math.random()*c); 6 | var firstStage={$match:{}}; 7 | var wid="W_ID"; 8 | var did="D_ID"; 9 | if (collection=="NEW_ORDER") { 10 | wid="NO_"+wid; 11 | did="NO_"+did; 12 | } else if (collection=="WAREHOUSE") { 13 | did=collection[0]+"_"+did; 14 | } else if (collection=="DISTRICT") { 15 | wid=collection[0]+"_"+wid; 16 | } else { 17 | wid=collection[0]+"_"+wid; 18 | did=collection[0]+"_"+did; 19 | } 20 | firstStage["$match"][wid]=start; 21 | /* firstStage["$match"][did]=(start%10)+1; */ 22 | agg.unshift(firstStage); 23 | } 24 | var r = db.getCollection(collection).aggregate(agg, {maxTimeMS:60000}).toArray(); 25 | if (r.length != 0) { 26 | print("Consistency rule " + rule + " failed with count " + r[0]["c"]); 27 | } 28 | } 29 | 30 | c=[]; 31 | c[1]={r:"1", c:"WAREHOUSE", pipeline: [ 32 | {$lookup:{ 33 | from:"DISTRICT", 34 | as:"d", 35 | localField:"W_ID", 36 | foreignField:"D_W_ID"}}, 37 | {$match:{$expr:{$ne:[0,{$toLong:{$subtract:["$W_YTD",{$sum:"$d.D_YTD"}]}}]}}}, 38 | ]}; 39 | // consistency check 2 40 | c[2]={r:"2", c:"DISTRICT", pipeline: [ 41 | {$project:{w:"$D_W_ID", d:"$D_ID",next:"$D_NEXT_O_ID"}}, 42 | {$lookup:{from:"ORDERS",as:"maxOID",let:{w:"$w",d:"$d"},pipeline:[ 43 | {$match:{$expr:{$and:[{$eq:["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}]}}}, 44 | {$sort:{"O_ID":-1}}, 45 | {$limit:1}, 46 | {$group:{_id:0,maxO:{$first:"$O_ID"}}}]}}, 47 | {$unwind:"$maxOID"}, 48 | {$lookup:{from:"NEW_ORDER",as:"maxNOID",let:{w:"$w",d:"$d"},pipeline:[ 49 | {$match:{$expr:{$and:[{$eq:["$$w","$NO_W_ID"]},{$eq:["$$d","$NO_D_ID"]}]}}}, 50 | {$sort:{"NO_O_ID":-1}}, 51 | {$limit:1}, 52 | {$group:{_id:0,maxO:{$max:"$NO_O_ID"}}}]}}, 53 | {$unwind:"$maxNOID"}, 54 | {$match:{$or:[{$expr:{$ne:["$maxOID.maxO","$maxNOID.maxO"]}}, {$expr:{$ne:[ "$maxOID.maxO",{$subtract:["$next",1]}]}} ]}}, 55 | ]}; 56 | // consistency check 3 57 | c[3]={r:"3", c:"NEW_ORDER", pipeline:[ 58 | {$group:{_id:{w:"$NO_W_ID",d:"$NO_D_ID"},min:{$min:"$NO_O_ID"},max:{$max:"$NO_O_ID"},count:{$sum:1}}}, 59 | {$project:{count:1, diff:{$add:[1,{$subtract:["$max","$min"]}]}}}, 60 | {$match:{$expr:{$ne:["$count","$diff"]}}}, 61 | ]}; 62 | // consistency check 4 63 | c[4]={r:"4", c:"ORDERS", pipeline:[ 64 | {$sort:{O_W_ID:1, O_D_ID:1}}, 65 | {$limit:10000}, 66 | {$group:{_id:{w:"$O_W_ID",d:"$O_D_ID"},O_CLs:{$sum:"$O_OL_CNT"}, OL_CLs:{$sum:{$size:"$ORDER_LINE"}}}}, 67 | {$match:{$expr:{$ne:[ "$O_CLs","$OL_CLs"]}}}, 68 | ]}; 69 | // consistency check 5 70 | c[5]={r:"5", c:"ORDERS", pipeline:[ 71 | {$match:{O_CARRIER_ID:0}}, 72 | {$sort:{O_ID:-1}}, 73 | {$limit:10000}, 74 | {$lookup:{from:"NEW_ORDER", as:"NO_count",let:{w:"$O_W_ID",d:"$O_D_ID",o:"$O_ID"}, pipeline:[ 75 | {$match:{$expr:{$and:[{$eq:["$$w","$NO_W_ID"]},{$eq:["$$d","$NO_D_ID"]},{$eq:["$$o","$NO_O_ID"]} ]}}}, 76 | {$count:"c"}]}}, 77 | {$addFields:{count:{$ifNull:[{$arrayElemAt:["$NO_count.c",0]},0]}}}, 78 | {$match:{"count":{$ne:1}}}, 79 | ]}; 80 | // consistency check 6 81 | c[6]={r:"6", c:"ORDERS", pipeline:[ 82 | {$limit:10000}, 83 | {$match:{$expr:{$ne:[ "$O_OL_CNT",{$size:"$ORDER_LINE"}]}}}, 84 | ]}; 85 | // consistency check 7 86 | c[7]={r:"7", c:"ORDERS", pipeline:[ 87 | {$limit:10000}, 88 | {$match:{O_CARRIER_ID:{$ne:0}, "ORDER_LINE.OL_DELIVERY_D":null}}, 89 | ]}; 90 | // consistency 91 | c[8]={r:"8", c:"HISTORY", pipeline:[ 92 | {$group:{_id:"$H_W_ID",sum:{$sum:{$toDecimal:"$H_AMOUNT"}}}}, 93 | {$lookup:{from:"WAREHOUSE", localField:"_id", foreignField:"W_ID",as:"w"}}, 94 | {$match:{$expr:{$ne:[0, {$toLong:{$subtract:["$sum", {$arrayElemAt:["$w.W_YTD",0]}]}}]}}}, 95 | ]}; 96 | // consistency check 9 97 | c[9]={r:"9", c:"HISTORY", pipeline:[ 98 | {$group:{_id:{w:"$H_W_ID", d:"$H_D_ID"}, sum:{$sum:{$toDecimal:"$H_AMOUNT"}}}}, 99 | {$lookup:{from:"DISTRICT", as:"d", let:{ w: "$_id.w", d:"$_id.d"}, pipeline:[ 100 | {$match: {$expr: {$and: [ {$eq: ["$$w","$D_W_ID"]},{$eq:["$$d","$D_ID" ]}]}}}, 101 | {$group:{_id:0, sum:{$sum:{$toDecimal:"$D_YTD"}}}}]}}, 102 | {$match:{$expr:{$ne:[{$toLong:"$sum"},{$toLong:{$arrayElemAt:["$d.sum",0]}}]}}}, 103 | ]}; 104 | // *** consistency check 10 don't run unless there is an index 105 | /* adding one warehouse filter to limit checking, needed to add index to HISTORY.H_W_ID,etc to make reasonably fast even on one */ 106 | c[10]={r:"10", c:"CUSTOMER", pipeline:[ 107 | {$match:{C_W_ID:1,C_D_ID:1,C_ID:{$lt:100,$gt:80}}}, 108 | {$lookup:{from:"ORDERS", as:"o", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[ 109 | {$match: {O_CARRIER_ID:{$ne:0}, $expr: {$and: [ {$eq: ["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}, {$eq:["$$c","$O_C_ID"]}]}}}, 110 | {$group:{_id:0, sum:{$sum:{$sum:"$ORDER_LINE.OL_AMOUNT"}}}}]}}, 111 | {$lookup:{from:"HISTORY", as:"h", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[ 112 | {$match: {$expr: {$and: [ {$eq: ["$$w","$H_W_ID"]},{$eq:["$$d","$H_D_ID"]}, {$eq:["$$c","$H_C_ID"]}]}}}, 113 | {$group:{_id:0, sum:{$sum:"$H_AMOUNT"}}}]}}, 114 | {$project:{C_BALANCE:1, OSUM:{$ifNull:[{$arrayElemAt:["$o.sum",0]},0]},HSUM:{$arrayElemAt:["$h.sum",0]},_id:0, C_ID:1, C_W_ID:1, C_D_ID:1}}, 115 | {$match:{$expr:{$ne:["$C_BALANCE", {$subtract:["$OSUM","$HSUM"]}]}}}, 116 | ]}; 117 | // *** consistency check 11 Correct when first loaded! 118 | c[11]={r:"11", c:"DISTRICT", pipeline:[ 119 | {$project:{w:"$D_W_ID", d:"$D_ID"}}, 120 | {$lookup:{from:"ORDERS",as:"o",let:{w:"$w",d:"$d"},pipeline:[ 121 | {$match:{$expr:{$and:[{$eq:["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}]}}}, 122 | {$count:"c"}]}}, 123 | {$unwind:"$o"}, 124 | {$lookup:{from:"NEW_ORDER",as:"no",let:{w:"$w",d:"$d"},pipeline:[ 125 | {$match:{$expr:{$and:[{$eq:["$$w","$NO_W_ID"]},{$eq:["$$d","$NO_D_ID"]}]}}}, 126 | {$count:"c"}]}}, 127 | {$unwind:"$no"}, 128 | {$match:{$expr:{$ne:[2100, {$subtract:["$o.c","$no.c"]}]}}}, 129 | ]}; 130 | // consistency check 12 131 | c[12]={r:"12", c:"CUSTOMER", pipeline:[ 132 | {$lookup:{from:"ORDERS", as:"o", let:{ w: "$C_W_ID", d:"$C_D_ID", c:"$C_ID"}, pipeline:[ 133 | {$match: {O_CARRIER_ID:{$ne:0},$expr: {$and: [ {$eq: ["$$w","$O_W_ID"]},{$eq:["$$d","$O_D_ID"]}, {$eq:["$$c","$O_C_ID"]}]}}}, 134 | {$group:{_id:0, sum:{$sum:{$sum:"$ORDER_LINE.OL_AMOUNT"}}}}]}}, 135 | {$project:{C_BALANCE:1, C_YTD_PAYMENT:1, OLSUM:{$ifNull:[{$arrayElemAt:["$o.sum",0]},0]}}}, 136 | {$match:{$expr:{$ne:[0,{$toLong:{$subtract:["$OLSUM", {$add:["$C_BALANCE","$C_YTD_PAYMENT"]}]}}]}}}, 137 | ]}; 138 | 139 | for (i=1; i<13; i++) { 140 | print (""+ new ISODate() + " Checking " + i); 141 | if (i==10) continue; 142 | checkConsistency(c[i].r, c[i].c, c[i].pipeline,true); 143 | } 144 | 145 | -------------------------------------------------------------------------------- /pytpcc/tpcc_jsonb.sql: -------------------------------------------------------------------------------- 1 | -- CREATE TABLE ITEM ( 2 | -- I_ID INTEGER DEFAULT '0' NOT NULL, 3 | -- I_IM_ID INTEGER DEFAULT NULL, 4 | -- I_NAME VARCHAR(32) DEFAULT NULL, 5 | -- I_PRICE FLOAT DEFAULT NULL, 6 | -- I_DATA VARCHAR(64) DEFAULT NULL, 7 | -- CONSTRAINT I_PK_ARRAY PRIMARY KEY (I_ID) 8 | -- ); 9 | 10 | CREATE TABLE ITEM ( 11 | data JSONB 12 | ); 13 | 14 | CREATE UNIQUE INDEX item_pkey_idx ON ITEM ( 15 | ((data->>'I_ID')::INTEGER) 16 | ); 17 | 18 | 19 | -- CREATE TABLE WAREHOUSE ( 20 | -- W_ID SMALLINT DEFAULT '0' NOT NULL, 21 | -- W_NAME VARCHAR(16) DEFAULT NULL, 22 | -- W_STREET_1 VARCHAR(32) DEFAULT NULL, 23 | -- W_STREET_2 VARCHAR(32) DEFAULT NULL, 24 | -- W_CITY VARCHAR(32) DEFAULT NULL, 25 | -- W_STATE VARCHAR(2) DEFAULT NULL, 26 | -- W_ZIP VARCHAR(9) DEFAULT NULL, 27 | -- W_TAX FLOAT DEFAULT NULL, 28 | -- W_YTD FLOAT DEFAULT NULL, 29 | -- CONSTRAINT W_PK_ARRAY PRIMARY KEY (W_ID) 30 | -- ); 31 | 32 | CREATE TABLE WAREHOUSE ( 33 | data JSONB 34 | ); 35 | 36 | CREATE UNIQUE INDEX warehouse_pkey_idx ON WAREHOUSE ( 37 | ((data->>'W_ID')::SMALLINT) 38 | ); 39 | 40 | -- CREATE TABLE DISTRICT ( 41 | -- D_ID SMALLINT DEFAULT '0' NOT NULL, 42 | -- D_W_ID SMALLINT DEFAULT '0' NOT NULL REFERENCES WAREHOUSE (W_ID), 43 | -- D_NAME VARCHAR(16) DEFAULT NULL, 44 | -- D_STREET_1 VARCHAR(32) DEFAULT NULL, 45 | -- D_STREET_2 VARCHAR(32) DEFAULT NULL, 46 | -- D_CITY VARCHAR(32) DEFAULT NULL, 47 | -- D_STATE VARCHAR(2) DEFAULT NULL, 48 | -- D_ZIP VARCHAR(9) DEFAULT NULL, 49 | -- D_TAX FLOAT DEFAULT NULL, 50 | -- D_YTD FLOAT DEFAULT NULL, 51 | -- D_NEXT_O_ID INT DEFAULT NULL, 52 | -- PRIMARY KEY (D_W_ID,D_ID) 53 | -- ); 54 | 55 | CREATE TABLE DISTRICT ( 56 | data JSONB 57 | ); 58 | 59 | CREATE UNIQUE INDEX district_pkey_idx ON DISTRICT ( 60 | ((data->>'D_W_ID')::SMALLINT), 61 | ((data->>'D_ID')::SMALLINT) 62 | ); 63 | 64 | -- CREATE TABLE CUSTOMER ( 65 | -- C_ID INTEGER DEFAULT '0' NOT NULL, 66 | -- C_D_ID SMALLINT DEFAULT '0' NOT NULL, 67 | -- C_W_ID SMALLINT DEFAULT '0' NOT NULL, 68 | -- C_FIRST VARCHAR(32) DEFAULT NULL, 69 | -- C_MIDDLE VARCHAR(2) DEFAULT NULL, 70 | -- C_LAST VARCHAR(32) DEFAULT NULL, 71 | -- C_STREET_1 VARCHAR(32) DEFAULT NULL, 72 | -- C_STREET_2 VARCHAR(32) DEFAULT NULL, 73 | -- C_CITY VARCHAR(32) DEFAULT NULL, 74 | -- C_STATE VARCHAR(2) DEFAULT NULL, 75 | -- C_ZIP VARCHAR(9) DEFAULT NULL, 76 | -- C_PHONE VARCHAR(32) DEFAULT NULL, 77 | -- C_SINCE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 78 | -- C_CREDIT VARCHAR(2) DEFAULT NULL, 79 | -- C_CREDIT_LIM FLOAT DEFAULT NULL, 80 | -- C_DISCOUNT FLOAT DEFAULT NULL, 81 | -- C_BALANCE FLOAT DEFAULT NULL, 82 | -- C_YTD_PAYMENT FLOAT DEFAULT NULL, 83 | -- C_PAYMENT_CNT INTEGER DEFAULT NULL, 84 | -- C_DELIVERY_CNT INTEGER DEFAULT NULL, 85 | -- C_DATA VARCHAR(500), 86 | -- PRIMARY KEY (C_W_ID,C_D_ID,C_ID), 87 | -- UNIQUE (C_W_ID,C_D_ID,C_LAST,C_FIRST), 88 | -- CONSTRAINT C_FKEY_D FOREIGN KEY (C_D_ID, C_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) 89 | -- ); 90 | -- CREATE INDEX IDX_CUSTOMER ON CUSTOMER (C_W_ID,C_D_ID,C_LAST); 91 | CREATE TABLE CUSTOMER ( 92 | data JSONB 93 | ); 94 | 95 | CREATE UNIQUE INDEX customer_pkey_idx ON CUSTOMER ( 96 | ((data->>'C_W_ID')::SMALLINT), 97 | ((data->>'C_D_ID')::SMALLINT), 98 | ((data->>'C_ID')::INTEGER) 99 | ); 100 | 101 | CREATE UNIQUE INDEX customer_unique_name_idx ON CUSTOMER ( 102 | ((data->>'C_W_ID')::SMALLINT), 103 | ((data->>'C_D_ID')::SMALLINT), 104 | (data->>'C_LAST'), 105 | (data->>'C_FIRST') 106 | ); 107 | 108 | CREATE INDEX customer_idx_last ON CUSTOMER ( 109 | ((data->>'C_W_ID')::SMALLINT), 110 | ((data->>'C_D_ID')::SMALLINT), 111 | (data->>'C_LAST') 112 | ); 113 | 114 | 115 | -- CREATE TABLE STOCK ( 116 | -- S_I_ID INTEGER DEFAULT '0' NOT NULL REFERENCES ITEM (I_ID), 117 | -- S_W_ID SMALLINT DEFAULT '0 ' NOT NULL REFERENCES WAREHOUSE (W_ID), 118 | -- S_QUANTITY INTEGER DEFAULT '0' NOT NULL, 119 | -- S_DIST_01 VARCHAR(32) DEFAULT NULL, 120 | -- S_DIST_02 VARCHAR(32) DEFAULT NULL, 121 | -- S_DIST_03 VARCHAR(32) DEFAULT NULL, 122 | -- S_DIST_04 VARCHAR(32) DEFAULT NULL, 123 | -- S_DIST_05 VARCHAR(32) DEFAULT NULL, 124 | -- S_DIST_06 VARCHAR(32) DEFAULT NULL, 125 | -- S_DIST_07 VARCHAR(32) DEFAULT NULL, 126 | -- S_DIST_08 VARCHAR(32) DEFAULT NULL, 127 | -- S_DIST_09 VARCHAR(32) DEFAULT NULL, 128 | -- S_DIST_10 VARCHAR(32) DEFAULT NULL, 129 | -- S_YTD INTEGER DEFAULT NULL, 130 | -- S_ORDER_CNT INTEGER DEFAULT NULL, 131 | -- S_REMOTE_CNT INTEGER DEFAULT NULL, 132 | -- S_DATA VARCHAR(64) DEFAULT NULL, 133 | -- PRIMARY KEY (S_W_ID,S_I_ID) 134 | -- ); 135 | CREATE TABLE STOCK ( 136 | data JSONB 137 | ); 138 | 139 | CREATE UNIQUE INDEX stock_pkey_idx ON STOCK ( 140 | ((data->>'S_W_ID')::SMALLINT), 141 | ((data->>'S_I_ID')::INTEGER) 142 | ); 143 | 144 | -- CREATE TABLE HISTORY ( 145 | -- H_C_ID INTEGER DEFAULT NULL, 146 | -- H_C_D_ID SMALLINT DEFAULT NULL, 147 | -- H_C_W_ID SMALLINT DEFAULT NULL, 148 | -- H_D_ID SMALLINT DEFAULT NULL, 149 | -- H_W_ID SMALLINT DEFAULT '0' NOT NULL, 150 | -- H_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 151 | -- H_AMOUNT FLOAT DEFAULT NULL, 152 | -- H_DATA VARCHAR(32) DEFAULT NULL, 153 | -- CONSTRAINT H_FKEY_C FOREIGN KEY (H_C_ID, H_C_D_ID, H_C_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID), 154 | -- CONSTRAINT H_FKEY_D FOREIGN KEY (H_D_ID, H_W_ID) REFERENCES DISTRICT (D_ID, D_W_ID) 155 | -- ); 156 | CREATE TABLE HISTORY ( 157 | data JSONB 158 | ); 159 | 160 | -- CREATE TABLE ORDERS ( 161 | -- O_ID INTEGER DEFAULT '0' NOT NULL, 162 | -- O_C_ID INTEGER DEFAULT NULL, 163 | -- O_D_ID SMALLINT DEFAULT '0' NOT NULL, 164 | -- O_W_ID SMALLINT DEFAULT '0' NOT NULL, 165 | -- O_ENTRY_D TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 166 | -- O_CARRIER_ID INTEGER DEFAULT NULL, 167 | -- O_OL_CNT INTEGER DEFAULT NULL, 168 | -- O_ALL_LOCAL INTEGER DEFAULT NULL, 169 | -- PRIMARY KEY (O_W_ID,O_D_ID,O_ID), 170 | -- UNIQUE (O_W_ID,O_D_ID,O_C_ID,O_ID), 171 | -- CONSTRAINT O_FKEY_C FOREIGN KEY (O_C_ID, O_D_ID, O_W_ID) REFERENCES CUSTOMER (C_ID, C_D_ID, C_W_ID) 172 | -- ); 173 | -- CREATE INDEX IDX_ORDERS ON ORDERS (O_W_ID,O_D_ID,O_C_ID); 174 | CREATE TABLE ORDERS ( 175 | data JSONB 176 | ); 177 | 178 | CREATE UNIQUE INDEX orders_pkey_idx ON ORDERS ( 179 | ((data->>'O_W_ID')::SMALLINT), 180 | ((data->>'O_D_ID')::SMALLINT), 181 | ((data->>'O_ID')::INTEGER) 182 | ); 183 | 184 | CREATE UNIQUE INDEX orders_unique_idx ON ORDERS ( 185 | ((data->>'O_W_ID')::SMALLINT), 186 | ((data->>'O_D_ID')::SMALLINT), 187 | ((data->>'O_C_ID')::INTEGER), 188 | ((data->>'O_ID')::INTEGER) 189 | ); 190 | 191 | CREATE INDEX idx_orders_jsonb ON ORDERS ( 192 | ((data->>'O_W_ID')::SMALLINT), 193 | ((data->>'O_D_ID')::SMALLINT), 194 | ((data->>'O_C_ID')::INTEGER) 195 | ); 196 | 197 | -- CREATE TABLE NEW_ORDER ( 198 | -- NO_O_ID INTEGER DEFAULT '0' NOT NULL, 199 | -- NO_D_ID SMALLINT DEFAULT '0' NOT NULL, 200 | -- NO_W_ID SMALLINT DEFAULT '0' NOT NULL, 201 | -- CONSTRAINT NO_PK_TREE PRIMARY KEY (NO_D_ID,NO_W_ID,NO_O_ID), 202 | -- CONSTRAINT NO_FKEY_O FOREIGN KEY (NO_O_ID, NO_D_ID, NO_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID) 203 | -- ); 204 | CREATE TABLE NEW_ORDER ( 205 | data JSONB 206 | ); 207 | 208 | CREATE UNIQUE INDEX new_order_pkey_idx ON NEW_ORDER ( 209 | ((data->>'NO_D_ID')::SMALLINT), 210 | ((data->>'NO_W_ID')::SMALLINT), 211 | ((data->>'NO_O_ID')::INTEGER) 212 | ); 213 | 214 | -- CREATE TABLE ORDER_LINE ( 215 | -- OL_O_ID INTEGER DEFAULT '0' NOT NULL, 216 | -- OL_D_ID SMALLINT DEFAULT '0' NOT NULL, 217 | -- OL_W_ID SMALLINT DEFAULT '0' NOT NULL, 218 | -- OL_NUMBER INTEGER DEFAULT '0' NOT NULL, 219 | -- OL_I_ID INTEGER DEFAULT NULL, 220 | -- OL_SUPPLY_W_ID SMALLINT DEFAULT NULL, 221 | -- OL_DELIVERY_D TIMESTAMP DEFAULT NULL, 222 | -- OL_QUANTITY INTEGER DEFAULT NULL, 223 | -- OL_AMOUNT FLOAT DEFAULT NULL, 224 | -- OL_DIST_INFO VARCHAR(32) DEFAULT NULL, 225 | -- PRIMARY KEY (OL_W_ID,OL_D_ID,OL_O_ID,OL_NUMBER), 226 | -- CONSTRAINT OL_FKEY_O FOREIGN KEY (OL_O_ID, OL_D_ID, OL_W_ID) REFERENCES ORDERS (O_ID, O_D_ID, O_W_ID), 227 | -- CONSTRAINT OL_FKEY_S FOREIGN KEY (OL_I_ID, OL_SUPPLY_W_ID) REFERENCES STOCK (S_I_ID, S_W_ID) 228 | -- ); 229 | -- --CREATE INDEX IDX_ORDER_LINE_3COL ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); 230 | -- --CREATE INDEX IDX_ORDER_LINE_2COL ON ORDER_LINE (OL_W_ID,OL_D_ID); 231 | -- CREATE INDEX IDX_ORDER_LINE_TREE ON ORDER_LINE (OL_W_ID,OL_D_ID,OL_O_ID); 232 | CREATE TABLE ORDER_LINE ( 233 | data JSONB 234 | ); 235 | 236 | -- Indexes on ORDER_LINE are commented when running denormalized tests. Uncomment when running normalized tests 237 | -- CREATE UNIQUE INDEX order_line_pkey_idx ON ORDER_LINE ( 238 | -- ((data->>'OL_W_ID')::SMALLINT), 239 | -- ((data->>'OL_D_ID')::SMALLINT), 240 | -- ((data->>'OL_O_ID')::INTEGER), 241 | -- ((data->>'OL_NUMBER')::INTEGER) 242 | -- ); 243 | 244 | -- CREATE INDEX order_line_idx_tree ON ORDER_LINE ( 245 | -- ((data->>'OL_W_ID')::SMALLINT), 246 | -- ((data->>'OL_D_ID')::SMALLINT), 247 | -- ((data->>'OL_O_ID')::INTEGER) 248 | -- ); -------------------------------------------------------------------------------- /pytpcc/coordinator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------- 4 | # Copyright (C) 2011 5 | # Andy Pavlo & Yang Lu 6 | # http:##www.cs.brown.edu/~pavlo/ 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining 9 | # a copy of this software and associated documentation files (the 10 | # "Software"), to deal in the Software without restriction, including 11 | # without limitation the rights to use, copy, modify, merge, publish, 12 | # distribute, sublicense, and/or sell copies of the Software, and to 13 | # permit persons to whom the Software is furnished to do so, subject to 14 | # the following conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be 17 | # included in all copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 22 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | # OTHER DEALINGS IN THE SOFTWARE. 26 | # ----------------------------------------------------------------------- 27 | 28 | import sys 29 | import os 30 | import string 31 | import datetime 32 | import logging 33 | import re 34 | import argparse 35 | import glob 36 | import time 37 | import pickle 38 | import execnet 39 | import worker 40 | import message 41 | from configparser import ConfigParser 42 | from pprint import pprint, pformat 43 | 44 | from util import * 45 | from runtime import * 46 | import drivers 47 | 48 | logging.basicConfig(level = logging.INFO, 49 | format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", 50 | datefmt="%m-%d-%Y %H:%M:%S", 51 | filename='results.log') 52 | 53 | ## ============================================== 54 | ## createDriverClass 55 | ## ============================================== 56 | def createDriverClass(name): 57 | full_name = "%sDriver" % name.title() 58 | mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) 59 | klass = getattr(mod, full_name) 60 | return klass 61 | ## DEF 62 | 63 | ## ============================================== 64 | ## getDrivers 65 | ## ============================================== 66 | def getDrivers(): 67 | drivers = [] 68 | for f in map(lambda x: os.path.basename(x).replace("driver.py", ""), glob.glob("./drivers/*driver.py")): 69 | if f != "abstract": drivers.append(f) 70 | return drivers 71 | ## DEF 72 | 73 | ## ============================================== 74 | ## startLoading 75 | ## ============================================== 76 | def startLoading(scalParameters,args,config,channels): 77 | #Split the warehouses into chunks 78 | procs = len(channels) 79 | w_ids = map(lambda x:[], range(procs)) 80 | for w_id in range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1): 81 | idx = w_id % procs 82 | w_ids[idx].append(w_id) 83 | print(w_ids) 84 | 85 | load_start=time.time() 86 | for i in range(len(channels)): 87 | m=message.Message(header=message.CMD_LOAD,data=[scalParameters,args,config,w_ids[i]]) 88 | channels[i].send(pickle.dumps(m,-1)) 89 | for ch in channels: 90 | ch.receive() 91 | pass 92 | return time.time()-load_start 93 | 94 | 95 | ## ============================================== 96 | ## startExecution 97 | ## ============================================== 98 | def startExecution(scaleParameters, args, config,channels): 99 | procs = len(channels) 100 | total_results = results.Results() 101 | 102 | for ch in channels: 103 | m=message.Message(header=message.CMD_EXECUTE,data=[scaleParameters,args,config]) 104 | ch.send(pickle.dumps(m,-1)) 105 | for ch in channels: 106 | r=pickle.loads(ch.receive()).data 107 | total_results.append(r) 108 | return total_results 109 | ## DEF 110 | 111 | 112 | ## ============================================== 113 | ## main 114 | ## ============================================== 115 | if __name__ == '__main__': 116 | aparser = argparse.ArgumentParser(description='Python implementation of the TPC-C Benchmark') 117 | aparser.add_argument('system', choices=getDrivers(), 118 | help='Target system driver') 119 | aparser.add_argument('--config', type=str, 120 | help='Path to driver configuration file') 121 | aparser.add_argument('--reset', action='store_true', 122 | help='Instruct the driver to reset the contents of the database') 123 | aparser.add_argument('--scalefactor', default=1, type=float, metavar='SF', 124 | help='Benchmark scale factor') 125 | aparser.add_argument('--warehouses', default=4, type=int, metavar='W', 126 | help='Number of Warehouses') 127 | aparser.add_argument('--duration', default=60, type=int, metavar='D', 128 | help='How long to run the benchmark in seconds') 129 | aparser.add_argument('--ddl', default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), 130 | help='Path to the TPC-C DDL SQL file') 131 | ## number of processes per node 132 | aparser.add_argument('--clientprocs', default=1, type=int, metavar='N', 133 | help='Number of processes on each client node.') 134 | 135 | aparser.add_argument('--samewh', default=85, type=float, metavar='PP', 136 | help='Percent paying same warehouse') 137 | aparser.add_argument('--stop-on-error', action='store_true', 138 | help='Stop the transaction execution when the driver throws an exception.') 139 | aparser.add_argument('--no-load', action='store_true', 140 | help='Disable loading the data') 141 | aparser.add_argument('--no-execute', action='store_true', 142 | help='Disable executing the workload') 143 | aparser.add_argument('--print-config', action='store_true', 144 | help='Print out the default configuration file for the system and exit') 145 | aparser.add_argument('--debug', action='store_true', 146 | help='Enable debug log messages') 147 | args = vars(aparser.parse_args()) 148 | 149 | if args['debug']: logging.getLogger().setLevel(logging.DEBUG) 150 | 151 | ## Create a handle to the target client driver 152 | driverClass = createDriverClass(args['system']) 153 | assert driverClass != None, "Failed to find '%s' class" % args['system'] 154 | driver = driverClass(args['ddl']) 155 | assert driver != None, "Failed to create '%s' driver" % args['system'] 156 | if args['print_config']: 157 | config = driver.makeDefaultConfig() 158 | print(driver.formatConfig(config)) 159 | print() 160 | sys.exit(0) 161 | 162 | ## Load Configuration file 163 | configFilePath = args['config'] 164 | if configFilePath: 165 | logging.debug("Loading configuration file '%s'" % configFilePath) 166 | cparser = ConfigParser() 167 | cparser.read(os.path.realpath(configFilePath)) 168 | config = dict(cparser.items(args['system'])) 169 | else: 170 | logging.debug("Using default configuration for %s" % args['system']) 171 | defaultConfig = driver.makeDefaultConfig() 172 | config = dict(map(lambda x: (x, defaultConfig[x][1]), defaultConfig.keys())) 173 | config['reset'] = args['reset'] 174 | config['load'] = False 175 | config['execute'] = False 176 | if config['reset']: logging.info("Reseting database") 177 | config['warehouses'] = args['warehouses'] 178 | driver.loadConfig(config) 179 | logging.info("Initializing TPC-C benchmark using %s" % driver) 180 | 181 | 182 | ##Get a list of clientnodes from configuration file. 183 | clients=[] 184 | channels=[] 185 | assert config['clients']!='' 186 | clients=re.split(r"\s+",str(config['clients'])) 187 | #print clients, len(clients),args['clientprocs'] 188 | ##Create ssh channels to client nodes 189 | for node in clients: 190 | cmd = 'ssh='+ node 191 | cmd += r"//chdir=" 192 | cmd += config['path'] 193 | #print cmd 194 | for i in range(args['clientprocs']): 195 | gw=execnet.makegateway(cmd) 196 | ch=gw.remote_exec(worker) 197 | channels.append(ch) 198 | 199 | ## Create ScaleParameters 200 | scaleParameters = scaleparameters.makeWithScaleFactor(args['warehouses'], args['scalefactor']) 201 | nurand = rand.setNURand(nurand.makeForLoad()) 202 | if args['debug']: logging.debug("Scale Parameters:\n%s" % scaleParameters) 203 | 204 | ## DATA LOADER!!! 205 | load_time = None 206 | if not args['no_load']: 207 | load_time = startLoading(scaleParameters, args, config,channels) 208 | #print load_time 209 | ## IF 210 | 211 | ## WORKLOAD DRIVER!!! 212 | if not args['no_execute']: 213 | results = startExecution(scaleParameters, args, config,channels) 214 | assert results 215 | logging.info(results.show(load_time, driver, len(channels), args['samewh'])) 216 | print(results.show(load_time, driver, len(channels), args['samewh'])) 217 | ## IF 218 | 219 | ## MAIN 220 | -------------------------------------------------------------------------------- /pytpcc/runtime/executor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Original Java Version: 8 | # Copyright (C) 2008 9 | # Evan Jones 10 | # Massachusetts Institute of Technology 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining 13 | # a copy of this software and associated documentation files (the 14 | # "Software"), to deal in the Software without restriction, including 15 | # without limitation the rights to use, copy, modify, merge, publish, 16 | # distribute, sublicense, and/or sell copies of the Software, and to 17 | # permit persons to whom the Software is furnished to do so, subject to 18 | # the following conditions: 19 | # 20 | # The above copyright notice and this permission notice shall be 21 | # included in all copies or substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 26 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 27 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 28 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | # OTHER DEALINGS IN THE SOFTWARE. 30 | # ----------------------------------------------------------------------- 31 | 32 | import sys 33 | import multiprocessing 34 | import time 35 | import random 36 | import traceback 37 | import logging 38 | from datetime import datetime 39 | from pprint import pprint,pformat 40 | 41 | import constants 42 | from util import * 43 | 44 | 45 | class Executor: 46 | 47 | def __init__(self, driver, scaleParameters, stop_on_error = False, sameWH = 85): 48 | self.driver = driver 49 | self.scaleParameters = scaleParameters 50 | self.stop_on_error = stop_on_error 51 | self.same_wh = sameWH 52 | ## DEF 53 | 54 | def execute(self, duration): 55 | global_result = results.Results() 56 | assert global_result, "Failed to return a Results object" 57 | logging.debug("Executing benchmark for %d seconds" % duration) 58 | start = global_result.startBenchmark() 59 | debug = logging.getLogger().isEnabledFor(logging.DEBUG) 60 | # Batch Results 61 | batch_result = results.Results() 62 | start_batch = batch_result.startBenchmark() 63 | while (time.time() - start) <= duration: 64 | txn, params = self.doOne() 65 | global_txn_id = global_result.startTransaction(txn) 66 | batch_txn_id = batch_result.startTransaction(txn) 67 | if debug: logging.debug("Executing '%s' transaction" % txn) 68 | try: 69 | (val, retries) = self.driver.executeTransaction(txn, params) 70 | except KeyboardInterrupt: 71 | return -1 72 | except (Exception, AssertionError) as ex: 73 | logging.warn("Failed to execute Transaction '%s': %s" % (txn, ex)) 74 | traceback.print_exc(file=sys.stdout) 75 | print("Aborting some transaction with some error %s %s" % (txn, ex)) 76 | global_result.abortTransaction(global_txn_id) 77 | batch_result.abortTransaction(batch_txn_id) 78 | if self.stop_on_error: raise 79 | continue 80 | 81 | if val is None: 82 | global_result.abortTransaction(global_txn_id, retries) 83 | batch_result.abortTransaction(batch_txn_id, retries) 84 | continue 85 | 86 | batch_result.stopTransaction(batch_txn_id, retries) 87 | global_result.stopTransaction(global_txn_id, retries) 88 | 89 | if time.time() - start_batch > 1800: # every 30 minutes 90 | batch_result.stopBenchmark() 91 | logging.info(batch_result.show()) 92 | batch_result = results.Results() 93 | start_batch = batch_result.startBenchmark() 94 | 95 | ## WHILE 96 | batch_result.stopBenchmark() 97 | global_result.stopBenchmark() 98 | return (global_result) 99 | ## DEF 100 | 101 | def doOne(self): 102 | """Selects and executes a transaction at random. The number of new order transactions executed per minute is the official "tpmC" metric. See TPC-C 5.4.2 (page 71).""" 103 | 104 | ## This is not strictly accurate: The requirement is for certain 105 | ## *minimum* percentages to be maintained. This is close to the right 106 | ## thing, but not precisely correct. See TPC-C 5.2.4 (page 68). 107 | x = rand.number(1, 100) 108 | params = None 109 | txn = None 110 | 111 | #======================================================================================== 112 | # To debug use this to run a specific tpcc test. Run x=100 for new order before running other tests 113 | #x = 100 # new order 114 | #x = 44 # payment 115 | #x = 9 # order status 116 | #x = 7 # delivery 117 | #x = 3 # stock level 118 | #======================================================================================== 119 | 120 | if x <= 4: ## 4% 121 | txn, params = (constants.TransactionTypes.STOCK_LEVEL, self.generateStockLevelParams()) 122 | elif x <= 4 + 4: ## 4% 123 | txn, params = (constants.TransactionTypes.DELIVERY, self.generateDeliveryParams()) 124 | elif x <= 4 + 4 + 4: ## 4% 125 | txn, params = (constants.TransactionTypes.ORDER_STATUS, self.generateOrderStatusParams()) 126 | elif x <= 43 + 4 + 4 + 4: ## 43% 127 | txn, params = (constants.TransactionTypes.PAYMENT, self.generatePaymentParams()) 128 | else: ## 45% 129 | assert x > 100 - 45, "Random number wasn't within specified range or percentages don't add up (%d)" % x 130 | txn, params = (constants.TransactionTypes.NEW_ORDER, self.generateNewOrderParams()) 131 | 132 | return (txn, params) 133 | ## DEF 134 | 135 | ## ---------------------------------------------- 136 | ## generateDeliveryParams 137 | ## ---------------------------------------------- 138 | def generateDeliveryParams(self): 139 | """Return parameters for DELIVERY""" 140 | w_id = self.makeWarehouseId() 141 | o_carrier_id = rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) 142 | ol_delivery_d = datetime.now() 143 | return makeParameterDict(locals(), "w_id", "o_carrier_id", "ol_delivery_d") 144 | ## DEF 145 | 146 | ## ---------------------------------------------- 147 | ## generateNewOrderParams 148 | ## ---------------------------------------------- 149 | def generateNewOrderParams(self): 150 | """Return parameters for NEW_ORDER""" 151 | w_id = self.makeWarehouseId() 152 | d_id = self.makeDistrictId() 153 | c_id = self.makeCustomerId() 154 | ol_cnt = rand.number(constants.MIN_OL_CNT, constants.MAX_OL_CNT) 155 | o_entry_d = datetime.now() 156 | 157 | ## 1% of transactions roll back 158 | rollback = rand.number(1, 100) == 1 159 | 160 | i_ids = [ ] 161 | i_w_ids = [ ] 162 | i_qtys = [ ] 163 | for i in range(0, ol_cnt): 164 | if rollback and i + 1 == ol_cnt: 165 | i_ids.append(self.scaleParameters.items + 1) 166 | else: 167 | i_id = self.makeItemId() 168 | while i_id in i_ids: 169 | i_id = self.makeItemId() 170 | i_ids.append(i_id) 171 | 172 | ## 1% of items are from a remote warehouse 173 | remote = (rand.number(1, 100) == 1) 174 | if self.scaleParameters.warehouses > 1 and remote: 175 | i_w_ids.append(rand.numberExcluding(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse, w_id)) 176 | else: 177 | i_w_ids.append(w_id) 178 | 179 | i_qtys.append(rand.number(1, constants.MAX_OL_QUANTITY)) 180 | ## FOR 181 | 182 | return makeParameterDict(locals(), "w_id", "d_id", "c_id", "o_entry_d", "i_ids", "i_w_ids", "i_qtys") 183 | ## DEF 184 | 185 | ## ---------------------------------------------- 186 | ## generateOrderStatusParams 187 | ## ---------------------------------------------- 188 | def generateOrderStatusParams(self): 189 | """Return parameters for ORDER_STATUS""" 190 | w_id = self.makeWarehouseId() 191 | d_id = self.makeDistrictId() 192 | c_last = None 193 | c_id = None 194 | 195 | ## 60%: order status by last name 196 | if rand.number(1, 100) <= 60: 197 | c_last = rand.makeRandomLastName(self.scaleParameters.customersPerDistrict) 198 | 199 | ## 40%: order status by id 200 | else: 201 | c_id = self.makeCustomerId() 202 | 203 | return makeParameterDict(locals(), "w_id", "d_id", "c_id", "c_last") 204 | ## DEF 205 | 206 | ## ---------------------------------------------- 207 | ## generatePaymentParams 208 | ## ---------------------------------------------- 209 | def generatePaymentParams(self): 210 | """Return parameters for PAYMENT""" 211 | x = rand.number(1, 100) 212 | y = rand.number(1, 100) 213 | 214 | w_id = self.makeWarehouseId() 215 | d_id = self.makeDistrictId() 216 | c_w_id = None 217 | c_d_id = None 218 | c_id = None 219 | c_last = None 220 | h_amount = rand.fixedPoint(2, constants.MIN_PAYMENT, constants.MAX_PAYMENT) 221 | h_date = datetime.now() 222 | 223 | ## 85%: paying through own warehouse (or there is only 1 warehouse) 224 | if self.scaleParameters.warehouses == 1 or x <= self.same_wh: 225 | c_w_id = w_id 226 | c_d_id = d_id 227 | ## 15%: paying through another warehouse: 228 | else: 229 | ## select in range [1, num_warehouses] excluding w_id 230 | c_w_id = rand.numberExcluding(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse, w_id) 231 | assert c_w_id != w_id, "Failed to generate W_ID that's not equal to C_W_ID" 232 | c_d_id = self.makeDistrictId() 233 | 234 | ## 60%: payment by last name 235 | if y <= 60: 236 | c_last = rand.makeRandomLastName(self.scaleParameters.customersPerDistrict) 237 | ## 40%: payment by id 238 | else: 239 | assert y > 60, "Bad random payment value generated %d" % y 240 | c_id = self.makeCustomerId() 241 | 242 | return makeParameterDict(locals(), "w_id", "d_id", "h_amount", "c_w_id", "c_d_id", "c_id", "c_last", "h_date") 243 | ## DEF 244 | 245 | ## ---------------------------------------------- 246 | ## generateStockLevelParams 247 | ## ---------------------------------------------- 248 | def generateStockLevelParams(self): 249 | """Returns parameters for STOCK_LEVEL""" 250 | w_id = self.makeWarehouseId() 251 | d_id = self.makeDistrictId() 252 | threshold = rand.number(constants.MIN_STOCK_LEVEL_THRESHOLD, constants.MAX_STOCK_LEVEL_THRESHOLD) 253 | return makeParameterDict(locals(), "w_id", "d_id", "threshold") 254 | ## DEF 255 | 256 | def makeWarehouseId(self): 257 | w_id = rand.number(self.scaleParameters.starting_warehouse, self.scaleParameters.ending_warehouse) 258 | assert(w_id >= self.scaleParameters.starting_warehouse), "Invalid W_ID: %d" % w_id 259 | assert(w_id <= self.scaleParameters.ending_warehouse), "Invalid W_ID: %d" % w_id 260 | return w_id 261 | ## DEF 262 | 263 | def makeDistrictId(self): 264 | return rand.number(1, self.scaleParameters.districtsPerWarehouse) 265 | ## DEF 266 | 267 | def makeCustomerId(self): 268 | return rand.NURand(1023, 1, self.scaleParameters.customersPerDistrict) 269 | ## DEF 270 | 271 | def makeItemId(self): 272 | return rand.NURand(8191, 1, self.scaleParameters.items) 273 | ## DEF 274 | ## CLASS 275 | 276 | def makeParameterDict(values, *args): 277 | return dict(map(lambda x: (x, values[x]), args)) 278 | ## DEF 279 | -------------------------------------------------------------------------------- /pytpcc/util/results.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining 8 | # a copy of this software and associated documentation files (the 9 | # "Software"), to deal in the Software without restriction, including 10 | # without limitation the rights to use, copy, modify, merge, publish, 11 | # distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so, subject to 13 | # the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 21 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | # OTHER DEALINGS IN THE SOFTWARE. 25 | # ----------------------------------------------------------------------- 26 | 27 | import logging 28 | import time 29 | import os 30 | from collections import Counter 31 | 32 | class Results: 33 | 34 | def __init__(self): 35 | self.start = None 36 | self.stop = None 37 | self.txn_id = 0 38 | 39 | self.txn_counters = {} 40 | self.txn_times = {} 41 | self.txn_mins = {} 42 | self.txn_maxs = {} 43 | self.latencies = {} 44 | self.txn_retries = {} 45 | self.txn_aborts = {} 46 | self.retries = {} 47 | self.running = {} 48 | 49 | def startBenchmark(self): 50 | """Mark the benchmark as having been started""" 51 | assert self.start is None, "start is not none on start" 52 | logging.debug("Starting benchmark statistics collection") 53 | self.start = time.time() 54 | return self.start 55 | 56 | def stopBenchmark(self): 57 | """Mark the benchmark as having been stopped""" 58 | assert self.start != None, "start is none on stop" 59 | assert self.stop is None, "stop isn't none on stop" 60 | logging.debug("Stopping benchmark statistics collection") 61 | self.stop = time.time() 62 | 63 | def startTransaction(self, txn): 64 | self.txn_id += 1 65 | id = self.txn_id 66 | self.running[id] = (txn, time.time()) 67 | return id 68 | 69 | def abortTransaction(self, id, retries=0): 70 | """Abort a transaction and discard its times""" 71 | assert id in self.running, "Didn't find self in running" 72 | txn_name, txn_start = self.running[id] 73 | del self.running[id] 74 | total_retries = self.txn_retries.get(txn_name, 0) 75 | self.txn_retries[txn_name] = total_retries + retries 76 | total_aborts = self.txn_aborts.get(txn_name, 0) 77 | self.txn_aborts[txn_name] = total_aborts + 1 78 | if txn_name not in self.retries: 79 | self.retries[txn_name] = [] 80 | self.retries[txn_name].append(retries) 81 | # txn_start is currently unused, which means we don't include aborted txn in timing metrics 82 | 83 | def stopTransaction(self, id, retries=0): 84 | """Record that the benchmark completed an invocation of the given transaction""" 85 | assert id in self.running, "Didn't find self in running" 86 | txn_name, txn_start = self.running[id] 87 | del self.running[id] 88 | 89 | duration = time.time() - txn_start 90 | total_time = self.txn_times.get(txn_name, 0) 91 | self.txn_times[txn_name] = total_time + duration 92 | if txn_name not in self.latencies: 93 | self.latencies[txn_name] = [] 94 | self.latencies[txn_name].append(duration) 95 | 96 | total_aborts = self.txn_aborts.get(txn_name, 0) 97 | self.txn_aborts[txn_name] = total_aborts 98 | total_retries = self.txn_retries.get(txn_name, 0) 99 | self.txn_retries[txn_name] = total_retries + retries 100 | if txn_name not in self.retries: 101 | self.retries[txn_name] = [] 102 | self.retries[txn_name].append(retries) 103 | 104 | total_cnt = self.txn_counters.get(txn_name, 0) 105 | self.txn_counters[txn_name] = total_cnt + 1 106 | 107 | min_time = self.txn_mins.get(txn_name, 10000000) 108 | if duration < min_time: 109 | self.txn_mins[txn_name] = duration 110 | 111 | max_time = self.txn_maxs.get(txn_name, 0) 112 | if duration > max_time: 113 | self.txn_maxs[txn_name] = duration 114 | 115 | def append(self, r): 116 | for txn_name in r.txn_counters.keys(): 117 | orig_cnt = self.txn_counters.get(txn_name, 0) 118 | orig_min = self.txn_mins.get(txn_name, 10000000) 119 | orig_max = self.txn_maxs.get(txn_name, 0) 120 | orig_time = self.txn_times.get(txn_name, 0) 121 | orig_retries = self.txn_retries.get(txn_name, 0) 122 | orig_aborts = self.txn_aborts.get(txn_name, 0) 123 | 124 | self.txn_counters[txn_name] = orig_cnt + r.txn_counters[txn_name] 125 | self.txn_mins[txn_name] = orig_min if orig_min < r.txn_mins[txn_name] else r.txn_mins[txn_name] 126 | self.txn_maxs[txn_name] = orig_max if orig_max > r.txn_maxs[txn_name] else r.txn_maxs[txn_name] 127 | self.txn_times[txn_name] = orig_time + r.txn_times[txn_name] 128 | self.txn_retries[txn_name] = orig_retries + r.txn_retries[txn_name] 129 | self.txn_aborts[txn_name] = orig_aborts + r.txn_aborts[txn_name] 130 | print("%s [cnt=%d, time=%d]" % (txn_name, self.txn_counters[txn_name], self.txn_times[txn_name])) 131 | # logging.debug("%s [cnt=%d, time=%d]" % (txn_name, self.txn_counters[txn_name], self.txn_times[txn_name])) 132 | if txn_name not in self.latencies: 133 | self.latencies[txn_name] = [] 134 | self.latencies[txn_name].extend(r.latencies[txn_name]) 135 | if txn_name not in self.retries: 136 | self.retries[txn_name] = [] 137 | self.retries[txn_name].extend(r.retries[txn_name]) 138 | 139 | ## HACK 140 | self.start = r.start 141 | self.stop = r.stop 142 | 143 | def __str__(self): 144 | return self.show() 145 | 146 | def show(self, load_time=None, driver=None, threads=1, samewh=85): 147 | if not self.start: 148 | return "Benchmark not started" 149 | if not self.stop: 150 | duration = time.time() - self.start 151 | else: 152 | duration = self.stop - self.start 153 | 154 | col_width = 16 155 | num_columns = 13 156 | total_width = (col_width*num_columns)-8 157 | f = "\n " + (("%-" + str(col_width) + "s")*num_columns) 158 | line = "-"*total_width 159 | 160 | ret = u"\n" + "="*total_width + "\n" 161 | if load_time: 162 | ret += "Data Loading Time: %d seconds\n\n" % (load_time) 163 | 164 | ret += "Execution Results after %d seconds\n%s" % (duration, line) 165 | ret += f % ("", "Complete", u"Time (µs)", u"Percentage", u"Retries", u"minLatMs", u"p50", u"p75", u"p90", u"p95", u"p99", u"maxLatMs", u"Aborts") 166 | 167 | total_time = 0 168 | total_cnt = 0 169 | total_retries = 0 170 | total_aborts = 0 171 | for txn in self.txn_counters: 172 | txn_time = self.txn_times[txn] 173 | txn_cnt = self.txn_counters[txn] 174 | total_time += txn_time 175 | total_cnt += txn_cnt 176 | 177 | result_doc = {} 178 | for txn in sorted(self.txn_counters): 179 | txn_time = self.txn_times[txn] 180 | txn_cnt = self.txn_counters[txn] 181 | min_latency = u"%5.2f" % (1000 * self.txn_mins[txn]) 182 | max_latency = u"%6.2f" % (1000 * self.txn_maxs[txn]) 183 | samples = len(self.latencies[txn]) 184 | lat = sorted(self.latencies[txn]) 185 | ip50 = u"%6.2f" % (1000* lat[int(samples/2)]) 186 | ip75 = u"%6.2f" % (1000*lat[int(samples/100.0*75)]) 187 | ip90 = u"%6.2f" % (1000*lat[int(samples/100.0*90)]) 188 | ip95 = u"%6.2f" % (1000*lat[int(samples/100.0*95)]) 189 | ip99 = u"%6.2f" % (1000*lat[int(samples/100.0*99)]) 190 | txn_aborts = self.txn_aborts[txn] 191 | total_aborts += txn_aborts 192 | txn_retries = self.txn_retries[txn] 193 | total_retries += txn_retries 194 | perc_cnt = u"%5.02f" % ((100.0*txn_cnt / total_cnt)) 195 | perc_time = u"%5.02f" % ((100.0*txn_time / total_time)) 196 | just_retries = [x for x in self.retries[txn] if x>0] 197 | freq_dist = {str(k):v for k,v in dict(Counter(self.retries[txn])).items()} 198 | ret += f % (txn, str(txn_cnt), u"%9.3f" % (txn_time), perc_cnt, 199 | str(len(just_retries))+","+str(sum(just_retries)), 200 | min_latency, ip50, ip75, ip90, ip95, ip99, max_latency, txn_aborts) 201 | result_doc[txn] = {'latency':{'min':1000*self.txn_mins[txn], 'max':1000*self.txn_maxs[txn], 'p50':1000* lat[int(samples/2)], 202 | 'p75':1000*lat[int(samples/100.0*75)],'p90':1000*lat[int(samples/100.0*90)], 203 | 'p95':1000*lat[int(samples/100.0*95)],'p99':1000*lat[int(samples/100.0*99)]}, 204 | 'total':txn_cnt} 205 | if just_retries: 206 | result_doc[txn]['retries']={'retries_ops':freq_dist, 'retries_txn_total':total_retries, 'retries_total_ops':len(just_retries)} 207 | 208 | print(self.txn_counters) 209 | txn_new_order = self.txn_counters.get('NEW_ORDER', 0) 210 | ret += "\n" + line # ("-"*total_width) 211 | #total_rate = "%.02f txn/s" % ((total_cnt / total_time)) 212 | lat = sorted(self.latencies.get('NEW_ORDER',[0])) 213 | samples = len(lat) 214 | ret += f % ("TOTAL", str(total_cnt), u"%12.3f" % total_time, "", "", "", "", "", "", "", "", "", "") 215 | # Only MongoDB Driver returns extra configuration data not available on other drivers 216 | if driver.__class__.__name__ == "MongodbDriver": 217 | result_doc['tpmc'] = txn_new_order*60/duration 218 | result_doc['denorm'] = driver.denormalize 219 | result_doc['duration'] = duration 220 | result_doc['warehouses'] = driver.warehouses 221 | result_doc['date'] = time.strftime("%Y-%m-%d %H:%M:%S") 222 | result_doc['threads'] = threads 223 | result_doc['txn'] = not driver.no_transactions 224 | result_doc['batch_writes'] = driver.batch_writes 225 | result_doc['find_and_modify'] = driver.find_and_modify 226 | result_doc['read_preference'] = driver.read_preference 227 | result_doc['write_concern'] = str(driver.write_concern.document['w']) 228 | result_doc['causal'] = driver.causal_consistency 229 | result_doc['no_global_items'] = driver.no_global_items 230 | result_doc['all_in_one_txn'] = driver.all_in_one_txn 231 | result_doc['retry_writes'] = driver.retry_writes 232 | result_doc['read_concern'] = driver.read_concern 233 | result_doc['shards'] = driver.shards 234 | result_doc['total_retries'] = total_retries 235 | result_doc['samewh'] = samewh 236 | result_doc['total'] = total_cnt 237 | result_doc['aborts'] = total_aborts 238 | result_doc['instance'] = os.getenv('INSTANCE') 239 | ret += "\n%s TpmC for %s %s thr %s txn %d WH: %d %d total %d durSec, batch %s %d retries %s%% %s fnM %s p50 %s p75 %s p90 %s p95 %s p99 %s max %s WC %s causal %s 10in1 %s retry %s %d %d correct %d noGlobalItems %s" % ( 240 | time.strftime("%Y-%m-%d %H:%M:%S"), 241 | ("normal", "denorm")[driver.denormalize], 242 | threads, 243 | ("with", "w/o ")[driver.no_transactions], 244 | driver.warehouses, 245 | round(txn_new_order*60/duration), txn_new_order, duration, 246 | ("off", "on")[driver.batch_writes], total_retries, str(100.0*total_retries/total_cnt)[:5], 247 | ("w/o ", "with")[driver.find_and_modify], 248 | driver.read_preference, 249 | u"%6.2f" % (1000* lat[int(samples/2)]), u"%6.2f" % (1000*lat[int(samples/100.0*75)]), 250 | u"%6.2f" % (1000*lat[int(samples/100.0*90)]), u"%6.2f" % (1000*lat[int(samples/100.0*95)]), 251 | u"%6.2f" % (1000*lat[int(samples/100.0*99)]), 252 | u"%6.2f" % (1000.0*lat[-1]), 253 | str(driver.write_concern), ('false', 'true')[driver.causal_consistency], 254 | ('false', 'true')[driver.all_in_one_txn], ('false', 'true')[driver.retry_writes],total_cnt,total_aborts, samewh, ('false', 'true')[driver.no_global_items]) 255 | driver.save_result(result_doc) 256 | print(result_doc) 257 | # PostgreSQL driver returns a shorter version of the summary without extra configuration data 258 | elif driver.__class__.__name__ == "PostgresqlDriver" or driver.__class__.__name__ == "PostgresqljsonbDriver": 259 | ret += "\n%s TpmC for %s thr %d WH: %d %d total %d durSec, %d retries %s%% p50 %s p75 %s p90 %s p95 %s p99 %s max %s %d %d" % ( 260 | time.strftime("%Y-%m-%d %H:%M:%S"), 261 | threads, 262 | driver.getNumberWH(), 263 | round(txn_new_order*60/duration), txn_new_order, duration, 264 | total_retries, str(100.0*total_retries/total_cnt)[:5], 265 | u"%6.2f" % (1000* lat[int(samples/2)]), u"%6.2f" % (1000*lat[int(samples/100.0*75)]), 266 | u"%6.2f" % (1000*lat[int(samples/100.0*90)]), u"%6.2f" % (1000*lat[int(samples/100.0*95)]), 267 | u"%6.2f" % (1000*lat[int(samples/100.0*99)]), 268 | u"%6.2f" % (1000.0*lat[-1]), 269 | total_cnt,total_aborts) 270 | print(result_doc) 271 | return ret 272 | ## CLASS 273 | -------------------------------------------------------------------------------- /pytpcc/tpcc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------- 4 | # Copyright (C) 2011 5 | # Andy Pavlo 6 | # http:##www.cs.brown.edu/~pavlo/ 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining 9 | # a copy of this software and associated documentation files (the 10 | # "Software"), to deal in the Software without restriction, including 11 | # without limitation the rights to use, copy, modify, merge, publish, 12 | # distribute, sublicense, and/or sell copies of the Software, and to 13 | # permit persons to whom the Software is furnished to do so, subject to 14 | # the following conditions: 15 | # 16 | # The above copyright notice and this permission notice shall be 17 | # included in all copies or substantial portions of the Software. 18 | # 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 22 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | # OTHER DEALINGS IN THE SOFTWARE. 26 | # ----------------------------------------------------------------------- 27 | 28 | import sys 29 | import os 30 | import string 31 | import datetime 32 | import logging 33 | import re 34 | import argparse 35 | import glob 36 | import time 37 | import multiprocessing 38 | import subprocess 39 | import random 40 | from configparser import ConfigParser 41 | from pprint import pprint, pformat 42 | 43 | from util import results, scaleparameters 44 | from runtime import executor, loader 45 | 46 | logging.basicConfig(level=logging.INFO, 47 | format="%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s", 48 | datefmt="%m-%d-%Y %H:%M:%S", 49 | # 50 | filename='results.log') 51 | 52 | console = logging.StreamHandler() 53 | console.setLevel(logging.INFO) 54 | console.setFormatter(logging.Formatter( 55 | '%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s')) 56 | logging.getLogger('').addHandler(console) 57 | 58 | NOTIFY_PHASE_START_PATH = '/data/workdir/src/flamegraph/notify_phase_start.py' 59 | NOTIFY_PHASE_END_PATH = '/data/workdir/src/flamegraph/notify_phase_end.py' 60 | 61 | ## ============================================== 62 | ## notifyDSIOfPhaseStart 63 | ## ============================================== 64 | def notifyDSIOfPhaseStart(phasename): 65 | if os.path.isfile(NOTIFY_PHASE_START_PATH): 66 | output = subprocess.run(["python3", NOTIFY_PHASE_START_PATH, phasename], capture_output=True) 67 | if output.returncode != 0: 68 | raise RuntimeError("Failed to notify DSI of phase starting:", output) 69 | ## DEF 70 | 71 | ## ============================================== 72 | ## notifyDSIOfPhaseEnd 73 | ## ============================================== 74 | def notifyDSIOfPhaseEnd(phasename): 75 | if os.path.isfile(NOTIFY_PHASE_END_PATH): 76 | output = subprocess.run(["python3", NOTIFY_PHASE_END_PATH, phasename], capture_output=True) 77 | if output.returncode != 0: 78 | raise RuntimeError("Failed to notify DSI of phase starting:", output) 79 | ## DEF 80 | 81 | ## ============================================== 82 | ## createDriverClass 83 | ## ============================================== 84 | def createDriverClass(name): 85 | full_name = "%sDriver" % name.title() 86 | mod = __import__('drivers.%s' % full_name.lower(), globals(), locals(), [full_name]) 87 | klass = getattr(mod, full_name) 88 | return klass 89 | ## DEF 90 | 91 | ## ============================================== 92 | ## getDrivers 93 | ## ============================================== 94 | def getDrivers(): 95 | drivers = [] 96 | for f in [os.path.basename(drv).replace("driver.py", "") for drv in glob.glob("./drivers/*driver.py")]: 97 | if f != "abstract": 98 | drivers.append(f) 99 | return drivers 100 | ## DEF 101 | 102 | ## ============================================== 103 | ## startLoading. 104 | # This intentionally uses multiprocess pool and intentionally starts new processes for each batch 105 | # because for long running, many hour long loads, the connection between the child process and the parent process is lost 106 | # and the parent process blocks indefinitelly waiting for the result. 107 | ## ============================================== 108 | def startLoading(driverClass, scaleParameters, args, config): 109 | """ 110 | Starts multiple worker processes to process warehouses in batches. Each batch 111 | consists of 'clients' number of workers, each handling one warehouse. 112 | """ 113 | clients = args['clients'] 114 | logging.debug("Creating client pool with %d processes", clients) 115 | pool = multiprocessing.Pool(clients) 116 | 117 | # Calculate total number of warehouses 118 | total_warehouses = scaleParameters.ending_warehouse - scaleParameters.starting_warehouse + 1 119 | logging.debug(f"Total warehouses: {total_warehouses}") 120 | 121 | loader_results = [] 122 | 123 | # Iterate through warehouses, processing them in batches of 'clients' 124 | for i in range(total_warehouses): 125 | w_id = scaleParameters.starting_warehouse + i 126 | logging.debug(f"Processing warehouse {w_id} in batch {i // clients}") 127 | 128 | # Apply the loader function asynchronously for the current warehouse 129 | r = pool.apply_async(loaderFunc, (driverClass, scaleParameters, args, config, [w_id])) 130 | loader_results.append(r) 131 | 132 | # If we've launched 'clients' workers, wait for them to complete before launching the next batch 133 | if (i + 1) % clients == 0: 134 | logging.debug(f"Waiting for batch {i // clients} to complete") 135 | for r in loader_results: 136 | try: 137 | error_message = r.get() 138 | if error_message: 139 | logging.error(f"Worker process reported error: {error_message}") 140 | raise RuntimeError(f"Failed to process batch: {error_message}") 141 | except Exception as e: 142 | logging.error(f"Exception raised by worker process: {e}") 143 | raise 144 | loader_results = [] # Clear results for next batch 145 | logging.debug(f"Starting batch {i // clients + 1}") 146 | time.sleep(5) 147 | 148 | # Wait for any remaining workers (in the last partial batch) to complete 149 | if loader_results: 150 | logging.debug("Waiting for the final batch to complete") 151 | for r in loader_results: 152 | try: 153 | error_message = r.get() 154 | if error_message: 155 | logging.error(f"Worker process reported error: {error_message}") 156 | raise RuntimeError(f"Failed to process final batch: {error_message}") 157 | except Exception as e: 158 | logging.error(f"Exception raised by worker process: {e}") 159 | raise 160 | 161 | pool.close() 162 | logging.debug("Waiting for all loaders to finish") 163 | pool.join() 164 | logging.info("All loading complete") 165 | ## DEF 166 | 167 | ## ============================================== 168 | ## loaderFunc 169 | ## ============================================== 170 | def loaderFunc(driverClass, scaleParameters, args, config, w_ids): 171 | driver = driverClass(args['ddl']) 172 | assert driver != None, "Driver in loadFunc is none!" 173 | logging.debug("Starting client execution: %s [warehouses=%d]", driver, len(w_ids)) 174 | 175 | config['load'] = True 176 | config['execute'] = False 177 | config['reset'] = False 178 | config['warehouses'] = args['warehouses'] 179 | driver.loadConfig(config) 180 | 181 | try: 182 | loadItems = (1 in w_ids) 183 | l = loader.Loader(driver, scaleParameters, w_ids, loadItems) 184 | driver.loadStart() 185 | l.execute() 186 | driver.loadFinish() 187 | except KeyboardInterrupt: 188 | return -1 189 | except (Exception, AssertionError) as ex: 190 | logging.warn("Failed to load data: %s", ex) 191 | raise 192 | 193 | ## DEF 194 | 195 | ## ============================================== 196 | ## startExecution 197 | ## ============================================== 198 | def startExecution(driverClass, scaleParameters, args, config): 199 | logging.debug("Creating client pool with %d processes", args['clients']) 200 | pool = multiprocessing.Pool(args['clients']) 201 | debug = logging.getLogger().isEnabledFor(logging.DEBUG) 202 | 203 | worker_results = [] 204 | for _ in range(args['clients']): 205 | r = pool.apply_async(executorFunc, (driverClass, scaleParameters, args, config, debug,)) 206 | worker_results.append(r) 207 | ## FOR 208 | pool.close() 209 | pool.join() 210 | 211 | total_results = results.Results() 212 | for asyncr in worker_results: 213 | asyncr.wait() 214 | r = asyncr.get() 215 | assert r != None, "No results object returned by thread!" 216 | if r == -1: 217 | sys.exit(1) 218 | total_results.append(r) 219 | ## FOR 220 | 221 | return total_results 222 | ## DEF 223 | 224 | ## ============================================== 225 | ## executorFunc 226 | ## ============================================== 227 | def executorFunc(driverClass, scaleParameters, args, config, debug): 228 | driver = driverClass(args['ddl']) 229 | assert driver != None, "No driver in executorFunc" 230 | logging.debug("Starting client execution: %s", driver) 231 | 232 | config['execute'] = True 233 | config['reset'] = False 234 | driver.loadConfig(config) 235 | 236 | e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error'], sameWH=args['samewh']) 237 | driver.executeStart() 238 | results = e.execute(args['duration']) 239 | driver.executeFinish() 240 | 241 | return results 242 | ## DEF 243 | 244 | ## ============================================== 245 | ## main 246 | ## ============================================== 247 | if __name__ == '__main__': 248 | aparser = argparse.ArgumentParser(description='Python implementation of the TPC-C Benchmark') 249 | aparser.add_argument('system', choices=getDrivers(), 250 | help='Target system driver') 251 | aparser.add_argument('--config', type=str, 252 | help='Path to driver configuration file') 253 | aparser.add_argument('--reset', action='store_true', 254 | help='Instruct the driver to reset the contents of the database') 255 | aparser.add_argument('--scalefactor', default=1, type=float, metavar='SF', 256 | help='Benchmark scale factor') 257 | aparser.add_argument('--samewh', default=85, type=float, metavar='PP', 258 | help='Percent paying same warehouse') 259 | aparser.add_argument('--warehouses', default=4, type=int, metavar='W', 260 | help='Number of Warehouses') 261 | aparser.add_argument('--duration', default=60, type=int, metavar='D', 262 | help='How long to run the benchmark in seconds') 263 | aparser.add_argument('--ddl', 264 | default=os.path.realpath(os.path.join(os.path.dirname(__file__), "tpcc.sql")), 265 | help='Path to the TPC-C DDL SQL file') 266 | aparser.add_argument('--clients', default=1, type=int, metavar='N', 267 | help='The number of blocking clients to fork') 268 | aparser.add_argument('--stop-on-error', action='store_true', 269 | help='Stop the transaction execution when the driver throws an exception.') 270 | aparser.add_argument('--no-load', action='store_true', 271 | help='Disable loading the data') 272 | aparser.add_argument('--no-execute', action='store_true', 273 | help='Disable executing the workload') 274 | aparser.add_argument('--print-config', action='store_true', 275 | help='Print out the default configuration file for the system and exit') 276 | aparser.add_argument('--debug', action='store_true', 277 | help='Enable debug log messages') 278 | args = vars(aparser.parse_args()) 279 | 280 | if args['debug']: 281 | logging.getLogger().setLevel(logging.DEBUG) 282 | 283 | ## Create a handle to the target client driver 284 | driverClass = createDriverClass(args['system']) 285 | assert driverClass != None, "Failed to find '%s' class" % args['system'] 286 | driver = driverClass(args['ddl']) 287 | assert driver != None, "Failed to create '%s' driver" % args['system'] 288 | if args['print_config']: 289 | config = driver.makeDefaultConfig() 290 | print(driver.formatConfig(config)) 291 | print() 292 | sys.exit(0) 293 | 294 | ## Load Configuration file 295 | configFilePath = args['config'] 296 | if configFilePath: 297 | logging.debug("Loading configuration file '%s'", configFilePath) 298 | cparser = ConfigParser() 299 | cparser.read(os.path.realpath(configFilePath)) 300 | config = dict(cparser.items(args['system'])) 301 | else: 302 | logging.debug("Using default configuration for %s", args['system']) 303 | defaultConfig = driver.makeDefaultConfig() 304 | config = dict([(param, defaultConfig[param][1]) for param in defaultConfig.keys()]) 305 | config['reset'] = args['reset'] 306 | config['load'] = False 307 | config['execute'] = False 308 | if config['reset']: 309 | logging.info("Reseting database") 310 | config['warehouses'] = args['warehouses'] 311 | driver.loadConfig(config) 312 | logging.info("Initializing TPC-C benchmark using %s", driver) 313 | 314 | ## Create ScaleParameters 315 | scaleParameters = scaleparameters.makeWithScaleFactor(args['warehouses'], args['scalefactor']) 316 | if args['debug']: 317 | logging.debug("Scale Parameters:\n%s", scaleParameters) 318 | 319 | ## DATA LOADER!!! 320 | load_time = None 321 | if not args['no_load']: 322 | logging.info("Loading TPC-C benchmark data using %s", (driver)) 323 | notifyDSIOfPhaseStart("TPC-C_load") 324 | load_start = time.time() 325 | if args['clients'] == 1: 326 | l = loader.Loader( 327 | driver, 328 | scaleParameters, 329 | range(scaleParameters.starting_warehouse, scaleParameters.ending_warehouse+1), 330 | True 331 | ) 332 | driver.loadStart() 333 | l.execute() 334 | driver.loadFinish() 335 | else: 336 | startLoading(driverClass, scaleParameters, args, config) 337 | load_time = time.time() - load_start 338 | notifyDSIOfPhaseEnd("TPC-C_load") 339 | ## IF 340 | 341 | ## WORKLOAD DRIVER!!! 342 | if not args['no_execute']: 343 | notifyDSIOfPhaseStart("TPC-C_workload") 344 | if args['clients'] == 1: 345 | e = executor.Executor(driver, scaleParameters, stop_on_error=args['stop_on_error'], sameWH=args['samewh']) 346 | driver.executeStart() 347 | results = e.execute(args['duration']) 348 | driver.executeFinish() 349 | else: 350 | results = startExecution(driverClass, scaleParameters, args, config) 351 | assert results, "No results from execution for %d client!" % args['clients'] 352 | logging.info("Final Results") 353 | logging.info("Threads: %d", args['clients']) 354 | logging.info(results.show(load_time, driver, args['clients'], args['samewh'])) 355 | notifyDSIOfPhaseEnd("TPC-C_workload") 356 | ## IF 357 | 358 | ## MAIN 359 | -------------------------------------------------------------------------------- /pytpcc/runtime/loader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http:##www.cs.brown.edu/~pavlo/ 6 | # 7 | # Original Java Version: 8 | # Copyright (C) 2008 9 | # Evan Jones 10 | # Massachusetts Institute of Technology 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining 13 | # a copy of this software and associated documentation files (the 14 | # "Software"), to deal in the Software without restriction, including 15 | # without limitation the rights to use, copy, modify, merge, publish, 16 | # distribute, sublicense, and/or sell copies of the Software, and to 17 | # permit persons to whom the Software is furnished to do so, subject to 18 | # the following conditions: 19 | # 20 | # The above copyright notice and this permission notice shall be 21 | # included in all copies or substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 26 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 27 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 28 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | # OTHER DEALINGS IN THE SOFTWARE. 30 | # ----------------------------------------------------------------------- 31 | 32 | import os 33 | import sys 34 | 35 | import logging 36 | from datetime import datetime 37 | from random import shuffle 38 | from pprint import pprint,pformat 39 | 40 | import constants 41 | from util import * 42 | 43 | class Loader: 44 | 45 | def __init__(self, handle, scaleParameters, w_ids, needLoadItems): 46 | self.handle = handle 47 | self.scaleParameters = scaleParameters 48 | self.w_ids = w_ids 49 | self.needLoadItems = needLoadItems 50 | self.batch_size = 500 51 | 52 | ## ============================================== 53 | ## execute 54 | ## ============================================== 55 | def execute(self): 56 | 57 | ## Item Table 58 | if self.needLoadItems: 59 | logging.debug("Loading ITEM table") 60 | self.loadItems() 61 | self.handle.loadFinishItem() 62 | 63 | ## Then create the warehouse-specific tuples 64 | for w_id in self.w_ids: 65 | self.loadWarehouse(w_id) 66 | self.handle.loadFinishWarehouse(w_id) 67 | ## FOR 68 | 69 | return (None) 70 | 71 | ## ============================================== 72 | ## loadItems 73 | ## ============================================== 74 | def loadItems(self): 75 | ## Select 10% of the rows to be marked "original" 76 | originalRows = rand.selectUniqueIds(self.scaleParameters.items // 10, 1, self.scaleParameters.items) 77 | 78 | ## Load all of the items 79 | tuples = [ ] 80 | total_tuples = 0 81 | for i in range(1, self.scaleParameters.items+1): 82 | original = (i in originalRows) 83 | tuples.append(self.generateItem(i, original)) 84 | total_tuples += 1 85 | if len(tuples) == self.batch_size: 86 | logging.debug("LOAD - %s: %5d / %d" % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items)) 87 | self.handle.loadTuples(constants.TABLENAME_ITEM, tuples) 88 | tuples = [ ] 89 | ## FOR 90 | if len(tuples) > 0: 91 | logging.debug("LOAD - %s: %5d / %d" % (constants.TABLENAME_ITEM, total_tuples, self.scaleParameters.items)) 92 | self.handle.loadTuples(constants.TABLENAME_ITEM, tuples) 93 | ## DEF 94 | 95 | ## ============================================== 96 | ## loadWarehouse 97 | ## ============================================== 98 | def loadWarehouse(self, w_id): 99 | logging.debug("LOAD - %s: %d / %d" % (constants.TABLENAME_WAREHOUSE, w_id, len(self.w_ids))) 100 | 101 | ## WAREHOUSE 102 | w_tuples = [ self.generateWarehouse(w_id) ] 103 | self.handle.loadTuples(constants.TABLENAME_WAREHOUSE, w_tuples) 104 | 105 | ## DISTRICT 106 | d_tuples = [ ] 107 | for d_id in range(1, self.scaleParameters.districtsPerWarehouse+1): 108 | d_next_o_id = self.scaleParameters.customersPerDistrict + 1 109 | d_tuples = [ self.generateDistrict(w_id, d_id, d_next_o_id) ] 110 | 111 | c_tuples = [ ] 112 | h_tuples = [ ] 113 | 114 | ## Select 10% of the customers to have bad credit 115 | selectedRows = rand.selectUniqueIds(self.scaleParameters.customersPerDistrict // 10, 1, self.scaleParameters.customersPerDistrict) 116 | 117 | ## TPC-C 4.3.3.1. says that o_c_id should be a permutation of [1, 3000]. But since it 118 | ## is a c_id field, it seems to make sense to have it be a permutation of the 119 | ## customers. For the "real" thing this will be equivalent 120 | cIdPermutation = [ ] 121 | 122 | for c_id in range(1, self.scaleParameters.customersPerDistrict+1): 123 | badCredit = (c_id in selectedRows) 124 | c_tuples.append(self.generateCustomer(w_id, d_id, c_id, badCredit, True)) 125 | h_tuples.append(self.generateHistory(w_id, d_id, c_id)) 126 | cIdPermutation.append(c_id) 127 | ## FOR 128 | assert cIdPermutation[0] == 1 129 | assert cIdPermutation[self.scaleParameters.customersPerDistrict - 1] == self.scaleParameters.customersPerDistrict 130 | shuffle(cIdPermutation) 131 | 132 | o_tuples = [ ] 133 | ol_tuples = [ ] 134 | no_tuples = [ ] 135 | 136 | for o_id in range(1, self.scaleParameters.customersPerDistrict+1): 137 | o_ol_cnt = rand.number(constants.MIN_OL_CNT, constants.MAX_OL_CNT) 138 | 139 | ## The last newOrdersPerDistrict are new orders 140 | newOrder = ((self.scaleParameters.customersPerDistrict - self.scaleParameters.newOrdersPerDistrict) < o_id) 141 | o_tuples.append(self.generateOrder(w_id, d_id, o_id, cIdPermutation[o_id - 1], o_ol_cnt, newOrder)) 142 | 143 | ## Generate each OrderLine for the order 144 | for ol_number in range(0, o_ol_cnt): 145 | ol_tuples.append(self.generateOrderLine(w_id, d_id, o_id, ol_number, self.scaleParameters.items, newOrder)) 146 | ## FOR 147 | 148 | ## This is a new order: make one for it 149 | if newOrder: no_tuples.append([o_id, d_id, w_id]) 150 | ## FOR 151 | 152 | self.handle.loadTuples(constants.TABLENAME_DISTRICT, d_tuples) 153 | self.handle.loadTuples(constants.TABLENAME_CUSTOMER, c_tuples) 154 | self.handle.loadTuples(constants.TABLENAME_ORDERS, o_tuples) 155 | self.handle.loadTuples(constants.TABLENAME_ORDER_LINE, ol_tuples) 156 | self.handle.loadTuples(constants.TABLENAME_NEW_ORDER, no_tuples) 157 | self.handle.loadTuples(constants.TABLENAME_HISTORY, h_tuples) 158 | self.handle.loadFinishDistrict(w_id, d_id) 159 | ## FOR 160 | 161 | ## Select 10% of the stock to be marked "original" 162 | s_tuples = [ ] 163 | selectedRows = rand.selectUniqueIds(self.scaleParameters.items // 10, 1, self.scaleParameters.items) 164 | total_tuples = 0 165 | for i_id in range(1, self.scaleParameters.items+1): 166 | original = (i_id in selectedRows) 167 | s_tuples.append(self.generateStock(w_id, i_id, original)) 168 | if len(s_tuples) >= self.batch_size: 169 | logging.debug("LOAD - %s [W_ID=%d]: %5d / %d" % (constants.TABLENAME_STOCK, w_id, total_tuples, self.scaleParameters.items)) 170 | self.handle.loadTuples(constants.TABLENAME_STOCK, s_tuples) 171 | s_tuples = [ ] 172 | total_tuples += 1 173 | ## FOR 174 | if len(s_tuples) > 0: 175 | logging.debug("LOAD - %s [W_ID=%d]: %5d / %d" % (constants.TABLENAME_STOCK, w_id, total_tuples, self.scaleParameters.items)) 176 | self.handle.loadTuples(constants.TABLENAME_STOCK, s_tuples) 177 | ## DEF 178 | 179 | ## ============================================== 180 | ## generateItem 181 | ## ============================================== 182 | def generateItem(self, id, original): 183 | i_id = id 184 | i_im_id = rand.number(constants.MIN_IM, constants.MAX_IM) 185 | i_name = rand.astring(constants.MIN_I_NAME, constants.MAX_I_NAME) 186 | i_price = rand.fixedPoint(constants.MONEY_DECIMALS, constants.MIN_PRICE, constants.MAX_PRICE) 187 | i_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA) 188 | if original: i_data = self.fillOriginal(i_data) 189 | 190 | return [i_id, i_im_id, i_name, i_price, i_data] 191 | ## DEF 192 | 193 | ## ============================================== 194 | ## generateWarehouse 195 | ## ============================================== 196 | def generateWarehouse(self, w_id): 197 | w_tax = self.generateTax() 198 | w_ytd = constants.INITIAL_W_YTD 199 | w_address = self.generateAddress() 200 | return [w_id] + w_address + [w_tax, w_ytd] 201 | ## DEF 202 | 203 | ## ============================================== 204 | ## generateDistrict 205 | ## ============================================== 206 | def generateDistrict(self, d_w_id, d_id, d_next_o_id): 207 | d_tax = self.generateTax() 208 | d_ytd = constants.INITIAL_D_YTD 209 | d_address = self.generateAddress() 210 | return [d_id, d_w_id] + d_address + [d_tax, d_ytd, d_next_o_id] 211 | ## DEF 212 | 213 | ## ============================================== 214 | ## generateCustomer 215 | ## ============================================== 216 | def generateCustomer(self, c_w_id, c_d_id, c_id, badCredit, doesReplicateName): 217 | c_first = rand.astring(constants.MIN_FIRST, constants.MAX_FIRST) 218 | c_middle = constants.MIDDLE 219 | 220 | assert 1 <= c_id and c_id <= constants.CUSTOMERS_PER_DISTRICT 221 | if c_id <= 1000: 222 | c_last = rand.makeLastName(c_id - 1) 223 | else: 224 | c_last = rand.makeRandomLastName(constants.CUSTOMERS_PER_DISTRICT) 225 | 226 | c_phone = rand.nstring(constants.PHONE, constants.PHONE) 227 | c_since = datetime.now() 228 | c_credit = constants.BAD_CREDIT if badCredit else constants.GOOD_CREDIT 229 | c_credit_lim = constants.INITIAL_CREDIT_LIM 230 | c_discount = rand.fixedPoint(constants.DISCOUNT_DECIMALS, constants.MIN_DISCOUNT, constants.MAX_DISCOUNT) 231 | c_balance = constants.INITIAL_BALANCE 232 | c_ytd_payment = constants.INITIAL_YTD_PAYMENT 233 | c_payment_cnt = constants.INITIAL_PAYMENT_CNT 234 | c_delivery_cnt = constants.INITIAL_DELIVERY_CNT 235 | c_data = rand.astring(constants.MIN_C_DATA, constants.MAX_C_DATA) 236 | 237 | c_street1 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) 238 | c_street2 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) 239 | c_city = rand.astring(constants.MIN_CITY, constants.MAX_CITY) 240 | c_state = rand.astring(constants.STATE, constants.STATE) 241 | c_zip = self.generateZip() 242 | 243 | return [ c_id, c_d_id, c_w_id, c_first, c_middle, c_last, \ 244 | c_street1, c_street2, c_city, c_state, c_zip, \ 245 | c_phone, c_since, c_credit, c_credit_lim, c_discount, c_balance, \ 246 | c_ytd_payment, c_payment_cnt, c_delivery_cnt, c_data ] 247 | ## DEF 248 | 249 | ## ============================================== 250 | ## generateOrder 251 | ## ============================================== 252 | def generateOrder(self, o_w_id, o_d_id, o_id, o_c_id, o_ol_cnt, newOrder): 253 | """Returns the generated o_ol_cnt value.""" 254 | o_entry_d = datetime.now() 255 | o_carrier_id = constants.NULL_CARRIER_ID if newOrder else rand.number(constants.MIN_CARRIER_ID, constants.MAX_CARRIER_ID) 256 | o_all_local = constants.INITIAL_ALL_LOCAL 257 | return [ o_id, o_c_id, o_d_id, o_w_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local ] 258 | ## DEF 259 | 260 | ## ============================================== 261 | ## generateOrderLine 262 | ## ============================================== 263 | def generateOrderLine(self, ol_w_id, ol_d_id, ol_o_id, ol_number, max_items, newOrder): 264 | ol_i_id = rand.number(1, max_items) 265 | ol_supply_w_id = ol_w_id 266 | ol_delivery_d = datetime.now() 267 | ol_quantity = constants.INITIAL_QUANTITY 268 | 269 | ## 1% of items are from a remote warehouse 270 | remote = (rand.number(1, 100) == 1) 271 | if self.scaleParameters.warehouses > 1 and remote: 272 | ol_supply_w_id = rand.numberExcluding(self.scaleParameters.starting_warehouse, 273 | self.scaleParameters.ending_warehouse, 274 | ol_w_id) 275 | 276 | ol_amount = rand.fixedPoint(constants.MONEY_DECIMALS, constants.MIN_AMOUNT, constants.MAX_PRICE * constants.MAX_OL_QUANTITY) 277 | if newOrder: 278 | ol_delivery_d = None 279 | ol_dist_info = rand.astring(constants.DIST, constants.DIST) 280 | 281 | return [ ol_o_id, ol_d_id, ol_w_id, ol_number, ol_i_id, ol_supply_w_id, ol_delivery_d, ol_quantity, ol_amount, ol_dist_info ] 282 | ## DEF 283 | 284 | ## ============================================== 285 | ## generateStock 286 | ## ============================================== 287 | def generateStock(self, s_w_id, s_i_id, original): 288 | s_quantity = rand.number(constants.MIN_QUANTITY, constants.MAX_QUANTITY); 289 | s_ytd = 0; 290 | s_order_cnt = 0; 291 | s_remote_cnt = 0; 292 | 293 | s_data = rand.astring(constants.MIN_I_DATA, constants.MAX_I_DATA); 294 | if original: self.fillOriginal(s_data) 295 | 296 | s_dists = [ ] 297 | for i in range(0, constants.DISTRICTS_PER_WAREHOUSE): 298 | s_dists.append(rand.astring(constants.DIST, constants.DIST)) 299 | 300 | return [ s_i_id, s_w_id, s_quantity ] + \ 301 | s_dists + \ 302 | [ s_ytd, s_order_cnt, s_remote_cnt, s_data ] 303 | ## DEF 304 | 305 | ## ============================================== 306 | ## generateHistory 307 | ## ============================================== 308 | def generateHistory(self, h_c_w_id, h_c_d_id, h_c_id): 309 | h_w_id = h_c_w_id 310 | h_d_id = h_c_d_id 311 | h_date = datetime.now() 312 | h_amount = constants.INITIAL_AMOUNT 313 | h_data = rand.astring(constants.MIN_DATA, constants.MAX_DATA) 314 | return [ h_c_id, h_c_d_id, h_c_w_id, h_d_id, h_w_id, h_date, h_amount, h_data ] 315 | ## DEF 316 | 317 | ## ============================================== 318 | ## generateAddress 319 | ## ============================================== 320 | def generateAddress(self): 321 | """ 322 | Returns a name and a street address 323 | Used by both generateWarehouse and generateDistrict. 324 | """ 325 | name = rand.astring(constants.MIN_NAME, constants.MAX_NAME) 326 | return [ name ] + self.generateStreetAddress() 327 | ## DEF 328 | 329 | ## ============================================== 330 | ## generateStreetAddress 331 | ## ============================================== 332 | def generateStreetAddress(self): 333 | """ 334 | Returns a list for a street address 335 | Used for warehouses, districts and customers. 336 | """ 337 | street1 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) 338 | street2 = rand.astring(constants.MIN_STREET, constants.MAX_STREET) 339 | city = rand.astring(constants.MIN_CITY, constants.MAX_CITY) 340 | state = rand.astring(constants.STATE, constants.STATE) 341 | zip = self.generateZip() 342 | 343 | return [ street1, street2, city, state, zip ] 344 | ## DEF 345 | 346 | ## ============================================== 347 | ## generateTax 348 | ## ============================================== 349 | def generateTax(self): 350 | return rand.fixedPoint(constants.TAX_DECIMALS, constants.MIN_TAX, constants.MAX_TAX) 351 | ## DEF 352 | 353 | ## ============================================== 354 | ## generateZip 355 | ## ============================================== 356 | def generateZip(self): 357 | length = constants.ZIP_LENGTH - len(constants.ZIP_SUFFIX) 358 | return rand.nstring(length, length) + constants.ZIP_SUFFIX 359 | ## DEF 360 | 361 | ## ============================================== 362 | ## fillOriginal 363 | ## ============================================== 364 | def fillOriginal(self, data): 365 | """ 366 | a string with ORIGINAL_STRING at a random position 367 | """ 368 | originalLength = len(constants.ORIGINAL_STRING) 369 | position = rand.number(0, len(data) - originalLength) 370 | out = data[:position] + constants.ORIGINAL_STRING + data[position + originalLength:] 371 | assert len(out) == len(data) 372 | return out 373 | ## DEF 374 | ## CLASS 375 | -------------------------------------------------------------------------------- /pytpcc/drivers/sqlitedriver.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ----------------------------------------------------------------------- 3 | # Copyright (C) 2011 4 | # Andy Pavlo 5 | # http://www.cs.brown.edu/~pavlo/ 6 | # 7 | # Original Java Version: 8 | # Copyright (C) 2008 9 | # Evan Jones 10 | # Massachusetts Institute of Technology 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining 13 | # a copy of this software and associated documentation files (the 14 | # "Software"), to deal in the Software without restriction, including 15 | # without limitation the rights to use, copy, modify, merge, publish, 16 | # distribute, sublicense, and/or sell copies of the Software, and to 17 | # permit persons to whom the Software is furnished to do so, subject to 18 | # the following conditions: 19 | # 20 | # The above copyright notice and this permission notice shall be 21 | # included in all copies or substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 26 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 27 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 28 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | # OTHER DEALINGS IN THE SOFTWARE. 30 | # ----------------------------------------------------------------------- 31 | 32 | from __future__ import with_statement 33 | 34 | import os 35 | import sqlite3 36 | import logging 37 | import commands 38 | from pprint import pprint,pformat 39 | 40 | import constants 41 | from abstractdriver import * 42 | 43 | TXN_QUERIES = { 44 | "DELIVERY": { 45 | "getNewOrder": "SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID > -1 LIMIT 1", # 46 | "deleteNewOrder": "DELETE FROM NEW_ORDER WHERE NO_D_ID = ? AND NO_W_ID = ? AND NO_O_ID = ?", # d_id, w_id, no_o_id 47 | "getCId": "SELECT O_C_ID FROM ORDERS WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # no_o_id, d_id, w_id 48 | "updateOrders": "UPDATE ORDERS SET O_CARRIER_ID = ? WHERE O_ID = ? AND O_D_ID = ? AND O_W_ID = ?", # o_carrier_id, no_o_id, d_id, w_id 49 | "updateOrderLine": "UPDATE ORDER_LINE SET OL_DELIVERY_D = ? WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # o_entry_d, no_o_id, d_id, w_id 50 | "sumOLAmount": "SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = ? AND OL_D_ID = ? AND OL_W_ID = ?", # no_o_id, d_id, w_id 51 | "updateCustomer": "UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + ? WHERE C_ID = ? AND C_D_ID = ? AND C_W_ID = ?", # ol_total, c_id, d_id, w_id 52 | }, 53 | "NEW_ORDER": { 54 | "getWarehouseTaxRate": "SELECT W_TAX FROM WAREHOUSE WHERE W_ID = ?", # w_id 55 | "getDistrict": "SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = ? AND D_W_ID = ?", # d_id, w_id 56 | "incrementNextOrderId": "UPDATE DISTRICT SET D_NEXT_O_ID = ? WHERE D_ID = ? AND D_W_ID = ?", # d_next_o_id, d_id, w_id 57 | "getCustomer": "SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id 58 | "createOrder": "INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local 59 | "createNewOrder": "INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (?, ?, ?)", # o_id, d_id, w_id 60 | "getItemInfo": "SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = ?", # ol_i_id 61 | "getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_%02d FROM STOCK WHERE S_I_ID = ? AND S_W_ID = ?", # d_id, ol_i_id, ol_supply_w_id 62 | "updateStock": "UPDATE STOCK SET S_QUANTITY = ?, S_YTD = ?, S_ORDER_CNT = ?, S_REMOTE_CNT = ? WHERE S_I_ID = ? AND S_W_ID = ?", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id 63 | "createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info 64 | }, 65 | 66 | "ORDER_STATUS": { 67 | "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id 68 | "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last 69 | "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = ? AND O_D_ID = ? AND O_C_ID = ? ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id 70 | "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = ? AND OL_D_ID = ? AND OL_O_ID = ?", # w_id, d_id, o_id 71 | }, 72 | 73 | "PAYMENT": { 74 | "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = ?", # w_id 75 | "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + ? WHERE W_ID = ?", # h_amount, w_id 76 | "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", # w_id, d_id 77 | "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + ? WHERE D_W_ID = ? AND D_ID = ?", # h_amount, d_w_id, d_id 78 | "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # w_id, d_id, c_id 79 | "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = ? AND C_D_ID = ? AND C_LAST = ? ORDER BY C_FIRST", # w_id, d_id, c_last 80 | "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ?, C_DATA = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id 81 | "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = ?, C_YTD_PAYMENT = ?, C_PAYMENT_CNT = ? WHERE C_W_ID = ? AND C_D_ID = ? AND C_ID = ?", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id 82 | "insertHistory": "INSERT INTO HISTORY VALUES (?, ?, ?, ?, ?, ?, ?, ?)", 83 | }, 84 | 85 | "STOCK_LEVEL": { 86 | "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = ? AND D_ID = ?", 87 | "getStockCount": """ 88 | SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK 89 | WHERE OL_W_ID = ? 90 | AND OL_D_ID = ? 91 | AND OL_O_ID < ? 92 | AND OL_O_ID >= ? 93 | AND S_W_ID = ? 94 | AND S_I_ID = OL_I_ID 95 | AND S_QUANTITY < ? 96 | """, 97 | }, 98 | } 99 | 100 | 101 | ## ============================================== 102 | ## SqliteDriver 103 | ## ============================================== 104 | class SqliteDriver(AbstractDriver): 105 | DEFAULT_CONFIG = { 106 | "database": ("The path to the SQLite database", "/tmp/tpcc.db" ), 107 | } 108 | 109 | def __init__(self, ddl): 110 | super(SqliteDriver, self).__init__("sqlite", ddl) 111 | self.database = None 112 | self.conn = None 113 | self.cursor = None 114 | 115 | ## ---------------------------------------------- 116 | ## makeDefaultConfig 117 | ## ---------------------------------------------- 118 | def makeDefaultConfig(self): 119 | return SqliteDriver.DEFAULT_CONFIG 120 | 121 | ## ---------------------------------------------- 122 | ## loadConfig 123 | ## ---------------------------------------------- 124 | def loadConfig(self, config): 125 | for key in SqliteDriver.DEFAULT_CONFIG.keys(): 126 | assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) 127 | 128 | self.database = str(config["database"]) 129 | 130 | if config["reset"] and os.path.exists(self.database): 131 | logging.debug("Deleting database '%s'" % self.database) 132 | os.unlink(self.database) 133 | 134 | if os.path.exists(self.database) == False: 135 | logging.debug("Loading DDL file '%s'" % (self.ddl)) 136 | ## HACK 137 | cmd = "sqlite3 %s < %s" % (self.database, self.ddl) 138 | (result, output) = commands.getstatusoutput(cmd) 139 | assert result == 0, cmd + "\n" + output 140 | ## IF 141 | 142 | self.conn = sqlite3.connect(self.database) 143 | self.cursor = self.conn.cursor() 144 | 145 | ## ---------------------------------------------- 146 | ## loadTuples 147 | ## ---------------------------------------------- 148 | def loadTuples(self, tableName, tuples): 149 | if len(tuples) == 0: return 150 | 151 | p = ["?"]*len(tuples[0]) 152 | sql = "INSERT INTO %s VALUES (%s)" % (tableName, ",".join(p)) 153 | self.cursor.executemany(sql, tuples) 154 | 155 | logging.debug("Loaded %d tuples for tableName %s" % (len(tuples), tableName)) 156 | return 157 | 158 | ## ---------------------------------------------- 159 | ## loadFinish 160 | ## ---------------------------------------------- 161 | def loadFinish(self): 162 | logging.info("Commiting changes to database") 163 | self.conn.commit() 164 | 165 | ## ---------------------------------------------- 166 | ## doDelivery 167 | ## ---------------------------------------------- 168 | def doDelivery(self, params): 169 | q = TXN_QUERIES["DELIVERY"] 170 | 171 | w_id = params["w_id"] 172 | o_carrier_id = params["o_carrier_id"] 173 | ol_delivery_d = params["ol_delivery_d"] 174 | 175 | result = [ ] 176 | for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): 177 | self.cursor.execute(q["getNewOrder"], [d_id, w_id]) 178 | newOrder = self.cursor.fetchone() 179 | if newOrder == None: 180 | ## No orders for this district: skip it. Note: This must be reported if > 1% 181 | continue 182 | assert len(newOrder) > 0 183 | no_o_id = newOrder[0] 184 | 185 | self.cursor.execute(q["getCId"], [no_o_id, d_id, w_id]) 186 | c_id = self.cursor.fetchone()[0] 187 | 188 | self.cursor.execute(q["sumOLAmount"], [no_o_id, d_id, w_id]) 189 | ol_total = self.cursor.fetchone()[0] 190 | 191 | self.cursor.execute(q["deleteNewOrder"], [d_id, w_id, no_o_id]) 192 | self.cursor.execute(q["updateOrders"], [o_carrier_id, no_o_id, d_id, w_id]) 193 | self.cursor.execute(q["updateOrderLine"], [ol_delivery_d, no_o_id, d_id, w_id]) 194 | 195 | # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) 196 | # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure 197 | # them out 198 | # If there are no order lines, SUM returns null. There should always be order lines. 199 | assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" 200 | assert ol_total > 0.0 201 | 202 | self.cursor.execute(q["updateCustomer"], [ol_total, c_id, d_id, w_id]) 203 | 204 | result.append((d_id, no_o_id)) 205 | ## FOR 206 | 207 | self.conn.commit() 208 | return result 209 | 210 | ## ---------------------------------------------- 211 | ## doNewOrder 212 | ## ---------------------------------------------- 213 | def doNewOrder(self, params): 214 | q = TXN_QUERIES["NEW_ORDER"] 215 | 216 | w_id = params["w_id"] 217 | d_id = params["d_id"] 218 | c_id = params["c_id"] 219 | o_entry_d = params["o_entry_d"] 220 | i_ids = params["i_ids"] 221 | i_w_ids = params["i_w_ids"] 222 | i_qtys = params["i_qtys"] 223 | 224 | assert len(i_ids) > 0 225 | assert len(i_ids) == len(i_w_ids) 226 | assert len(i_ids) == len(i_qtys) 227 | 228 | all_local = True 229 | items = [ ] 230 | for i in range(len(i_ids)): 231 | ## Determine if this is an all local order or not 232 | all_local = all_local and i_w_ids[i] == w_id 233 | self.cursor.execute(q["getItemInfo"], [i_ids[i]]) 234 | items.append(self.cursor.fetchone()) 235 | assert len(items) == len(i_ids) 236 | 237 | ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. 238 | ## Note that this will happen with 1% of transactions on purpose. 239 | for item in items: 240 | if len(item) == 0: 241 | ## TODO Abort here! 242 | return 243 | ## FOR 244 | 245 | ## ---------------- 246 | ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER 247 | ## ---------------- 248 | self.cursor.execute(q["getWarehouseTaxRate"], [w_id]) 249 | w_tax = self.cursor.fetchone()[0] 250 | 251 | self.cursor.execute(q["getDistrict"], [d_id, w_id]) 252 | district_info = self.cursor.fetchone() 253 | d_tax = district_info[0] 254 | d_next_o_id = district_info[1] 255 | 256 | self.cursor.execute(q["getCustomer"], [w_id, d_id, c_id]) 257 | customer_info = self.cursor.fetchone() 258 | c_discount = customer_info[0] 259 | 260 | ## ---------------- 261 | ## Insert Order Information 262 | ## ---------------- 263 | ol_cnt = len(i_ids) 264 | o_carrier_id = constants.NULL_CARRIER_ID 265 | 266 | self.cursor.execute(q["incrementNextOrderId"], [d_next_o_id + 1, d_id, w_id]) 267 | self.cursor.execute(q["createOrder"], [d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, ol_cnt, all_local]) 268 | self.cursor.execute(q["createNewOrder"], [d_next_o_id, d_id, w_id]) 269 | 270 | ## ---------------- 271 | ## Insert Order Item Information 272 | ## ---------------- 273 | item_data = [ ] 274 | total = 0 275 | for i in range(len(i_ids)): 276 | ol_number = i + 1 277 | ol_supply_w_id = i_w_ids[i] 278 | ol_i_id = i_ids[i] 279 | ol_quantity = i_qtys[i] 280 | 281 | itemInfo = items[i] 282 | i_name = itemInfo[1] 283 | i_data = itemInfo[2] 284 | i_price = itemInfo[0] 285 | 286 | self.cursor.execute(q["getStockInfo"] % (d_id), [ol_i_id, ol_supply_w_id]) 287 | stockInfo = self.cursor.fetchone() 288 | if len(stockInfo) == 0: 289 | logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" % (ol_i_id, ol_supply_w_id)) 290 | continue 291 | s_quantity = stockInfo[0] 292 | s_ytd = stockInfo[2] 293 | s_order_cnt = stockInfo[3] 294 | s_remote_cnt = stockInfo[4] 295 | s_data = stockInfo[1] 296 | s_dist_xx = stockInfo[5] # Fetches data from the s_dist_[d_id] column 297 | 298 | ## Update stock 299 | s_ytd += ol_quantity 300 | if s_quantity >= ol_quantity + 10: 301 | s_quantity = s_quantity - ol_quantity 302 | else: 303 | s_quantity = s_quantity + 91 - ol_quantity 304 | s_order_cnt += 1 305 | 306 | if ol_supply_w_id != w_id: s_remote_cnt += 1 307 | 308 | self.cursor.execute(q["updateStock"], [s_quantity, s_ytd, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id]) 309 | 310 | if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: 311 | brand_generic = 'B' 312 | else: 313 | brand_generic = 'G' 314 | 315 | ## Transaction profile states to use "ol_quantity * i_price" 316 | ol_amount = ol_quantity * i_price 317 | total += ol_amount 318 | 319 | self.cursor.execute(q["createOrderLine"], [d_next_o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, o_entry_d, ol_quantity, ol_amount, s_dist_xx]) 320 | 321 | ## Add the info to be returned 322 | item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) 323 | ## FOR 324 | 325 | ## Commit! 326 | self.conn.commit() 327 | 328 | ## Adjust the total for the discount 329 | #print "c_discount:", c_discount, type(c_discount) 330 | #print "w_tax:", w_tax, type(w_tax) 331 | #print "d_tax:", d_tax, type(d_tax) 332 | total *= (1 - c_discount) * (1 + w_tax + d_tax) 333 | 334 | ## Pack up values the client is missing (see TPC-C 2.4.3.5) 335 | misc = [ (w_tax, d_tax, d_next_o_id, total) ] 336 | 337 | return [ customer_info, misc, item_data ] 338 | 339 | ## ---------------------------------------------- 340 | ## doOrderStatus 341 | ## ---------------------------------------------- 342 | def doOrderStatus(self, params): 343 | q = TXN_QUERIES["ORDER_STATUS"] 344 | 345 | w_id = params["w_id"] 346 | d_id = params["d_id"] 347 | c_id = params["c_id"] 348 | c_last = params["c_last"] 349 | 350 | assert w_id, pformat(params) 351 | assert d_id, pformat(params) 352 | 353 | if c_id != None: 354 | self.cursor.execute(q["getCustomerByCustomerId"], [w_id, d_id, c_id]) 355 | customer = self.cursor.fetchone() 356 | else: 357 | # Get the midpoint customer's id 358 | self.cursor.execute(q["getCustomersByLastName"], [w_id, d_id, c_last]) 359 | all_customers = self.cursor.fetchall() 360 | assert len(all_customers) > 0 361 | namecnt = len(all_customers) 362 | index = (namecnt-1)/2 363 | customer = all_customers[index] 364 | c_id = customer[0] 365 | assert len(customer) > 0 366 | assert c_id != None 367 | 368 | self.cursor.execute(q["getLastOrder"], [w_id, d_id, c_id]) 369 | order = self.cursor.fetchone() 370 | if order: 371 | self.cursor.execute(q["getOrderLines"], [w_id, d_id, order[0]]) 372 | orderLines = self.cursor.fetchall() 373 | else: 374 | orderLines = [ ] 375 | 376 | self.conn.commit() 377 | return [ customer, order, orderLines ] 378 | 379 | ## ---------------------------------------------- 380 | ## doPayment 381 | ## ---------------------------------------------- 382 | def doPayment(self, params): 383 | q = TXN_QUERIES["PAYMENT"] 384 | 385 | w_id = params["w_id"] 386 | d_id = params["d_id"] 387 | h_amount = params["h_amount"] 388 | c_w_id = params["c_w_id"] 389 | c_d_id = params["c_d_id"] 390 | c_id = params["c_id"] 391 | c_last = params["c_last"] 392 | h_date = params["h_date"] 393 | 394 | if c_id != None: 395 | self.cursor.execute(q["getCustomerByCustomerId"], [w_id, d_id, c_id]) 396 | customer = self.cursor.fetchone() 397 | else: 398 | # Get the midpoint customer's id 399 | self.cursor.execute(q["getCustomersByLastName"], [w_id, d_id, c_last]) 400 | all_customers = self.cursor.fetchall() 401 | assert len(all_customers) > 0 402 | namecnt = len(all_customers) 403 | index = (namecnt-1)/2 404 | customer = all_customers[index] 405 | c_id = customer[0] 406 | assert len(customer) > 0 407 | c_balance = customer[14] - h_amount 408 | c_ytd_payment = customer[15] + h_amount 409 | c_payment_cnt = customer[16] + 1 410 | c_data = customer[17] 411 | 412 | self.cursor.execute(q["getWarehouse"], [w_id]) 413 | warehouse = self.cursor.fetchone() 414 | 415 | self.cursor.execute(q["getDistrict"], [w_id, d_id]) 416 | district = self.cursor.fetchone() 417 | 418 | self.cursor.execute(q["updateWarehouseBalance"], [h_amount, w_id]) 419 | self.cursor.execute(q["updateDistrictBalance"], [h_amount, w_id, d_id]) 420 | 421 | # Customer Credit Information 422 | if customer[11] == constants.BAD_CREDIT: 423 | newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) 424 | c_data = (newData + "|" + c_data) 425 | if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] 426 | self.cursor.execute(q["updateBCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id]) 427 | else: 428 | c_data = "" 429 | self.cursor.execute(q["updateGCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id]) 430 | 431 | # Concatenate w_name, four spaces, d_name 432 | h_data = "%s %s" % (warehouse[0], district[0]) 433 | # Create the history record 434 | self.cursor.execute(q["insertHistory"], [c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data]) 435 | 436 | self.conn.commit() 437 | 438 | # TPC-C 2.5.3.3: Must display the following fields: 439 | # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, 440 | # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, 441 | # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, 442 | # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), 443 | # H_AMOUNT, and H_DATE. 444 | 445 | # Hand back all the warehouse, district, and customer data 446 | return [ warehouse, district, customer ] 447 | 448 | ## ---------------------------------------------- 449 | ## doStockLevel 450 | ## ---------------------------------------------- 451 | def doStockLevel(self, params): 452 | q = TXN_QUERIES["STOCK_LEVEL"] 453 | 454 | w_id = params["w_id"] 455 | d_id = params["d_id"] 456 | threshold = params["threshold"] 457 | 458 | self.cursor.execute(q["getOId"], [w_id, d_id]) 459 | result = self.cursor.fetchone() 460 | assert result 461 | o_id = result[0] 462 | 463 | self.cursor.execute(q["getStockCount"], [w_id, d_id, o_id, (o_id - 20), w_id, threshold]) 464 | result = self.cursor.fetchone() 465 | 466 | self.conn.commit() 467 | 468 | return int(result[0]) 469 | 470 | ## CLASS -------------------------------------------------------------------------------- /pytpcc/drivers/postgresqldriver.py: -------------------------------------------------------------------------------- 1 | # pytpcc/drivers/postgresqldriver.py 2 | 3 | from __future__ import with_statement 4 | 5 | import psycopg2 6 | import logging 7 | import traceback 8 | from pprint import pformat 9 | from time import sleep 10 | 11 | import constants 12 | from .abstractdriver import AbstractDriver 13 | 14 | TXN_QUERIES = { 15 | "DELIVERY": { 16 | "getNewOrder": "SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = %s AND NO_W_ID = %s AND NO_O_ID > -1 LIMIT 1", # 17 | "deleteNewOrder": "DELETE FROM NEW_ORDER WHERE NO_D_ID = %s AND NO_W_ID = %s AND NO_O_ID = %s", # d_id, w_id, no_o_id 18 | "getCId": "SELECT O_C_ID FROM ORDERS WHERE O_ID = %s AND O_D_ID = %s AND O_W_ID = %s", # no_o_id, d_id, w_id 19 | "updateOrders": "UPDATE ORDERS SET O_CARRIER_ID = %s WHERE O_ID = %s AND O_D_ID = %s AND O_W_ID = %s", # o_carrier_id, no_o_id, d_id, w_id 20 | "updateOrderLine": "UPDATE ORDER_LINE SET OL_DELIVERY_D = %s WHERE OL_O_ID = %s AND OL_D_ID = %s AND OL_W_ID = %s", # o_entry_d, no_o_id, d_id, w_id 21 | "sumOLAmount": "SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = %s AND OL_D_ID = %s AND OL_W_ID = %s", # no_o_id, d_id, w_id 22 | "updateCustomer": "UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + %s WHERE C_ID = %s AND C_D_ID = %s AND C_W_ID = %s", # ol_total, c_id, d_id, w_id 23 | }, 24 | "NEW_ORDER": { 25 | "getWarehouseTaxRate": "SELECT W_TAX FROM WAREHOUSE WHERE W_ID = %s", # w_id 26 | "getDistrict": "SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = %s AND D_W_ID = %s", # d_id, w_id 27 | "incrementNextOrderId": "UPDATE DISTRICT SET D_NEXT_O_ID = %s WHERE D_ID = %s AND D_W_ID = %s", # d_next_o_id, d_id, w_id 28 | "getCustomer": "SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE C_W_ID = %s AND C_D_ID = %s AND C_ID = %s", # w_id, d_id, c_id 29 | "createOrder": "INSERT INTO ORDERS (O_ID, O_D_ID, O_W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (%s, %s, %s, %s, %s, %s, %s, %s::integer)", # d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, o_ol_cnt, o_all_local 30 | "createNewOrder": "INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, NO_W_ID) VALUES (%s, %s, %s)", # o_id, d_id, w_id 31 | "getItemInfo": "SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = %s", # ol_i_id 32 | "getStockInfo": "SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_{:02d} FROM STOCK WHERE S_I_ID = %s AND S_W_ID = %s", # d_id, ol_i_id, ol_supply_w_id 33 | "updateStock": "UPDATE STOCK SET S_QUANTITY = %s, S_YTD = %s, S_ORDER_CNT = %s, S_REMOTE_CNT = %s WHERE S_I_ID = %s AND S_W_ID = %s", # s_quantity, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id 34 | "createOrderLine": "INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, OL_W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", # o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info 35 | }, 36 | 37 | "ORDER_STATUS": { 38 | "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = %s AND C_D_ID = %s AND C_ID = %s", # w_id, d_id, c_id 39 | "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE C_W_ID = %s AND C_D_ID = %s AND C_LAST = %s ORDER BY C_FIRST", # w_id, d_id, c_last 40 | "getLastOrder": "SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE O_W_ID = %s AND O_D_ID = %s AND O_C_ID = %s ORDER BY O_ID DESC LIMIT 1", # w_id, d_id, c_id 41 | "getOrderLines": "SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE OL_W_ID = %s AND OL_D_ID = %s AND OL_O_ID = %s", # w_id, d_id, o_id 42 | }, 43 | 44 | "PAYMENT": { 45 | "getWarehouse": "SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = %s", # w_id 46 | "updateWarehouseBalance": "UPDATE WAREHOUSE SET W_YTD = W_YTD + %s WHERE W_ID = %s", # h_amount, w_id 47 | "getDistrict": "SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE D_W_ID = %s AND D_ID = %s", # w_id, d_id 48 | "updateDistrictBalance": "UPDATE DISTRICT SET D_YTD = D_YTD + %s WHERE D_W_ID = %s AND D_ID = %s", # h_amount, d_w_id, d_id 49 | "getCustomerByCustomerId": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = %s AND C_D_ID = %s AND C_ID = %s", # w_id, d_id, c_id 50 | "getCustomersByLastName": "SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE C_W_ID = %s AND C_D_ID = %s AND C_LAST = %s ORDER BY C_FIRST", # w_id, d_id, c_last 51 | "updateBCCustomer": "UPDATE CUSTOMER SET C_BALANCE = %s, C_YTD_PAYMENT = %s, C_PAYMENT_CNT = %s, C_DATA = %s WHERE C_W_ID = %s AND C_D_ID = %s AND C_ID = %s", # c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id 52 | "updateGCCustomer": "UPDATE CUSTOMER SET C_BALANCE = %s, C_YTD_PAYMENT = %s, C_PAYMENT_CNT = %s WHERE C_W_ID = %s AND C_D_ID = %s AND C_ID = %s", # c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id 53 | "insertHistory": "INSERT INTO HISTORY VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", 54 | }, 55 | 56 | "STOCK_LEVEL": { 57 | "getOId": "SELECT D_NEXT_O_ID FROM DISTRICT WHERE D_W_ID = %s AND D_ID = %s", 58 | "getStockCount": """ 59 | SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK 60 | WHERE OL_W_ID = %s 61 | AND OL_D_ID = %s 62 | AND OL_O_ID < %s 63 | AND OL_O_ID >= %s 64 | AND S_W_ID = %s 65 | AND S_I_ID = OL_I_ID 66 | AND S_QUANTITY < %s 67 | """, 68 | }, 69 | } 70 | 71 | class PostgresqlDriver(AbstractDriver): 72 | DEFAULT_CONFIG = { 73 | "database": ("The name of the PostgreSQL database", "tpcc"), 74 | "host": ("The host address of the PostgreSQL server", "localhost"), 75 | "port": ("The port number of the PostgreSQL server", 5432), 76 | "user": ("The username to connect to the PostgreSQL database", "postgres"), 77 | "password": ("The password to connect to the PostgreSQL database", ""), 78 | "isolation-level": ("The transaction isolation-level for the PostgreSQL database. Valid values are `default`, `read_uncommitted`, `read_committed`, `repeatable_read`, `serializable`", "default") 79 | } 80 | 81 | def __init__(self, ddl): 82 | super(PostgresqlDriver, self).__init__("postgresql", ddl) 83 | self.conn = None 84 | self.cursor = None 85 | 86 | def makeDefaultConfig(self): 87 | return PostgresqlDriver.DEFAULT_CONFIG 88 | 89 | def loadConfig(self, config): 90 | for key in PostgresqlDriver.DEFAULT_CONFIG.keys(): 91 | assert key in config, "Missing parameter '%s' in %s configuration" % (key, self.name) 92 | 93 | self.conn = psycopg2.connect( 94 | dbname=config["database"], 95 | user=config["user"], 96 | password=config["password"], 97 | host=config["host"], 98 | port=config["port"] 99 | ) 100 | 101 | self.conn.isolation_level = psycopg2.extensions.ISOLATION_LEVEL_DEFAULT 102 | if config["isolation-level"]: 103 | isolation_level = config["isolation-level"].lower() 104 | 105 | if isolation_level == "read_uncommitted": 106 | self.conn.isolation_level = psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED 107 | print("Isolation level set to READ UNCOMMITTED.") 108 | elif isolation_level == "read_committed": 109 | self.conn.isolation_level = psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED 110 | print("Isolation level set to READ COMMITTED.") 111 | elif isolation_level == "repeatable_read": 112 | self.conn.isolation_level = psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ 113 | print("Isolation level set to REPEATABLE READ.") 114 | elif isolation_level == "serializable": 115 | self.conn.isolation_level = psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE 116 | print("Isolation level set to SERIALIZABLE.") 117 | elif isolation_level == "default": 118 | print("Isolation level set to DEFAULT (PostgreSQL's default).") 119 | else: 120 | print(f"Warning: Unrecognized isolation level '{config['isolation-level']}. Valid values are `default`, `read_uncommitted`, `read_committed`, `repeatable_read`, `serializable`'. Using the PostgreSQL default.") 121 | 122 | self.cursor = self.conn.cursor() 123 | 124 | if config["reset"]: 125 | logging.info("Dropping all tables in database '%s'", config["database"]) 126 | self.conn.autocommit = True # Enable autocommit mode 127 | self.cursor.execute(""" 128 | DO $$ DECLARE 129 | r RECORD; 130 | BEGIN 131 | FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP 132 | EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE'; 133 | END LOOP; 134 | END $$; 135 | """) 136 | self.conn.autocommit = False # Disable autocommit mode 137 | 138 | logging.info("Restoring schema definitions in database '%s'", config["database"]) 139 | with open(self.ddl, "r") as f: 140 | self.cursor.execute(f.read()) 141 | self.conn.commit() 142 | 143 | def loadStart(self): 144 | """Disable constraints before data load 145 | and check default isolation level""" 146 | self.cursor.execute("SHOW TRANSACTION ISOLATION LEVEL") 147 | isolation_level = self.cursor.fetchone()[0] 148 | if isolation_level == "read committed": 149 | logging.warn("Read Committed isolation level could cause duplicate key errors, to avoid them use 'repeatable read'") 150 | 151 | self.cursor.execute("set session_replication_role to replica") 152 | 153 | def loadTuples(self, tableName, tuples): 154 | if len(tuples) == 0: 155 | return 156 | placeholders = ', '.join(['%s'] * len(tuples[0])) 157 | sql = f"INSERT INTO {tableName} VALUES ({placeholders})" 158 | self.cursor.executemany(sql, tuples) 159 | logging.debug("Loaded %d tuples for tableName %s" % (len(tuples), tableName)) 160 | return 161 | 162 | def loadFinish(self): 163 | self.cursor.execute("set session_replication_role to default;") 164 | logging.debug("Committing changes to database") 165 | self.conn.commit() 166 | 167 | ## ---------------------------------------------- 168 | ## doDelivery 169 | ## ---------------------------------------------- 170 | def doDelivery(self, params): 171 | retries = 0 172 | q = TXN_QUERIES["DELIVERY"] 173 | 174 | w_id = params["w_id"] 175 | o_carrier_id = params["o_carrier_id"] 176 | ol_delivery_d = params["ol_delivery_d"] 177 | 178 | while True: 179 | try: 180 | result = [ ] 181 | for d_id in range(1, constants.DISTRICTS_PER_WAREHOUSE+1): 182 | self.cursor.execute(q["getNewOrder"], [d_id, w_id]) 183 | newOrder = self.cursor.fetchone() 184 | if newOrder == None: 185 | ## No orders for this district: skip it. Note: This must be reported if > 1% 186 | continue 187 | assert len(newOrder) > 0 188 | no_o_id = newOrder[0] 189 | 190 | self.cursor.execute(q["getCId"], [no_o_id, d_id, w_id]) 191 | c_id = self.cursor.fetchone()[0] 192 | 193 | self.cursor.execute(q["sumOLAmount"], [no_o_id, d_id, w_id]) 194 | ol_total = self.cursor.fetchone()[0] 195 | 196 | self.cursor.execute(q["deleteNewOrder"], [d_id, w_id, no_o_id]) 197 | self.cursor.execute(q["updateOrders"], [o_carrier_id, no_o_id, d_id, w_id]) 198 | self.cursor.execute(q["updateOrderLine"], [ol_delivery_d, no_o_id, d_id, w_id]) 199 | 200 | # These must be logged in the "result file" according to TPC-C 2.7.2.2 (page 39) 201 | # We remove the queued time, completed time, w_id, and o_carrier_id: the client can figure 202 | # them out 203 | # If there are no order lines, SUM returns null. There should always be order lines. 204 | assert ol_total != None, "ol_total is NULL: there are no order lines. This should not happen" 205 | assert ol_total > 0.0 206 | 207 | self.cursor.execute(q["updateCustomer"], [ol_total, c_id, d_id, w_id]) 208 | 209 | result.append((d_id, no_o_id)) 210 | ## FOR 211 | self.conn.commit() 212 | return (result,retries) 213 | except Exception as e: 214 | self.conn.rollback() # Rollback the transaction on error 215 | retries += 1 216 | sleep(retries * .1) 217 | 218 | ## ---------------------------------------------- 219 | ## doNewOrder 220 | ## ---------------------------------------------- 221 | def doNewOrder(self, params): 222 | q = TXN_QUERIES["NEW_ORDER"] 223 | retries = 0 224 | w_id = params["w_id"] 225 | d_id = params["d_id"] 226 | c_id = params["c_id"] 227 | o_entry_d = params["o_entry_d"] 228 | i_ids = params["i_ids"] 229 | i_w_ids = params["i_w_ids"] 230 | i_qtys = params["i_qtys"] 231 | 232 | assert len(i_ids) > 0 233 | assert len(i_ids) == len(i_w_ids) 234 | assert len(i_ids) == len(i_qtys) 235 | 236 | all_local = True 237 | 238 | while True: 239 | try: 240 | items = [ ] 241 | for i in range(len(i_ids)): 242 | ## Determine if this is an all local order or not 243 | all_local = all_local and i_w_ids[i] == w_id 244 | self.cursor.execute(q["getItemInfo"], [i_ids[i]]) 245 | items.append(self.cursor.fetchone()) 246 | assert len(items) == len(i_ids) 247 | 248 | ## TPCC defines 1% of neworder gives a wrong itemid, causing rollback. 249 | ## Note that this will happen with 1% of transactions on purpose. 250 | for item in items: 251 | if item is None: 252 | return (None, retries) 253 | ## FOR 254 | 255 | ## ---------------- 256 | ## Collect Information from WAREHOUSE, DISTRICT, and CUSTOMER 257 | ## ---------------- 258 | self.cursor.execute(q["getWarehouseTaxRate"], [w_id]) 259 | w_tax = self.cursor.fetchone()[0] 260 | 261 | self.cursor.execute(q["getDistrict"], [d_id, w_id]) 262 | district_info = self.cursor.fetchone() 263 | d_tax = district_info[0] 264 | d_next_o_id = district_info[1] 265 | 266 | self.cursor.execute(q["getCustomer"], [w_id, d_id, c_id]) 267 | customer_info = self.cursor.fetchone() 268 | c_discount = customer_info[0] 269 | 270 | ## ---------------- 271 | ## Insert Order Information 272 | ## ---------------- 273 | ol_cnt = len(i_ids) 274 | o_carrier_id = constants.NULL_CARRIER_ID 275 | 276 | self.cursor.execute(q["incrementNextOrderId"], [d_next_o_id + 1, d_id, w_id]) 277 | self.cursor.execute(q["createOrder"], [d_next_o_id, d_id, w_id, c_id, o_entry_d, o_carrier_id, ol_cnt, all_local]) 278 | self.cursor.execute(q["createNewOrder"], [d_next_o_id, d_id, w_id]) 279 | 280 | ## ---------------- 281 | ## Insert Order Item Information 282 | ## ---------------- 283 | item_data = [ ] 284 | total = 0 285 | for i in range(len(i_ids)): 286 | ol_number = i + 1 287 | ol_supply_w_id = i_w_ids[i] 288 | ol_i_id = i_ids[i] 289 | ol_quantity = i_qtys[i] 290 | 291 | itemInfo = items[i] 292 | i_name = itemInfo[1] 293 | i_data = itemInfo[2] 294 | i_price = itemInfo[0] 295 | 296 | self.cursor.execute(q["getStockInfo"].format(d_id), [ol_i_id, ol_supply_w_id]) 297 | 298 | stockInfo = self.cursor.fetchone() 299 | if len(stockInfo) == 0: 300 | logging.warn("No STOCK record for (ol_i_id=%d, ol_supply_w_id=%d)" % (ol_i_id, ol_supply_w_id)) 301 | continue 302 | s_quantity = stockInfo[0] 303 | s_ytd = stockInfo[2] 304 | s_order_cnt = stockInfo[3] 305 | s_remote_cnt = stockInfo[4] 306 | s_data = stockInfo[1] 307 | s_dist_xx = stockInfo[5] # Fetches data from the s_dist_[d_id] column 308 | 309 | ## Update stock 310 | s_ytd += ol_quantity 311 | if s_quantity >= ol_quantity + 10: 312 | s_quantity = s_quantity - ol_quantity 313 | else: 314 | s_quantity = s_quantity + 91 - ol_quantity 315 | s_order_cnt += 1 316 | 317 | if ol_supply_w_id != w_id: s_remote_cnt += 1 318 | 319 | self.cursor.execute(q["updateStock"], [s_quantity, s_ytd, s_order_cnt, s_remote_cnt, ol_i_id, ol_supply_w_id]) 320 | 321 | if i_data.find(constants.ORIGINAL_STRING) != -1 and s_data.find(constants.ORIGINAL_STRING) != -1: 322 | brand_generic = 'B' 323 | else: 324 | brand_generic = 'G' 325 | 326 | ## Transaction profile states to use "ol_quantity * i_price" 327 | ol_amount = ol_quantity * i_price 328 | total += ol_amount 329 | 330 | self.cursor.execute(q["createOrderLine"], [d_next_o_id, d_id, w_id, ol_number, ol_i_id, ol_supply_w_id, o_entry_d, ol_quantity, ol_amount, s_dist_xx]) 331 | 332 | ## Add the info to be returned 333 | item_data.append( (i_name, s_quantity, brand_generic, i_price, ol_amount) ) 334 | ## FOR 335 | ## Commit! 336 | self.conn.commit() 337 | 338 | total *= (1 - c_discount) * (1 + w_tax + d_tax) 339 | 340 | ## Pack up values the client is missing (see TPC-C 2.4.3.5) 341 | misc = [ (w_tax, d_tax, d_next_o_id, total) ] 342 | return ([ customer_info, misc, item_data ], retries) 343 | except Exception as e: 344 | self.conn.rollback() # Rollback the transaction on error 345 | retries += 1 346 | sleep(retries * .1) 347 | 348 | ## ---------------------------------------------- 349 | ## doOrderStatus 350 | ## ---------------------------------------------- 351 | def doOrderStatus(self, params): 352 | q = TXN_QUERIES["ORDER_STATUS"] 353 | 354 | retries = 0 355 | w_id = params["w_id"] 356 | d_id = params["d_id"] 357 | c_id = params["c_id"] 358 | c_last = params["c_last"] 359 | 360 | assert w_id, pformat(params) 361 | assert d_id, pformat(params) 362 | 363 | while True: 364 | try: 365 | if c_id != None: 366 | self.cursor.execute(q["getCustomerByCustomerId"], [w_id, d_id, c_id]) 367 | customer = self.cursor.fetchone() 368 | else: 369 | # Get the midpoint customer's id 370 | self.cursor.execute(q["getCustomersByLastName"], [w_id, d_id, c_last]) 371 | all_customers = self.cursor.fetchall() 372 | assert len(all_customers) > 0 373 | namecnt = len(all_customers) 374 | index = (namecnt-1)/2 375 | customer = all_customers[int(index)] 376 | c_id = customer[0] 377 | assert len(customer) > 0 378 | assert c_id != None 379 | 380 | self.cursor.execute(q["getLastOrder"], [w_id, d_id, c_id]) 381 | order = self.cursor.fetchone() 382 | if order: 383 | self.cursor.execute(q["getOrderLines"], [w_id, d_id, order[0]]) 384 | orderLines = self.cursor.fetchall() 385 | else: 386 | orderLines = [ ] 387 | 388 | self.conn.commit() 389 | return ([ customer, order, orderLines ],retries) 390 | except Exception as e: 391 | self.conn.rollback() # Rollback the transaction on error 392 | retries += 1 393 | sleep(retries * .1) 394 | 395 | ## ---------------------------------------------- 396 | ## doPayment 397 | ## ---------------------------------------------- 398 | def doPayment(self, params): 399 | q = TXN_QUERIES["PAYMENT"] 400 | retries = 0 401 | w_id = params["w_id"] 402 | d_id = params["d_id"] 403 | h_amount = params["h_amount"] 404 | c_w_id = params["c_w_id"] 405 | c_d_id = params["c_d_id"] 406 | c_id = params["c_id"] 407 | c_last = params["c_last"] 408 | h_date = params["h_date"] 409 | 410 | while True: 411 | try: 412 | if c_id != None: 413 | self.cursor.execute(q["getCustomerByCustomerId"], [w_id, d_id, c_id]) 414 | customer = self.cursor.fetchone() 415 | else: 416 | # Get the midpoint customer's id 417 | self.cursor.execute(q["getCustomersByLastName"], [w_id, d_id, c_last]) 418 | all_customers = self.cursor.fetchall() 419 | assert len(all_customers) > 0 420 | namecnt = len(all_customers) 421 | index = (namecnt-1)/2 422 | customer = all_customers[int(index)] 423 | c_id = customer[0] 424 | assert len(customer) > 0 425 | c_balance = customer[14] - h_amount 426 | c_ytd_payment = customer[15] + h_amount 427 | c_payment_cnt = customer[16] + 1 428 | c_data = customer[17] 429 | 430 | self.cursor.execute(q["getWarehouse"], [w_id]) 431 | warehouse = self.cursor.fetchone() 432 | 433 | self.cursor.execute(q["getDistrict"], [w_id, d_id]) 434 | district = self.cursor.fetchone() 435 | 436 | self.cursor.execute(q["updateWarehouseBalance"], [h_amount, w_id]) 437 | self.cursor.execute(q["updateDistrictBalance"], [h_amount, w_id, d_id]) 438 | 439 | # Customer Credit Information 440 | if customer[11] == constants.BAD_CREDIT: 441 | newData = " ".join(map(str, [c_id, c_d_id, c_w_id, d_id, w_id, h_amount])) 442 | c_data = (newData + "|" + c_data) 443 | if len(c_data) > constants.MAX_C_DATA: c_data = c_data[:constants.MAX_C_DATA] 444 | self.cursor.execute(q["updateBCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_data, c_w_id, c_d_id, c_id]) 445 | else: 446 | c_data = "" 447 | self.cursor.execute(q["updateGCCustomer"], [c_balance, c_ytd_payment, c_payment_cnt, c_w_id, c_d_id, c_id]) 448 | 449 | # Concatenate w_name, four spaces, d_name 450 | h_data = "%s %s" % (warehouse[0], district[0]) 451 | # Create the history record 452 | self.cursor.execute(q["insertHistory"], [c_id, c_d_id, c_w_id, d_id, w_id, h_date, h_amount, h_data]) 453 | 454 | self.conn.commit() 455 | 456 | # TPC-C 2.5.3.3: Must display the following fields: 457 | # W_ID, D_ID, C_ID, C_D_ID, C_W_ID, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP, 458 | # D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, 459 | # C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, 460 | # C_DISCOUNT, C_BALANCE, the first 200 characters of C_DATA (only if C_CREDIT = "BC"), 461 | # H_AMOUNT, and H_DATE. 462 | 463 | # Hand back all the warehouse, district, and customer data 464 | return ([ warehouse, district, customer ],retries) 465 | except Exception as e: 466 | self.conn.rollback() # Rollback the transaction on error 467 | retries += 1 468 | sleep(retries * .1) 469 | 470 | ## ---------------------------------------------- 471 | ## doStockLevel 472 | ## ---------------------------------------------- 473 | def doStockLevel(self, params): 474 | q = TXN_QUERIES["STOCK_LEVEL"] 475 | retries = 0 476 | w_id = params["w_id"] 477 | d_id = params["d_id"] 478 | threshold = params["threshold"] 479 | while True: 480 | try: 481 | self.cursor.execute(q["getOId"], [w_id, d_id]) 482 | result = self.cursor.fetchone() 483 | assert result 484 | o_id = result[0] 485 | 486 | self.cursor.execute(q["getStockCount"], [w_id, d_id, o_id, (o_id - 20), w_id, threshold]) 487 | result = self.cursor.fetchone() 488 | 489 | self.conn.commit() 490 | 491 | return (int(result[0]),retries) 492 | except Exception as e: 493 | self.conn.rollback() # Rollback the transaction on error 494 | retries += 1 495 | sleep(retries * .1) 496 | 497 | ## ---------------------------------------------- 498 | ## getNumberWH 499 | ## ---------------------------------------------- 500 | def getNumberWH(self): 501 | self.cursor.execute("SELECT max(w_id) FROM WAREHOUSE") 502 | return self.cursor.fetchone()[0] 503 | 504 | ## CLASS --------------------------------------------------------------------------------