├── .flake8 ├── .github └── FUNDING.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.spec ├── docker ├── .env ├── Dockerfile.build ├── docker-compose-mariadb.yml └── docker-compose-postgres.yml ├── image ├── NBA-ER.jpg └── james-harden-shot-analysis-2020-21.webp ├── requirements.txt ├── requirements_no_gui.txt ├── scripts ├── create_maria.sh ├── create_postgres.sh ├── create_sqlite.sh ├── drop.sql ├── drop_all.sh ├── drop_sqlite.sh ├── fill_postgres.sh ├── refresh_postgres.sh ├── refresh_sqlite.sh └── release │ ├── build.sh │ ├── build_docker.sh │ ├── build_exe.txt │ └── build_no_gui.sh └── stats ├── .gitignore ├── __init__.py ├── args.py ├── constants.py ├── db_utils.py ├── event_message_type.py ├── game.py ├── general_requester.py ├── gui.py ├── models ├── EventMessageType.py ├── Game.py ├── PlayByPlay.py ├── PlayByPlayV3.py ├── Player.py ├── PlayerGameLog.py ├── PlayerGameLogTemp.py ├── PlayerGeneralTraditionalTotal.py ├── PlayerSeason.py ├── Season.py ├── ShotChartDetail.py ├── ShotChartDetailTemp.py ├── Team.py ├── TeamGameLog.py ├── TeamSeason.py └── __init__.py ├── nba_sql.py ├── play_by_play.py ├── play_by_playv3.py ├── player.py ├── player_game_log.py ├── player_general_traditional_total.py ├── player_season.py ├── season.py ├── settings.py ├── shot_chart_detail.py ├── team.py └── utils.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | count=True 3 | filename=*.py 4 | exclude = 5 | .git, 6 | .conf, 7 | __pycache__, 8 | build, 9 | dist, 10 | .gitignore 11 | .flake8, 12 | *.md, 13 | *.egg-info, 14 | *.ini, 15 | *.conf, 16 | *.yaml, 17 | *.yml, 18 | *.txt 19 | max-line-length=120 20 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | 4 | patreon: mpope 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | .vscode/ 3 | build/ 4 | dist/ 5 | *.swp 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # :basketball: nba-sql 2 | 3 | [![Github All Releases](https://img.shields.io/github/downloads/mpope9/nba-sql/total.svg)]() 4 | 5 | An application to build a Postgres, MySQL/MariaDB, or SQLite NBA database from the public API. 6 | 7 | The latest Linux, MacOS, and Windows releases [can be found in the releases section.](https://github.com/mpope9/nba-sql/releases/tag/v0.1.0). 8 | 9 | Here is an example query which can be used after building the database. Lets say we want to find Russell Westbrook's total Triple-Doubles: 10 | ```SQL 11 | SELECT SUM(td3) 12 | FROM player_game_log 13 | LEFT JOIN player ON player.player_id = player_game_log.player_id 14 | WHERE player.player_name = 'Russell Westbrook'; 15 | ``` 16 | Check the [wiki/Example-Queries](https://github.com/mpope9/nba-sql/wiki/Example-Queries) for more example queries. 17 | 18 | Here is an example shot visualization using the `shot_chart_detail` table ([Apache ECharts](https://echarts.apache.org/en/index.html)): 19 | 20 | ![James Harden Shot Chart 2020-21](image/james-harden-shot-analysis-2020-21.webp) 21 | 22 | The default behavior is to load the current season (or update if it already exists) into a SQLite database. 23 | 24 | To load data into DuckDB, use the [SQLite Extension](https://duckdb.org/docs/extensions/sqlite.html) after creating a SQLite database. 25 | 26 | # Getting Started 27 | 28 | * [A good place for more information is the wiki](https://github.com/mpope9/nba-sql/wiki). 29 | * [Looking to contribute? Check the list of open issues!](https://github.com/mpope9/nba-sql/issues) 30 | 31 | The following environment variables _must_ be set. *There are no commandline arguments to specify these.* The following example are connection details for the provided docker-compose database: 32 | ```bash 33 | DB_NAME="nba" 34 | DB_HOST="localhost" 35 | DB_USER="nba_sql" 36 | DB_PASSWORD="nba_sql" 37 | ``` 38 | 39 | It will take an estimated 6 hours to build the whole database. However, some tables take much longer than others due to the amount of data: `play_by_play`, `play_by_playv3` `shot_chart_detail`, and `pgtt` in particular. These can be skilled with the `--skip-tables` option. Most basic queries can use the `player_game_log` (which is unskippable). 40 | 41 | Note there are `play_by_play` and `play_by_playv3` tables. `play_by_play` had more detailed descriptions but `play_by_playv3` is broken down in a more sensible way. It is very difficult to correlate which player is associated with a `_description` column in the `play_by_play` table. 42 | 43 | ## Commandline Reference 44 | ``` 45 | >python stats/nba_sql.py --help 46 | usage: nba_sql.py [-h] [--database {mysql,postgres,sqlite}] [--database_name DATABASE_NAME] [--database_host DATABASE_HOST] [--username USERNAME] [--create-schema] [--time-between-requests REQUEST_GAP] 47 | [--batch_size BATCH_SIZE] [--sqlite-path SQLITE_PATH] [--quiet] [--default-mode] [--current-season-mode] [--password PASSWORD] 48 | [--seasons [{1997-98,1998-99,1999-00,2000-01,2001-02,2002-03,2003-04,2004-05,2005-06,2006-07,2007-08,2008-09,2009-10,2010-11,2011-12,2012-13,2013-14,2014-15,2015-16,2016-17,2017-18,2018-19,2019-20,2020-21,2021-22,2022-23,2023-24,2024-25} ...]] 49 | [--skip-tables [{player_season,player_game_log,play_by_play,pgtt,shot_chart_detail,game,event_message_type,team,player,} ...]] 50 | 51 | nba-sql 52 | 53 | options: 54 | -h, --help show this help message and exit 55 | --database {mysql,postgres,sqlite} 56 | The database flag specifies which database protocol to use. Defaults to "sqlite", but also accepts "postgres" and "mysql". 57 | --database_name DATABASE_NAME 58 | Database Name (Not Needed For SQLite) 59 | --database_host DATABASE_HOST 60 | Database Hostname (Not Needed For SQLite) 61 | --username USERNAME Database Username (Not Needed For SQLite) 62 | --create-schema Flag to initialize the database schema before loading data. If the schema already exists then nothing will happen. 63 | --time-between-requests REQUEST_GAP 64 | This flag exists to prevent rate limiting, and injects the desired amount of time inbetween requesting resources. 65 | --batch_size BATCH_SIZE 66 | Inserts BATCH_SIZE chunks of rows to the database. This value is ignored when selecting database 'sqlite'. 67 | --sqlite-path SQLITE_PATH 68 | Setting to define sqlite path. 69 | --quiet Setting to define stdout logging level. If set, only "ok" will be printed if ran successfully. This currently only applies to refreshing a db, and not loading one. 70 | --default-mode Mode to create the database and load historic data. Use this mode when creating a new database or when trying to load a specific season or a range of seasons. 71 | --current-season-mode 72 | Mode to refresh the current season. Use this mode on an existing database to update it with the latest data. 73 | --password PASSWORD Database Password (Not Needed For SQLite) 74 | --seasons [{1997-98,1998-99,1999-00,2000-01,2001-02,2002-03,2003-04,2004-05,2005-06,2006-07,2007-08,2008-09,2009-10,2010-11,2011-12,2012-13,2013-14,2014-15,2015-16,2016-17,2017-18,2018-19,2019-20,2020-21,2021-22,2022-23,2023-24,2024-25} ...] 75 | The seasons flag loads the database with the specified season. The format of the season should be in the form "YYYY-YY". The default behavior is loading the current season. 76 | --skip-tables [{player_season,player_game_log,play_by_play,pgtt,shot_chart_detail,game,event_message_type,team,player,} ...] 77 | Use this option to skip loading certain tables. 78 | ``` 79 | 80 | ## :crystal_ball: Schema 81 | #### Supported Tables 82 | * player 83 | * team 84 | * game 85 | * play_by_play 86 | * player_game_log 87 | * player_season 88 | * team_game_log 89 | * team_season 90 | * player_general_traditional_total (Also referred to in short as pgtt) 91 | * shot_chart_detail 92 | 93 | An up-to-date ER diagram can be found in [`image/NBA-ER.jpg`](https://github.com/mpope9/nba-sql/blob/main/image/NBA-ER.jpg). 94 | ![ERD](image/NBA-ER.jpg) 95 | 96 | ## :wrench: Building From Scratch 97 | 98 | Requirements: 99 | 100 | Python >= 3.8 101 | 102 | ### :scroll: Provided Scripts 103 | 104 | In the `scripts` directory, we provide a way to create the schema and load the data for a Postgres database. We also provide a docker-compose setup for development and to preview the data. 105 | 106 | ```shell 107 | # Required if you're on Debian based systems 108 | sudo service postgresql stop 109 | 110 | docker-compose -f docker/docker-compose-postgres.yml up -d 111 | 112 | pip install -r requirements.txt 113 | 114 | ./scripts/create_postgres.sh 115 | ``` 116 | 117 | If you want to use MariaDB, start it with: 118 | ``` 119 | docker-compose -f docker/docker-compose-mariadb.yml up -d 120 | 121 | ./scripts/create_maria.sh 122 | ``` 123 | 124 | ### :snake: Directly Calling Python 125 | 126 | The entrypoint is `stats/nba_sql.py`. To see the available arguments, you can use: 127 | ```bash 128 | python stats/nba_sql.py -h 129 | ``` 130 | 131 | To create the schema, use the `--create-schema`. Example: 132 | ```bash 133 | pyhton stats/nba_sql.py --create-schema 134 | ``` 135 | 136 | To enable a Postgres database, use the `--database` flag. Example: 137 | ```bash 138 | python stats/nba_sql.py --database="postgres" 139 | ``` 140 | 141 | We have added a half second delay between making requests to the NBA stats API. To configure the amount of time use the `--time-between-requests` flag. 142 | ```bash 143 | python stats/nba_sql.py --time-between-requests=.5 144 | ``` 145 | 146 | The script `nba_sql.py` adds several tables into the database. Loading these tables takes time, notably, the `play_by_play` table. 147 | Some of these tables can be skipped by using the `--skip-tables` CLI option. Example: 148 | 149 | ```bash 150 | python stats/nba_sql.py --create-schema --database postgres --skip-tables play_by_play pgtt 151 | ``` 152 | 153 | ### :computer: Local development 154 | 155 | #### Setup 156 | Create your virtual environment if you don’t have one already. In this case we use `venv` as the target folder for storing packages. 157 | 158 | `python -m venv venv` 159 | 160 | Then activate it: 161 | `source venv/bin/activate` 162 | 163 | Install dependencies using: 164 | `pip install -r requirements.txt` 165 | 166 | Or if you don't want to install the GUI deps you can use: 167 | `pip install -r requirements_no_gui.txt` 168 | 169 | ##### MacOS Errors 170 | 171 | If you try to setup on MacOS and see an error like 172 | ``` 173 | Error: pg_config executable not found. 174 | ``` 175 | 176 | This can be resolved by installing `postgresql` through Homebrew: 177 | ```bash 178 | brew install postgresql 179 | ``` 180 | 181 | # :pray: Acknowledgements 182 | * [@avadhanij](https://github.com/avadhanij): For guidance and knowledge. 183 | * [nba_api project](https://github.com/swar/nba_api): A great resource to reference for endpoint documentation. 184 | * BurntSushi's [nfldb](https://github.com/BurntSushi/nfldb): The inspiration for this project. 185 | -------------------------------------------------------------------------------- /build.spec: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import gooey 4 | from PyInstaller.building.api import EXE, PYZ, COLLECT 5 | from PyInstaller.building.build_main import Analysis 6 | from PyInstaller.building.datastruct import Tree 7 | from PyInstaller.building.osx import BUNDLE 8 | 9 | gooey_root = os.path.dirname(gooey.__file__) 10 | gooey_languages = Tree(os.path.join(gooey_root, 'languages'), prefix='gooey/languages') 11 | gooey_images = Tree(os.path.join(gooey_root, 'images'), prefix='gooey/images') 12 | 13 | block_cipher = None 14 | 15 | # noinspection PyUnresolvedReferences 16 | a = Analysis(['nba_sql.py'], 17 | pathex=[os.path.abspath(SPECPATH)], 18 | binaries=[], 19 | datas=[], 20 | hiddenimports=["Team"], 21 | hookspath=[], 22 | runtime_hooks=[], 23 | excludes=[], 24 | win_no_prefer_redirects=False, 25 | win_private_assemblies=False, 26 | cipher=block_cipher) 27 | pyz = PYZ(a.pure, a.zipped_data, 28 | cipher=block_cipher) 29 | exe = EXE(pyz, 30 | a.scripts, 31 | exclude_binaries=True, 32 | name='nba-sql', 33 | debug=False, 34 | strip=False, 35 | upx=True, 36 | console=True) 37 | coll = COLLECT(exe, 38 | a.binaries, 39 | a.zipfiles, 40 | a.datas, 41 | gooey_languages, 42 | gooey_images, 43 | strip=False, 44 | upx=True, 45 | name='nba-sql') 46 | 47 | app = BUNDLE(coll, 48 | name='nba-sql.exe', 49 | icon=None, 50 | bundle_identifier=None, 51 | info_plist={ 52 | 'NSHighResolutionCapable': 'True' 53 | } 54 | ) -------------------------------------------------------------------------------- /docker/.env: -------------------------------------------------------------------------------- 1 | # 2 | # Environment variables for the nba_sql application. 3 | # 4 | COMPOSE_PROJECT_NAME=nba_sql 5 | 6 | POSTGRES_DB=nba 7 | POSTGRES_USER=nba_sql 8 | POSTGRES_PASSWORD=nba_sql 9 | -------------------------------------------------------------------------------- /docker/Dockerfile.build: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM python:3.9-bullseye 4 | 5 | WORKDIR /app 6 | 7 | RUN apt update 8 | RUN apt install -y libpq-dev build-essential 9 | 10 | COPY requirements_no_gui.txt requirements_no_gui.txt 11 | 12 | RUN pip3 install pip --upgrade 13 | RUN pip3 install -r requirements_no_gui.txt 14 | 15 | COPY stats/ /app/stats 16 | 17 | COPY scripts/release/build_no_gui.sh . 18 | 19 | RUN chmod +x build_no_gui.sh 20 | 21 | CMD ["./build_no_gui.sh"] 22 | -------------------------------------------------------------------------------- /docker/docker-compose-mariadb.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Development environment for the nba_sql application. 3 | # Thank you, superset, for the great template :) 4 | # 5 | version: "3.7" 6 | 7 | services: 8 | db: 9 | image: mariadb:latest 10 | volumes: 11 | - db_data:/var/lib/mysql 12 | restart: always 13 | ports: 14 | - "127.0.0.1:3306:3306" 15 | environment: 16 | MYSQL_ROOT_PASSWORD: admin 17 | MYSQL_DATABASE: nba 18 | MYSQL_USER: nba_sql 19 | MYSQL_PASSWORD: nba_sql 20 | 21 | volumes: 22 | db_data: 23 | external: false 24 | -------------------------------------------------------------------------------- /docker/docker-compose-postgres.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Development environment for the nba_sql application. 3 | # Thank you, superset, for the great template :) 4 | # 5 | version: "3.7" 6 | 7 | services: 8 | db: 9 | env_file: .env 10 | image: postgres:17 11 | container_name: nba_sql_db 12 | restart: unless-stopped 13 | ports: 14 | - "127.0.0.1:5432:5432" 15 | volumes: 16 | - db_home:/var/lib/postgresql/data 17 | 18 | volumes: 19 | db_home: 20 | external: false 21 | -------------------------------------------------------------------------------- /image/NBA-ER.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpope9/nba-sql/5ea009fad7a74608dd4eada71303ede5e238aa49/image/NBA-ER.jpg -------------------------------------------------------------------------------- /image/james-harden-shot-analysis-2020-21.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpope9/nba-sql/5ea009fad7a74608dd4eada71303ede5e238aa49/image/james-harden-shot-analysis-2020-21.webp -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.2 2 | certifi==2024.7.4 3 | cffi==1.14.5 4 | chardet==4.0.0 5 | cryptography 6 | idna==3.7 7 | peewee==3.17.0 8 | pycparser==2.20 9 | PyMySQL==1.1.1 10 | python-dotenv==1.0.0 11 | six==1.16.0 12 | urllib3<2 13 | psycopg2==2.9.10 14 | psycopg2-binary==2.9.10 15 | gooey==1.0.8.1 16 | pyinstaller 17 | -------------------------------------------------------------------------------- /requirements_no_gui.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.2 2 | certifi==2024.7.4 3 | cffi==1.14.5 4 | chardet==4.0.0 5 | cryptography 6 | idna==3.7 7 | peewee==3.17.0 8 | pycparser==2.20 9 | PyMySQL==1.1.1 10 | python-dotenv==1.0.0 11 | six==1.16.0 12 | urllib3<2 13 | psycopg2==2.9.10 14 | psycopg2-binary==2.9.10 15 | pyinstaller 16 | -------------------------------------------------------------------------------- /scripts/create_maria.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --default-mode --database="mysql" --create-schema 4 | -------------------------------------------------------------------------------- /scripts/create_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --default-mode --database="postgres" --skip-tables play_by_play pgtt 4 | -------------------------------------------------------------------------------- /scripts/create_sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --default-mode --skip-tables play_by_play pgtt 4 | -------------------------------------------------------------------------------- /scripts/drop.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE player_game_log; 2 | DROP TABLE player_season; 3 | DROP TABLE play_by_play; 4 | DROP TABLE player_general_traditional_total; 5 | DROP TABLE shot_chart_detail; 6 | 7 | -- Base Tables Come Last 8 | DROP TABLE event_message_type; 9 | DROP TABLE game; 10 | DROP TABLE player; 11 | DROP TABLE team; 12 | -------------------------------------------------------------------------------- /scripts/drop_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | COMMAND=docker 4 | if command -v podman 2>&1 >/dev/null 5 | then 6 | COMMAND=podman 7 | fi 8 | echo "Running command with $COMMAND" 9 | $COMMAND exec -i nba_sql_db psql -U nba_sql nba < scripts/drop.sql 10 | -------------------------------------------------------------------------------- /scripts/drop_sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm nba_sql.db* 4 | -------------------------------------------------------------------------------- /scripts/fill_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --database="postgres" 4 | -------------------------------------------------------------------------------- /scripts/refresh_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --current-season-mode --database="postgres" 4 | -------------------------------------------------------------------------------- /scripts/refresh_sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --current-season-mode 4 | -------------------------------------------------------------------------------- /scripts/release/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | pyinstaller --windowed -n nba_sql --paths=stats stats/gui.py -F 3 | -------------------------------------------------------------------------------- /scripts/release/build_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## Script to build an executable in a linux container. 3 | rm -rf build/ dist/ 4 | mkdir dist 5 | 6 | DOCKER_BUILDKIT=1 docker build --tag nba-sql-builder -f docker/Dockerfile.build . 7 | 8 | DIST_PATH="$PWD/dist" 9 | 10 | docker run -v $DIST_PATH:/app/dist/ nba-sql-builder 11 | -------------------------------------------------------------------------------- /scripts/release/build_exe.txt: -------------------------------------------------------------------------------- 1 | py -m PyInstaller --windowed -n nba_sql --paths=stats stats\gui.py -F 2 | -------------------------------------------------------------------------------- /scripts/release/build_no_gui.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | pyinstaller --paths=stats stats/nba_sql.py -F 4 | -------------------------------------------------------------------------------- /stats/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # jupyter notebooks 107 | .debugging_file.ipynb 108 | debugging_file.ipynb 109 | -------------------------------------------------------------------------------- /stats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpope9/nba-sql/5ea009fad7a74608dd4eada71303ede5e238aa49/stats/__init__.py -------------------------------------------------------------------------------- /stats/args.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Creates an argument parser for the commandline version. 20 | """ 21 | 22 | 23 | def create_parser(parser): 24 | """ 25 | Creates and returns a Gooey parser. 26 | """ 27 | 28 | parser.add_argument( 29 | '--database', 30 | dest='database_type', 31 | default='sqlite', 32 | choices=['mysql', 'postgres', 'sqlite'], 33 | help=''' 34 | The database flag specifies which database protocol to use. 35 | Defaults to "sqlite", but also accepts "postgres" and "mysql". 36 | ''') 37 | 38 | parser.add_argument( 39 | '--database_name', 40 | help="Database Name (Not Needed For SQLite)", 41 | default='nba') 42 | 43 | parser.add_argument( 44 | '--database_host', 45 | help="Database Hostname (Not Needed For SQLite)", 46 | default=None) 47 | 48 | parser.add_argument( 49 | '--username', 50 | help="Database Username (Not Needed For SQLite)", 51 | default=None) 52 | 53 | parser.add_argument( 54 | '--create-schema', 55 | dest='create_schema', 56 | action="store_true", 57 | default=True, 58 | help=''' 59 | Flag to initialize the database schema before loading data. 60 | If the schema already exists then nothing will happen. 61 | ''') 62 | 63 | parser.add_argument( 64 | '--time-between-requests', 65 | dest='request_gap', 66 | default='.7', 67 | help=''' 68 | This flag exists to prevent rate limiting, and injects the 69 | desired amount of time inbetween requesting resources. 70 | ''') 71 | 72 | # To fix issue https://github.com/mpope9/nba-sql/issues/56 73 | parser.add_argument( 74 | '--batch_size', 75 | default='10000', 76 | type=int, 77 | help=''' 78 | Inserts BATCH_SIZE chunks of rows to the database. 79 | This value is ignored when selecting database 'sqlite'. 80 | ''') 81 | 82 | parser.add_argument( 83 | '--sqlite-path', 84 | dest='sqlite_path', 85 | default='nba_sql.db', 86 | help='Setting to define sqlite path.') 87 | 88 | parser.add_argument( 89 | '--quiet', 90 | dest='quiet', 91 | action='store_true', 92 | help=''' 93 | Setting to define stdout logging level. If set, only 94 | "ok" will be printed if ran successfully. 95 | This currently only applies to refreshing a db, and not loading one. 96 | ''') 97 | 98 | return parser 99 | -------------------------------------------------------------------------------- /stats/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Constants used in the application. 20 | """ 21 | 22 | """ 23 | Headers. 24 | """ 25 | headers = { 26 | 'Connection': 'keep-alive', 27 | 'Accept': 'application/json, text/plain, */*', 28 | 'x-nba-stats-token': 'true', 29 | 'User-Agent': ( 30 | # 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) ' 31 | # 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130' 32 | # 'Safari/537.36' 33 | '''Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)\ 34 | AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36''' 35 | ), 36 | 'x-nba-stats-origin': 'stats', 37 | 'Sec-Fetch-Site': 'same-origin', 38 | 'Sec-Fetch-Mode': 'cors', 39 | 'Referer': 'https://stats.nba.com/', 40 | 'Accept-Encoding': 'gzip, deflate, br', 41 | 'Accept-Language': 'en-US,en;q=0.9', 42 | } 43 | 44 | """ 45 | Team IDs. (Thank you nba-api). 46 | """ 47 | team_ids = [ 48 | 1610612737, # 'ATL' 49 | 1610612738, # 'BOS' 50 | 1610612739, # 'CLE' 51 | 1610612740, # 'NOP' 52 | 1610612741, # 'CHI' 53 | 1610612742, # 'DAL' 54 | 1610612743, # 'DEN' 55 | 1610612744, # 'GSW' 56 | 1610612745, # 'HOU' 57 | 1610612746, # 'LAC' 58 | 1610612747, # 'LAL' 59 | 1610612748, # 'MIA' 60 | 1610612749, # 'MIL' 61 | 1610612750, # 'MIN' 62 | 1610612751, # 'BKN' 63 | 1610612752, # 'NYK' 64 | 1610612753, # 'ORL' 65 | 1610612754, # 'IND' 66 | 1610612755, # 'PHI' 67 | 1610612756, # 'PHX' 68 | 1610612757, # 'POR' 69 | 1610612758, # 'SAC' 70 | 1610612759, # 'SAS' 71 | 1610612760, # 'OKC' 72 | 1610612761, # 'TOR' 73 | 1610612762, # 'UTA' 74 | 1610612763, # 'MEM' 75 | 1610612764, # 'WAS' 76 | 1610612765, # 'DET' 77 | 1610612766, # 'CHA' 78 | ] 79 | 80 | """ 81 | Mapping from team abbrev to id. 82 | """ 83 | team_abbrev_mapping = { 84 | 'ATL': 1610612737, 85 | 'BOS': 1610612738, 86 | 'CLE': 1610612739, 87 | 'NOP': 1610612740, 88 | 'NOK': 1610612740, # Old name. 89 | 'NOH': 1610612740, # Old name. 90 | 'CHI': 1610612741, 91 | 'DAL': 1610612742, 92 | 'DEN': 1610612743, 93 | 'GSW': 1610612744, 94 | 'HOU': 1610612745, 95 | 'LAC': 1610612746, 96 | 'LAL': 1610612747, 97 | 'MIA': 1610612748, 98 | 'MIL': 1610612749, 99 | 'MIN': 1610612750, 100 | 'BKN': 1610612751, 101 | 'NJN': 1610612751, # Old name. 102 | 'NYK': 1610612752, 103 | 'ORL': 1610612753, 104 | 'IND': 1610612754, 105 | 'PHI': 1610612755, 106 | 'PHX': 1610612756, 107 | 'POR': 1610612757, 108 | 'SAC': 1610612758, 109 | 'SAS': 1610612759, 110 | 'OKC': 1610612760, 111 | 'SEA': 1610612760, 112 | 'TOR': 1610612761, 113 | 'UTA': 1610612762, 114 | 'VAN': 1610612763, # Old name. 115 | 'MEM': 1610612763, 116 | 'WAS': 1610612764, 117 | 'DET': 1610612765, 118 | 'CHA': 1610612766, 119 | 'CHH': 1610612766, # Old name. 120 | } 121 | 122 | 123 | """ 124 | Play-by-play data has an EventMsgType field. This is an enum. There 125 | is also the EventMsgActionField, which is a complex enum of 126 | (EventMsgType, SubType). 127 | We're going to make a lookup table of enum to value, then a lookup 128 | table for the (EventMsgType, EventMsgActionType) pair. 129 | """ 130 | event_message_types = [ 131 | {'id': 1, 'string': 'FIELD_GOAL_MADE'}, 132 | {'id': 2, 'string': 'FIELD_GOAL_MISSED'}, 133 | {'id': 3, 'string': 'FREE_THROW'}, 134 | {'id': 4, 'string': 'REBOUND'}, 135 | {'id': 5, 'string': 'TURNOVER'}, 136 | {'id': 6, 'string': 'FOUL'}, 137 | {'id': 7, 'string': 'VIOLATION'}, 138 | {'id': 8, 'string': 'SUBSTITUTION'}, 139 | {'id': 9, 'string': 'TIMEOUT'}, 140 | {'id': 10, 'string': 'JUMP_BALL'}, 141 | {'id': 11, 'string': 'EJECTION'}, 142 | {'id': 12, 'string': 'PERIOD_BEGIN'}, 143 | {'id': 13, 'string': 'PERIOD_END'}, 144 | {'id': 18, 'string': 'UNKNOWN'} 145 | ] 146 | -------------------------------------------------------------------------------- /stats/db_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Database utilities (future middleware layer if we decide to use DuckDB by default.) 20 | """ 21 | 22 | from utils import chunk_list 23 | 24 | 25 | def insert_many(settings, table, rows): 26 | """ 27 | Entry function on insert_many. 28 | """ 29 | 30 | chunked_rows = chunk_list(rows, settings.batch_size) 31 | if settings.db_type == 'sqlite': 32 | __insert_many_sqlite(settings, table, rows) 33 | else: 34 | with settings.db.atomic(): 35 | for row in chunked_rows: 36 | table.insert_many(row).execute() 37 | 38 | 39 | def __insert_many_sqlite(settings, table, rows): 40 | """ 41 | SQLite has a limit on number of rows. Chunk the rows and batch insert. 42 | """ 43 | 44 | chunked_rows = chunk_list(rows, 500) 45 | with settings.db.atomic(): 46 | for row in chunked_rows: 47 | table.insert_many(row).execute() 48 | 49 | 50 | def insert_many_on_conflict_ignore(settings, table, rows): 51 | """ 52 | Entry function on insert_many, ignoring conflicts on key issues. 53 | """ 54 | 55 | if settings.db_type == 'sqlite': 56 | __insert_many_on_conflict_ignore_sqlite(settings, table, rows) 57 | else: 58 | with settings.db.atomic(): 59 | table.insert_many(rows).on_conflict_ignore().execute() 60 | 61 | 62 | def __insert_many_on_conflict_ignore_sqlite(settings, table, rows): 63 | """ 64 | SQLite has a limit on number of rows. Chunk the rows and batch insert. 65 | """ 66 | 67 | chunked_rows = chunk_list(rows, 500) 68 | with settings.db.atomic(): 69 | for row in chunked_rows: 70 | table.insert_many(row).on_conflict_ignore().execute() 71 | -------------------------------------------------------------------------------- /stats/event_message_type.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Initializer for the EventMessageType object. 20 | """ 21 | 22 | from models import EventMessageType 23 | from constants import event_message_types 24 | 25 | 26 | class EventMessageTypeBuilder: 27 | 28 | def __init__(self, settings): 29 | self.settings = settings 30 | self.settings.db.bind([EventMessageType]) 31 | 32 | def create_ddl(self): 33 | """ 34 | Initialize the table schema. 35 | """ 36 | self.settings.db.create_tables([EventMessageType], safe=True) 37 | 38 | def initialize(self): 39 | """ 40 | Build table from const mappings. 41 | """ 42 | 43 | EventMessageType.insert_many(event_message_types).execute() 44 | -------------------------------------------------------------------------------- /stats/game.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Game builder. 20 | """ 21 | 22 | from models import Game 23 | from constants import team_abbrev_mapping 24 | from collections import namedtuple 25 | from db_utils import insert_many, insert_many_on_conflict_ignore 26 | 27 | 28 | GameEntry = namedtuple("GameEntry", "season_id, game_id, game_date, matchup_in, winner, loser, playoff_game") 29 | 30 | 31 | class GameBuilder: 32 | 33 | def __init__(self, settings): 34 | self.settings = settings 35 | self.settings.db.bind([Game]) 36 | 37 | def create_ddl(self): 38 | """ 39 | Creates the game table from the model. 40 | """ 41 | self.settings.db.create_tables([Game], safe=True) 42 | 43 | def game_id_predicate(self): 44 | """ 45 | Returns a selection of the game id. 46 | """ 47 | return Game.select(Game.game_id) 48 | 49 | def fetch_season_game_id_set(self, season_id): 50 | """ 51 | Returns an ID set for games in the specified season, formatted to match what the 52 | NBA's API expects. 53 | """ 54 | return set([str(game.game_id).zfill(10) for game in Game.select(Game.game_id).where(Game.season_id == season_id)]) 55 | 56 | def populate_table(self, game_set, ignore_dups=False): 57 | """ 58 | Takes a set of tuples and builds the game table. 59 | @params: 60 | game_set - Required : Set of GameEntry namedtuple entries (Set) 61 | ignore_dups - Optional : Will ignore duplicate entries if present. This is mainly used to refresh existing seasons. 62 | """ 63 | rows = [] 64 | 65 | # Sometimes the API returns two entries for a game with the teams ids switched. 66 | # it is safe to dedup them here before storing them in the game table, which has 67 | # a PK on the game_id. 68 | game_id_set = set() 69 | game_set_deduplicated = set() 70 | for entry in game_set: 71 | if entry.game_id not in game_id_set: 72 | game_id_set.add(entry.game_id) 73 | game_set_deduplicated.add(entry) 74 | else: 75 | print(f"Found duplicate game entry: {entry}") 76 | 77 | for entry in game_set_deduplicated: 78 | # A bit of a hack. We shouldn't rely on data in requests 79 | # because it could change and invalidate this logic. 80 | away_team, home_team = entry.matchup_in.split(" @ ") 81 | 82 | try: 83 | new_row = { 84 | 'game_id': entry.game_id, 85 | 'team_id_home': team_abbrev_mapping[home_team], 86 | 'team_id_away': team_abbrev_mapping[away_team], 87 | 'season_id': entry.season_id, 88 | 'playoff_game': entry.playoff_game, 89 | 'date': entry.game_date 90 | } 91 | except KeyError as e: 92 | # TODO Support these. 93 | print(f"Unsupported team abbreviation: {e.args[0]}") 94 | continue 95 | 96 | teams = [new_row['team_id_away'], new_row['team_id_home']] 97 | if entry.winner: 98 | new_row['team_id_winner'] = entry.winner 99 | new_row['team_id_loser'] = teams[teams.index(entry.winner) - 1] 100 | else: 101 | new_row['team_id_loser'] = entry.loser 102 | new_row['team_id_winner'] = teams[teams.index(entry.loser) - 1] 103 | 104 | rows.append(new_row) 105 | 106 | if ignore_dups: 107 | insert_many_on_conflict_ignore(self.settings, Game, rows) 108 | else: 109 | insert_many(self.settings, Game, rows) 110 | -------------------------------------------------------------------------------- /stats/general_requester.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from utils import get_rowset_mapping, column_names_from_table 4 | from constants import headers 5 | from db_utils import insert_many 6 | 7 | 8 | class GenericRequester: 9 | 10 | def __init__(self, settings, url, table): 11 | """ 12 | Constructor. 13 | """ 14 | self.settings = settings 15 | self.url = url 16 | self.table = table 17 | self.settings.db.bind([self.table]) 18 | # Okay without this variables bleed inbetween requesters that inherit 19 | # this base class. Why does that happen? 20 | self.rows = [] 21 | 22 | def create_ddl(self): 23 | """ 24 | Initialize the table schema. 25 | """ 26 | self.settings.db.create_tables([self.table], safe=True) 27 | 28 | def generate_rows(self, params): 29 | """ 30 | Build GET REST request and fill the table. 31 | """ 32 | 33 | # json response 34 | response = requests.get(url=self.url, headers=headers, params=params).json() 35 | 36 | result_sets = response['resultSets'][0] 37 | rowset = result_sets['rowSet'] 38 | 39 | column_names = column_names_from_table(self.settings.db, self.table._meta.table_name) 40 | 41 | column_mapping = get_rowset_mapping(result_sets, column_names) 42 | 43 | for row in rowset: 44 | new_row = {column_name: row[row_index] for column_name, row_index in column_mapping.items()} 45 | self.rows.append(new_row) 46 | 47 | def populate(self): 48 | """ 49 | Bulk insert. Remove row cache from object once finished. 50 | """ 51 | insert_many(self.settings, self.table, self.rows) 52 | self.rows = [] 53 | -------------------------------------------------------------------------------- /stats/gui.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | This is a wrapper, to allow building the cmdline executable without 20 | having to include the full GUI libs. 21 | """ 22 | 23 | from nba_sql import main 24 | from args import create_parser 25 | 26 | from gooey import Gooey 27 | from gooey import GooeyParser 28 | 29 | from utils import generate_valid_seasons 30 | 31 | import codecs 32 | import sys 33 | 34 | 35 | # This fixes an issue with Gooey and PyInstaller. 36 | if sys.stdout.encoding != 'UTF-8': 37 | sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict') 38 | if sys.stderr.encoding != 'UTF-8': 39 | sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict') 40 | 41 | 42 | # This 'fixes' an issue with printing in the Gooey console, kinda sorta not really. 43 | class Unbuffered(object): 44 | 45 | def __init__(self, stream): 46 | self.stream = stream 47 | 48 | def write(self, data): 49 | self.stream.write(data) 50 | self.stream.flush() 51 | 52 | def writelines(self, datas): 53 | self.stream.writelines(datas) 54 | self.stream.flush() 55 | 56 | def __getattr__(self, attr): 57 | return getattr(self.stream, attr) 58 | 59 | 60 | sys.stdout = Unbuffered(sys.stdout) 61 | 62 | 63 | # Bad practice? Yes. Any other alternative? Not at this point. 64 | # Only enable Gooey if there are no arguments passed to the script. 65 | if len(sys.argv) >= 2: 66 | if '--ignore-gooey' not in sys.argv: 67 | sys.argv.append('--ignore-gooey') 68 | 69 | 70 | @Gooey( 71 | program_name='nba-sql', 72 | program_description='An application to build a database of NBA data.', 73 | header_show_title=True) 74 | def gui_main(): 75 | parser = GooeyParser(description="nba-sql") 76 | create_parser(parser) 77 | 78 | # Add the 'mode args' that the regular python arg parse doesn't support. 79 | mode_parser = parser.add_mutually_exclusive_group( 80 | required=True, 81 | gooey_options={ 82 | 'initial_selection': 0 83 | }) 84 | mode_parser.add_argument( 85 | '--default_mode', 86 | help=''' 87 | Mode to create the database and load historic data. 88 | Use this mode when creating a new database or when 89 | trying to load a specific season or a range of seasons. 90 | ''', 91 | action='store_true') 92 | mode_parser.add_argument( 93 | '--current_season_mode', 94 | help=''' 95 | Mode to refresh the current season. Use this mode on an 96 | existing database to update it with the latest data. 97 | ''', 98 | action='store_true') 99 | 100 | parser.add_argument( 101 | '--password', 102 | help="Database Password (Not Needed For SQLite)", 103 | widget='PasswordField', 104 | default=None) 105 | 106 | valid_seasons = generate_valid_seasons() 107 | last_loadable_season = valid_seasons[-1] 108 | 109 | parser.add_argument( 110 | '--seasons', 111 | dest='seasons', 112 | default=[last_loadable_season], 113 | choices=valid_seasons, 114 | widget='Listbox', 115 | nargs="*", 116 | help=''' 117 | The seasons flag loads the database with the specified season. 118 | The format of the season should be in the form "YYYY-YY". 119 | The default behavior is loading the current season. 120 | ''') 121 | 122 | parser.add_argument( 123 | '--skip-tables', 124 | action='store', 125 | nargs="*", 126 | default='', 127 | choices=[ 128 | 'player_season', 129 | 'player_game_log', 130 | 'play_by_play', 131 | 'pgtt', 132 | 'shot_chart_detail', 133 | 'game', 134 | 'event_message_type', 135 | 'team', 136 | 'player', 137 | '' 138 | ], 139 | widget='Listbox', 140 | help='Use this option to skip loading certain tables.') 141 | 142 | args = parser.parse_args() 143 | 144 | main(args, True) 145 | 146 | 147 | if __name__ == "__main__": 148 | gui_main() 149 | -------------------------------------------------------------------------------- /stats/models/EventMessageType.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | EventMessageType model definition. 20 | """ 21 | 22 | from peewee import IntegerField, CharField, Model 23 | 24 | 25 | class EventMessageType(Model): 26 | 27 | id = IntegerField(primary_key=True) 28 | string = CharField() 29 | 30 | class Meta: 31 | db_table = 'event_message_type' 32 | -------------------------------------------------------------------------------- /stats/models/Game.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Game model definition. 20 | """ 21 | 22 | from peewee import ( 23 | ForeignKeyField, 24 | IntegerField, 25 | BooleanField, 26 | DateField, 27 | Model 28 | ) 29 | from . import Team 30 | 31 | 32 | class Game(Model): 33 | 34 | # Primary Key 35 | game_id = IntegerField(primary_key=True) 36 | 37 | # Foreign Keys 38 | team_id_home = ForeignKeyField(Team, index=True, null=False, column_name='team_id_home') 39 | team_id_away = ForeignKeyField(Team, index=True, null=False, column_name='team_id_away') 40 | team_id_winner = ForeignKeyField(Team, index=True, null=False, column_name='team_id_winner') 41 | team_id_loser = ForeignKeyField(Team, index=True, null=False, column_name='team_id_loser') 42 | 43 | # Indexes 44 | season_id = IntegerField(index=True, null=False) 45 | 46 | playoff_game = BooleanField(null=True) 47 | date = DateField(null=False) 48 | 49 | class Meta: 50 | db_table = 'game' 51 | -------------------------------------------------------------------------------- /stats/models/PlayByPlay.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayByPlay model definition. 20 | """ 21 | 22 | from peewee import ForeignKeyField, IntegerField, CharField, Model 23 | from . import EventMessageType 24 | from . import Player 25 | from . import Team 26 | from . import Game 27 | 28 | 29 | class PlayByPlay(Model): 30 | 31 | # Indexes 32 | game_id = ForeignKeyField(Game, index=True) 33 | 34 | event_num = IntegerField() 35 | event_msg_type = ForeignKeyField(EventMessageType, index=True) 36 | event_msg_action_type = IntegerField(index=True) 37 | period = IntegerField() 38 | 39 | # Why not time field? WELL, some times like "24:11 PM" are returned. 40 | wc_time = CharField() 41 | 42 | home_description = CharField(null=True) 43 | neutral_description = CharField(null=True) 44 | visitor_description = CharField(null=True) 45 | score = CharField(null=True) 46 | score_margin = CharField(null=True) 47 | 48 | player1_id = ForeignKeyField(Player, index=True, null=True) 49 | player1_team_id = ForeignKeyField(Team, index=True, null=True) 50 | 51 | player2_id = ForeignKeyField(Player, index=True, null=True) 52 | player2_team_id = ForeignKeyField(Team, index=True, null=True) 53 | 54 | player3_id = ForeignKeyField(Player, index=True, null=True) 55 | player3_team_id = ForeignKeyField(Team, index=True, null=True) 56 | 57 | class Meta: 58 | db_table = 'play_by_play' 59 | -------------------------------------------------------------------------------- /stats/models/PlayByPlayV3.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayByPlayV3 model definition. 20 | """ 21 | 22 | from peewee import ForeignKeyField, IntegerField, CharField, Model 23 | from . import Player 24 | from . import Game 25 | from . import Team 26 | 27 | class PlayByPlayV3(Model): 28 | 29 | # Indexes 30 | game_id = ForeignKeyField(Game, index=True, null=False) 31 | player_id = ForeignKeyField(Player, index=True, null=True) 32 | team_id = ForeignKeyField(Team, index=True, null=True) 33 | 34 | action_number = CharField() 35 | clock = CharField() 36 | period = CharField() 37 | x_legacy = CharField() 38 | y_legacy = CharField() 39 | shot_distance = CharField() 40 | shot_result = CharField() 41 | is_field_goal = CharField() 42 | score_home = CharField() 43 | score_away = CharField() 44 | points_total = CharField() 45 | location = CharField() 46 | description = CharField() 47 | action_type = CharField() 48 | sub_type = CharField() 49 | 50 | class Meta: 51 | db_table = 'play_by_playv3' 52 | 53 | -------------------------------------------------------------------------------- /stats/models/Player.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Player model definition. 20 | """ 21 | 22 | from peewee import ( 23 | IntegerField, 24 | CharField, 25 | Model 26 | ) 27 | 28 | 29 | class Player(Model): 30 | 31 | player_id = IntegerField(primary_key=True) 32 | 33 | player_name = CharField(null=True) 34 | college = CharField(null=True) 35 | country = CharField(null=True) 36 | draft_year = CharField(null=True) 37 | draft_round = CharField(null=True) 38 | draft_number = CharField(null=True) 39 | 40 | class Meta: 41 | db_table = 'player' 42 | -------------------------------------------------------------------------------- /stats/models/PlayerGameLog.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerGameLog model definition. 20 | """ 21 | 22 | from peewee import ( 23 | ForeignKeyField, 24 | IntegerField, 25 | FloatField, 26 | Model, 27 | CompositeKey, 28 | FixedCharField 29 | ) 30 | from . import Player 31 | from . import Team 32 | from . import Game 33 | 34 | 35 | class PlayerGameLog(Model): 36 | 37 | # Composite PK Fields 38 | player_id = ForeignKeyField(Player, index=True) 39 | game_id = ForeignKeyField(Game, null=True) 40 | 41 | # Foreign Keys 42 | team_id = ForeignKeyField(Team, index=True) 43 | 44 | # Indexes 45 | season_id = IntegerField(index=True) 46 | 47 | wl = FixedCharField(null=True, max_length=1) 48 | min = FloatField(null=True) 49 | fgm = FloatField(null=True) 50 | fga = FloatField(null=True) 51 | fg_pct = FloatField(null=True) 52 | fg3m = FloatField(null=True) 53 | fg3a = FloatField(null=True) 54 | fg3_pct = FloatField(null=True) 55 | ftm = FloatField(null=True) 56 | fta = FloatField(null=True) 57 | ft_pct = FloatField(null=True) 58 | oreb = FloatField(null=True) 59 | dreb = FloatField(null=True) 60 | reb = FloatField(null=True) 61 | ast = FloatField(null=True) 62 | tov = FloatField(null=True) 63 | stl = FloatField(null=True) 64 | blk = FloatField(null=True) 65 | blka = FloatField(null=True) 66 | pf = FloatField(null=True) 67 | pfd = FloatField(null=True) 68 | pts = FloatField(null=True) 69 | plus_minus = FloatField(null=True) 70 | nba_fantasy_pts = FloatField(null=True) 71 | dd2 = FloatField(null=True) 72 | td3 = FloatField(null=True) 73 | 74 | class Meta: 75 | db_table = 'player_game_log' 76 | primary_key = CompositeKey( 77 | 'player_id', 78 | 'game_id' 79 | ) 80 | indexes = ( 81 | (('player_id', 'season_id'), False), 82 | (('player_id', 'season_id', 'team_id'), False), 83 | ) 84 | -------------------------------------------------------------------------------- /stats/models/PlayerGameLogTemp.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerGameLogTemp model definition. 20 | """ 21 | 22 | from peewee import ( 23 | IntegerField, 24 | FloatField, 25 | Model, 26 | CompositeKey, 27 | FixedCharField 28 | ) 29 | 30 | 31 | class PlayerGameLogTemp(Model): 32 | 33 | # Composite PK Fields 34 | player_id = IntegerField(index=True) 35 | game_id = IntegerField(null=True) 36 | 37 | team_id = IntegerField(index=True) 38 | 39 | # Indexes 40 | season_id = IntegerField(index=True) 41 | 42 | wl = FixedCharField(null=True, max_length=1) 43 | min = FloatField(null=True) 44 | fgm = FloatField(null=True) 45 | fga = FloatField(null=True) 46 | fg_pct = FloatField(null=True) 47 | fg3m = FloatField(null=True) 48 | fg3a = FloatField(null=True) 49 | fg3_pct = FloatField(null=True) 50 | ftm = FloatField(null=True) 51 | fta = FloatField(null=True) 52 | ft_pct = FloatField(null=True) 53 | oreb = FloatField(null=True) 54 | dreb = FloatField(null=True) 55 | reb = FloatField(null=True) 56 | ast = FloatField(null=True) 57 | tov = FloatField(null=True) 58 | stl = FloatField(null=True) 59 | blk = FloatField(null=True) 60 | blka = FloatField(null=True) 61 | pf = FloatField(null=True) 62 | pfd = FloatField(null=True) 63 | pts = FloatField(null=True) 64 | plus_minus = FloatField(null=True) 65 | nba_fantasy_pts = FloatField(null=True) 66 | dd2 = FloatField(null=True) 67 | td3 = FloatField(null=True) 68 | 69 | class Meta: 70 | db_table = 'player_game_log_temp' 71 | primary_key = CompositeKey( 72 | 'player_id', 73 | 'game_id' 74 | ) 75 | temporary = True 76 | -------------------------------------------------------------------------------- /stats/models/PlayerGeneralTraditionalTotal.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerGeneralTraditionalTotal model definition. 20 | """ 21 | 22 | from peewee import ( 23 | ForeignKeyField, 24 | IntegerField, 25 | CharField, 26 | FloatField, 27 | Model 28 | ) 29 | from . import Player 30 | from . import Team 31 | 32 | 33 | class PlayerGeneralTraditionalTotal(Model): 34 | 35 | # Defaulting to auto generated id column. We do this because 36 | # team_id is nullable for players in a season and that breaks the 37 | # team table's Primary Key constraint. 38 | 39 | # Composite Unique Index 40 | player_id = ForeignKeyField(Player, null=False, index=True) 41 | season_id = IntegerField(null=False, index=True) 42 | team_id = ForeignKeyField(Team, index=True, null=True) 43 | 44 | age = IntegerField(null=True) 45 | gp = IntegerField(null=True) 46 | w = IntegerField(null=True) 47 | l = IntegerField(null=True) # NOQA 48 | w_pct = FloatField(null=True) 49 | min = FloatField(null=True) 50 | fgm = FloatField(null=True) 51 | fga = FloatField(null=True) 52 | fg_pct = FloatField(null=True) 53 | fg3m = FloatField(null=True) 54 | fg3a = FloatField(null=True) 55 | fg3_pct = FloatField(null=True) 56 | ftm = FloatField(null=True) 57 | fta = FloatField(null=True) 58 | ft_pct = FloatField(null=True) 59 | oreb = FloatField(null=True) 60 | dreb = FloatField(null=True) 61 | reb = FloatField(null=True) 62 | ast = FloatField(null=True) 63 | tov = FloatField(null=True) 64 | stl = FloatField(null=True) 65 | blk = FloatField(null=True) 66 | blka = FloatField(null=True) 67 | pf = FloatField(null=True) 68 | pfd = FloatField(null=True) 69 | pts = FloatField(null=True) 70 | plus_minus = FloatField(null=True) 71 | nba_fantasy_pts = FloatField(null=True) 72 | dd2 = FloatField(null=True) 73 | td3 = FloatField(null=True) 74 | gp_rank = IntegerField(null=True) 75 | w_rank = IntegerField(null=True) 76 | l_rank = IntegerField(null=True) 77 | w_pct_rank = IntegerField(null=True) 78 | min_rank = IntegerField(null=True) 79 | fgm_rank = IntegerField(null=True) 80 | fga_rank = IntegerField(null=True) 81 | fg_pct_rank = IntegerField(null=True) 82 | fg3m_rank = IntegerField(null=True) 83 | fg3a_rank = IntegerField(null=True) 84 | fg3_pct_rank = IntegerField(null=True) 85 | ftm_rank = IntegerField(null=True) 86 | fta_rank = IntegerField(null=True) 87 | ft_pct_rank = IntegerField(null=True) 88 | oreb_rank = IntegerField(null=True) 89 | dreb_rank = IntegerField(null=True) 90 | reb_rank = IntegerField(null=True) 91 | ast_rank = IntegerField(null=True) 92 | tov_rank = IntegerField(null=True) 93 | stl_rank = IntegerField(null=True) 94 | blk_rank = IntegerField(null=True) 95 | blka_rank = IntegerField(null=True) 96 | pf_rank = IntegerField(null=True) 97 | pfd_rank = IntegerField(null=True) 98 | pts_rank = IntegerField(null=True) 99 | plus_minus_rank = IntegerField(null=True) 100 | nba_fantasy_pts_rank = IntegerField(null=True) 101 | dd2_rank = IntegerField(null=True) 102 | td3_rank = IntegerField(null=True) 103 | 104 | # This column is obscelete but it is kept for 105 | # backwards compatiblity. It is no longer returned 106 | # by the NBA API. 107 | cfid = IntegerField(null=True) 108 | 109 | cfparams = CharField(null=True) 110 | 111 | class Meta: 112 | db_table = 'player_general_traditional_total' 113 | indexes = ( 114 | (('player_id', 'season_id', 'team_id'), True), 115 | ) 116 | -------------------------------------------------------------------------------- /stats/models/PlayerSeason.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerSeason model definition. 20 | """ 21 | 22 | from peewee import ( 23 | ForeignKeyField, 24 | IntegerField, 25 | CharField, 26 | FloatField, 27 | Model 28 | ) 29 | from . import Player 30 | from . import Team 31 | 32 | 33 | class PlayerSeason(Model): 34 | 35 | # Defaulting to auto generated id column. We do this because 36 | # team_id is nullable for players in a season and that breaks the 37 | # team table's Primary Key constraint. 38 | 39 | # Composite Unique Index 40 | player_id = ForeignKeyField(Player, index=True) 41 | season_id = IntegerField(null=False, index=True) 42 | team_id = ForeignKeyField(Team, index=True, null=True) 43 | 44 | age = IntegerField(null=True) 45 | player_height = CharField(null=True) 46 | player_height_inches = IntegerField(null=True) 47 | player_weight = CharField(null=True) 48 | gp = IntegerField(null=True) 49 | pts = FloatField(null=True) 50 | reb = FloatField(null=True) 51 | ast = FloatField(null=True) 52 | net_rating = FloatField(null=True) 53 | oreb_pct = FloatField(null=True) 54 | dreb_pct = FloatField(null=True) 55 | usg_pct = FloatField(null=True) 56 | ts_pct = FloatField(null=True) 57 | ast_pct = FloatField(null=True) 58 | 59 | class Meta: 60 | db_table = 'player_season' 61 | indexes = ( 62 | (('player_id', 'season_id', 'team_id'), False), 63 | ) 64 | -------------------------------------------------------------------------------- /stats/models/Season.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Season model definition. 20 | """ 21 | 22 | from peewee import ( 23 | IntegerField, 24 | Model 25 | ) 26 | 27 | 28 | class Season(Model): 29 | season_id = IntegerField(primary_key=True) 30 | 31 | class Meta: 32 | db_table = 'season' 33 | -------------------------------------------------------------------------------- /stats/models/ShotChartDetail.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | ShotChartDetail model definition. 20 | """ 21 | 22 | from peewee import ( 23 | ForeignKeyField, 24 | IntegerField, 25 | BooleanField, 26 | FloatField, 27 | DecimalField, 28 | CharField, 29 | Model 30 | ) 31 | from . import Team 32 | from . import Game 33 | from . import Player 34 | 35 | 36 | class ShotChartDetail(Model): 37 | 38 | # Autogenerated id column here. 39 | 40 | game_id = ForeignKeyField(Game, index=True, unique=False) 41 | player_id = ForeignKeyField(Player, index=True, unique=False) 42 | team_id = ForeignKeyField(Team, index=True, unique=False) 43 | 44 | game_event_id = IntegerField(null=True) 45 | period = IntegerField(null=True) 46 | minutes_remaining = IntegerField(null=True) 47 | seconds_remaining = IntegerField(null=True) 48 | event_type = CharField(null=True) 49 | action_type = CharField(null=True) 50 | shot_type = CharField(null=True) 51 | shot_zone_basic = CharField(null=True) 52 | shot_zone_area = CharField(null=True) 53 | shot_zone_range = CharField(null=True) 54 | shot_distance = FloatField(null=True) 55 | loc_x = DecimalField(null=True) 56 | loc_y = DecimalField(null=True) 57 | shot_attempted_flag = BooleanField(null=True) 58 | shot_made_flag = BooleanField(null=True) 59 | htm = CharField(null=True) 60 | vtm = CharField(null=True) 61 | 62 | class Meta: 63 | db_table = 'shot_chart_detail' 64 | -------------------------------------------------------------------------------- /stats/models/ShotChartDetailTemp.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | ShotChartDetailTemp model definition. 20 | """ 21 | 22 | from peewee import ( 23 | IntegerField, 24 | BooleanField, 25 | FloatField, 26 | DecimalField, 27 | CharField, 28 | Model 29 | ) 30 | 31 | # This table exists because we fetch _all_ shot_chart_detail records, 32 | # even if the game doesn't exist in the game table. So, we stage all 33 | # records in this temporary table, then insert into the main table 34 | # filtering by existing game_ids. 35 | 36 | 37 | class ShotChartDetailTemp(Model): 38 | 39 | # Autogenerated id column here. 40 | 41 | game_id = IntegerField(index=True, unique=False) 42 | 43 | player_id = IntegerField(unique=False) 44 | team_id = IntegerField(unique=False) 45 | 46 | game_event_id = IntegerField(null=True) 47 | period = IntegerField(null=True) 48 | minutes_remaining = IntegerField(null=True) 49 | seconds_remaining = IntegerField(null=True) 50 | event_type = CharField(null=True) 51 | action_type = CharField(null=True) 52 | shot_type = CharField(null=True) 53 | shot_zone_basic = CharField(null=True) 54 | shot_zone_area = CharField(null=True) 55 | shot_zone_range = CharField(null=True) 56 | shot_distance = FloatField(null=True) 57 | loc_x = DecimalField(null=True) 58 | loc_y = DecimalField(null=True) 59 | shot_attempted_flag = BooleanField(null=True) 60 | shot_made_flag = BooleanField(null=True) 61 | htm = CharField(null=True) 62 | vtm = CharField(null=True) 63 | 64 | class Meta: 65 | db_table = 'shot_chart_detail_temp' 66 | temporary = True 67 | -------------------------------------------------------------------------------- /stats/models/Team.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Team model definition. 20 | """ 21 | 22 | from peewee import ( 23 | IntegerField, 24 | CharField, 25 | Model, 26 | ) 27 | 28 | 29 | class Team(Model): 30 | 31 | # Primary Key 32 | team_id = IntegerField(primary_key=True) 33 | 34 | abbreviation = CharField(null=True) 35 | nickname = CharField(null=True) 36 | yearfounded = CharField(null=True) 37 | city = CharField(null=True) 38 | 39 | class Meta: 40 | db_table = 'team' 41 | -------------------------------------------------------------------------------- /stats/models/TeamGameLog.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | TeamGameLog model definition. 20 | """ 21 | 22 | from peewee import ( 23 | ForeignKeyField, 24 | IntegerField, 25 | CharField, 26 | FloatField, 27 | Model, 28 | CompositeKey 29 | ) 30 | from . import Team 31 | from . import Game 32 | 33 | 34 | class TeamGameLog(Model): 35 | 36 | # Composite PK Fields 37 | team_id = ForeignKeyField(Team, index=True) 38 | game_id = ForeignKeyField(Game, index=True) 39 | 40 | # Indexes 41 | season_id = IntegerField(index=True) 42 | 43 | game_date = CharField(null=True) 44 | matchup = CharField(null=True) 45 | wl = CharField(null=True) 46 | min = FloatField(null=True) 47 | fgm = FloatField(null=True) 48 | fga = FloatField(null=True) 49 | fg_pct = FloatField(null=True) 50 | fg3m = FloatField(null=True) 51 | fg3a = FloatField(null=True) 52 | fg3_pct = FloatField(null=True) 53 | ftm = FloatField(null=True) 54 | fta = FloatField(null=True) 55 | ft_pct = FloatField(null=True) 56 | oreb = FloatField(null=True) 57 | dreb = FloatField(null=True) 58 | reb = FloatField(null=True) 59 | ast = FloatField(null=True) 60 | stl = FloatField(null=True) 61 | blk = FloatField(null=True) 62 | tov = FloatField(null=True) 63 | pf = FloatField(null=True) 64 | pts = FloatField(null=True) 65 | plus_minus = FloatField(null=True) 66 | video_available = IntegerField(null=True) 67 | 68 | class Meta: 69 | db_table = 'team_game_log' 70 | primary_key = CompositeKey( 71 | 'team_id', 72 | 'game_id', 73 | ) 74 | -------------------------------------------------------------------------------- /stats/models/TeamSeason.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | TeamSeason model definition. 20 | """ 21 | 22 | from peewee import ( 23 | IntegerField, 24 | CharField, 25 | Model 26 | ) 27 | 28 | 29 | class TeamSeason(Model): 30 | 31 | team_id = IntegerField(primary_key=True) 32 | 33 | # Indexes 34 | season_id = IntegerField(index=True) 35 | 36 | owner = CharField(null=True) 37 | general_manager = CharField(null=True) 38 | head_coach = CharField(null=True) 39 | dleague_affiliation = CharField(null=True) 40 | 41 | class Meta: 42 | db_table = 'team' 43 | -------------------------------------------------------------------------------- /stats/models/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Import wrapper. 20 | """ 21 | 22 | # Base Tables 23 | from .Player import Player 24 | from .Team import Team 25 | from .Game import Game 26 | from .Season import Season 27 | 28 | # Misc Tables 29 | from .EventMessageType import EventMessageType 30 | 31 | # Team Tables 32 | from .TeamSeason import TeamSeason 33 | from .TeamGameLog import TeamGameLog 34 | 35 | # Player Tables 36 | from .PlayerSeason import PlayerSeason 37 | from .PlayerGameLog import PlayerGameLog 38 | from .PlayerGameLogTemp import PlayerGameLogTemp 39 | from .PlayerGeneralTraditionalTotal import PlayerGeneralTraditionalTotal 40 | from .PlayByPlay import PlayByPlay 41 | from .PlayByPlayV3 import PlayByPlayV3 42 | from .ShotChartDetail import ShotChartDetail 43 | from .ShotChartDetailTemp import ShotChartDetailTemp 44 | 45 | __all__ = [ 46 | Player, 47 | Team, 48 | Game, 49 | Season, 50 | EventMessageType, 51 | TeamSeason, 52 | TeamGameLog, 53 | PlayerSeason, 54 | PlayerGameLog, 55 | PlayerGameLogTemp, 56 | PlayerGeneralTraditionalTotal, 57 | PlayByPlay, 58 | PlayByPlayV3, 59 | ShotChartDetail, 60 | ShotChartDetailTemp 61 | ] 62 | -------------------------------------------------------------------------------- /stats/nba_sql.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Main application driver. 20 | """ 21 | 22 | from team import TeamRequester 23 | from player import PlayerRequester 24 | from event_message_type import EventMessageTypeBuilder 25 | from game import GameBuilder 26 | from season import SeasonBuilder 27 | 28 | from player_season import PlayerSeasonRequester 29 | from player_game_log import PlayerGameLogRequester 30 | from player_general_traditional_total import ( 31 | PlayerGeneralTraditionalTotalRequester 32 | ) 33 | from play_by_play import PlayByPlayRequester 34 | from play_by_playv3 import PlayByPlayV3Requester 35 | from shot_chart_detail import ShotChartDetailRequester 36 | 37 | from constants import team_ids 38 | from settings import Settings 39 | from utils import progress_bar, generate_valid_seasons, generate_valid_season 40 | 41 | from args import create_parser 42 | 43 | import concurrent.futures 44 | import argparse 45 | import time 46 | import copy 47 | import sys 48 | 49 | description = """ 50 | nba_sql application. 51 | 52 | The command loads the database with historic data from the 53 | 1996-97 / 2019-20 seasons. 54 | 55 | EX: 56 | python3 stats/nba_sql.py 57 | """ 58 | 59 | 60 | # TODO: load these args into the settings class. 61 | def default_mode(settings, create_schema, request_gap, seasons, skip_tables, quiet): 62 | """ 63 | The default mode of loading data. This is for initializing the database 64 | and loading specific seasons. 65 | """ 66 | 67 | print("Loading the database in the default mode.") 68 | 69 | player_requester = PlayerRequester(settings) 70 | team_requester = TeamRequester(settings) 71 | event_message_type_builder = EventMessageTypeBuilder(settings) 72 | game_builder = GameBuilder(settings) 73 | season_builder = SeasonBuilder(settings) 74 | 75 | player_season_requester = PlayerSeasonRequester(settings) 76 | player_game_log_requester = PlayerGameLogRequester(settings) 77 | pgtt_requester = PlayerGeneralTraditionalTotalRequester(settings) 78 | play_by_play_requester = PlayByPlayRequester(settings) 79 | play_by_playv3_requester = PlayByPlayV3Requester(settings) 80 | shot_chart_requester = ShotChartDetailRequester(settings) 81 | 82 | object_list = [ 83 | # Base Objects 84 | player_requester, 85 | team_requester, 86 | event_message_type_builder, 87 | game_builder, 88 | season_builder, 89 | 90 | # Dependent Objects 91 | player_season_requester, 92 | player_game_log_requester, 93 | play_by_play_requester, 94 | play_by_playv3_requester, 95 | pgtt_requester, 96 | shot_chart_requester 97 | ] 98 | 99 | if create_schema: 100 | do_create_schema(object_list) 101 | 102 | season_builder.populate(seasons) 103 | 104 | if 'team' not in skip_tables: 105 | print('Populating team table.') 106 | 107 | team_bar = progress_bar(team_ids, prefix='team Table Loading', suffix='', length=30, quiet=quiet) 108 | for team_id in team_bar: 109 | team_requester.generate_rows(team_id) 110 | time.sleep(request_gap) 111 | 112 | team_requester.populate() 113 | 114 | if 'event_message_type' not in skip_tables: 115 | print('Loading event types.') 116 | event_message_type_builder.initialize() 117 | 118 | if 'player' not in skip_tables: 119 | print('Populating player data') 120 | 121 | player_bar = progress_bar(seasons, prefix='player Table Loading', suffix='', length=30, quiet=quiet) 122 | for season_id in player_bar: 123 | player_requester.generate_rows(season_id) 124 | time.sleep(request_gap) 125 | player_requester.populate() 126 | 127 | player_game_seasons_bar = progress_bar( 128 | seasons, 129 | prefix='Loading player_game_log regular season data', 130 | suffix='This one will take a while...', 131 | length=30, 132 | quiet=quiet) 133 | 134 | # Fetch player_game_log and build game_id set. 135 | for season_id in player_game_seasons_bar: 136 | 137 | player_game_log_requester.fetch_season(season_id, False) 138 | time.sleep(request_gap) 139 | 140 | player_game_seasons_bar = progress_bar( 141 | seasons, 142 | prefix='Loading player_game_log playoff season data', 143 | suffix='This one will take a while...', 144 | length=30, 145 | quiet=quiet) 146 | 147 | for season_id in player_game_seasons_bar: 148 | player_game_log_requester.fetch_season(season_id, True) 149 | time.sleep(request_gap) 150 | 151 | game_set = player_game_log_requester.get_game_set() 152 | 153 | # Fetch ids from tuples. 154 | game_list = [game[1] for game in game_set] 155 | 156 | game_progress_bar = progress_bar( 157 | game_list, 158 | prefix='Loading PlayByPlay Data', 159 | length=30, 160 | quiet=quiet) 161 | 162 | # First, load game specific data. 163 | if 'game' not in skip_tables: 164 | print('Loading cached game table.') 165 | game_builder.populate_table(game_set) 166 | 167 | if 'play_by_play' not in skip_tables: 168 | play_by_play_helper( 169 | play_by_play_requester, 170 | player_requester, 171 | game_list, 172 | 'Loading PlayByPlay Data', 173 | quiet, 174 | request_gap) 175 | 176 | if 'play_by_playv3' not in skip_tables: 177 | play_by_play_helper( 178 | play_by_playv3_requester, 179 | player_requester, 180 | game_list, 181 | 'Loading PlayByPlayV3 Data', 182 | quiet, 183 | request_gap) 184 | 185 | if 'player_game_log' not in skip_tables: 186 | 187 | print("Starting PlayerGameLog Insert") 188 | player_game_log_requester.populate() 189 | print("Finished PlayerGameLog Insert") 190 | 191 | if 'shot_chart_detail' not in skip_tables: 192 | 193 | print("Fetching set of team_id and player_ids for the ShotChartData.") 194 | team_player_set = player_game_log_requester.get_team_player_id_set() 195 | print("Finished fetching.") 196 | shot_chart_bar = progress_bar( 197 | team_player_set, 198 | prefix='Loading Shot Chart Data', 199 | suffix='', 200 | length=30, 201 | quiet=quiet) 202 | 203 | for id_tuple in shot_chart_bar: 204 | 205 | shot_chart_requester.generate_rows(id_tuple[0], id_tuple[1]) 206 | shot_chart_requester.populate() 207 | time.sleep(request_gap) 208 | 209 | shot_chart_requester.finalize(game_builder.game_id_predicate()) 210 | 211 | season_bar = progress_bar( 212 | seasons, 213 | prefix='Loading Seasonal Data', 214 | suffix='This one will take a while...', 215 | length=30, 216 | quiet=quiet) 217 | 218 | # Load seasonal data. 219 | for season_id in season_bar: 220 | if 'player_season' not in skip_tables: 221 | player_season_requester.populate_season(season_id) 222 | time.sleep(request_gap) 223 | 224 | if 'pgtt' not in skip_tables: 225 | pgtt_requester.generate_rows(season_id) 226 | time.sleep(request_gap) 227 | 228 | print("Done! Enjoy the hot, fresh database.") 229 | 230 | 231 | def do_create_schema(object_list): 232 | """ 233 | Function to initialize database schema. 234 | """ 235 | print("Initializing schema.") 236 | 237 | for obj in object_list: 238 | obj.create_ddl() 239 | 240 | 241 | def current_season_mode(settings, request_gap, skip_tables, quiet): 242 | """ 243 | Refreshes the current season in a previously existing database. 244 | """ 245 | 246 | player_requester = PlayerRequester(settings) 247 | player_game_log_requester = PlayerGameLogRequester(settings) 248 | game_builder = GameBuilder(settings) 249 | shot_chart_requester = ShotChartDetailRequester(settings) 250 | play_by_play_requester = PlayByPlayRequester(settings) 251 | play_by_playv3_requester = PlayByPlayV3Requester(settings) 252 | season_builder = SeasonBuilder(settings) 253 | 254 | season_id = season_builder.current_season_loaded() 255 | 256 | if season_id is None: 257 | sys.exit(''' 258 | Error: option '--current-season-mode' passed on an uninitialized database. 259 | First load a season using the '--default-mode' flag! 260 | ''') 261 | 262 | if not quiet: 263 | print("Refreshing the current season in the existing database.") 264 | 265 | season = generate_valid_season(season_id) 266 | 267 | if not quiet: 268 | print("Fetching current season data.") 269 | 270 | game_set_old = game_builder.fetch_season_game_id_set(season_id) 271 | 272 | player_game_log_requester.fetch_season(season, False) 273 | time.sleep(request_gap) 274 | player_game_log_requester.fetch_season(season, True) 275 | player_game_log_requester.populate_temp() 276 | time.sleep(request_gap) 277 | 278 | if 'player_game_log' not in skip_tables: 279 | player_game_log_requester.insert_from_temp_into_reg() 280 | 281 | game_set = player_game_log_requester.get_game_set() 282 | game_set_new = set([game[1] for game in game_set]) 283 | 284 | game_set_net_new = game_set_new.difference(game_set_old) 285 | print(f"Net new games found: {len(game_set_net_new)}") 286 | 287 | # Insert new games and ignore duplicates, becuase its difficult to 288 | # do this the correct way. 289 | game_builder.populate_table(game_set, True) 290 | 291 | game_list = list(game_set_net_new) 292 | if 'play_by_play' not in skip_tables: 293 | play_by_play_helper( 294 | play_by_play_requester, 295 | player_requester, 296 | game_list, 297 | 'Loading PlayByPlay Data', 298 | quiet, 299 | request_gap) 300 | 301 | if 'play_by_playv3' not in skip_tables: 302 | play_by_play_helper( 303 | play_by_playv3_requester, 304 | player_requester, 305 | game_list, 306 | 'Loading PlayByPlayV3 Data', 307 | quiet, 308 | request_gap) 309 | 310 | if 'shot_chart_detail' not in skip_tables: 311 | team_player_set = player_game_log_requester.get_team_player_id_set(True) 312 | 313 | shot_chart_bar = progress_bar( 314 | team_player_set, 315 | prefix='Loading Shot Chart Data', 316 | suffix='', 317 | length=30, 318 | quiet=quiet) 319 | 320 | for id_tuple in shot_chart_bar: 321 | 322 | shot_chart_requester.generate_rows(id_tuple[0], id_tuple[1]) 323 | shot_chart_requester.populate() 324 | time.sleep(request_gap) 325 | 326 | scd_predicate = shot_chart_requester.temp_table_except_predicate() 327 | shot_chart_requester.finalize(scd_predicate) 328 | 329 | if quiet: 330 | print("ok") 331 | 332 | 333 | def main(args, from_gui): 334 | """ 335 | Main driver for the nba-sql application. 336 | """ 337 | 338 | if from_gui: 339 | print("In GUI mode. Note, this mode has minimal feedback. It might take some time to print progress.") 340 | 341 | # CMD line args. 342 | default_mode_set = args.default_mode 343 | current_season_mode_set = args.current_season_mode 344 | 345 | if not default_mode_set and not current_season_mode_set: 346 | sys.exit(''' 347 | Error: Pass either '--default-mode' to create the database / add a new season, or 348 | '--current-season-mode' to refresh the last season loaded in an existing database. 349 | ''') 350 | 351 | create_schema = args.create_schema 352 | request_gap = float(args.request_gap) 353 | seasons = args.seasons 354 | skip_tables = args.skip_tables 355 | quiet = args.quiet 356 | 357 | if not quiet: 358 | print(f"Loading seasons: {seasons}.") 359 | 360 | settings = Settings( 361 | args.database_type, 362 | args.database_name, 363 | args.username, 364 | args.password, 365 | args.database_host, 366 | args.batch_size, 367 | args.sqlite_path, 368 | args.quiet) 369 | 370 | if default_mode_set: 371 | default_mode(settings, create_schema, request_gap, seasons, skip_tables, quiet or from_gui) 372 | if current_season_mode_set: 373 | current_season_mode(settings, request_gap, skip_tables, quiet) 374 | 375 | def play_by_play_helper(pbp_requester, player_requester, game_list, display_str, quiet, request_gap): 376 | """ 377 | Helper function to take care of concurrent fetching and insertion. 378 | """ 379 | 380 | # Load game dependent data. 381 | player_id_set = player_requester.get_id_set() 382 | rows = [] 383 | game_progress_bar = progress_bar(game_list, prefix=display_str, length=30, quiet=quiet) 384 | 385 | # Okay so this takes a really long time due to rate 386 | # limiting and over 25K games. Best we can do so 387 | # far is batch the rows into groups of 100K and insert them 388 | # in a different thread. 389 | with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: 390 | for game_id in game_progress_bar: 391 | new_rows = pbp_requester.fetch_game(game_id) 392 | rows += new_rows 393 | 394 | if len(rows) > 100000: 395 | # We should be good for the race condition here. 396 | # It takes a wee bit to insert 100K rows. 397 | copy_list = copy.deepcopy(rows) 398 | executor.submit( 399 | pbp_requester.insert_batch, 400 | copy_list, player_id_set 401 | ) 402 | rows = [] 403 | time.sleep(request_gap) 404 | 405 | if rows: 406 | print(f"Inserting excess {len(rows)} PlayByPlay/PlayByPlayV3 rows.") 407 | pbp_requester.insert_batch(rows, player_id_set) 408 | 409 | # Default non-gui executable. 410 | if __name__ == "__main__": 411 | parser = argparse.ArgumentParser(description='nba-sql') 412 | create_parser(parser) 413 | 414 | parser.add_argument( 415 | '--default-mode', 416 | help=''' 417 | Mode to create the database and load historic data. Use this mode when creating a 418 | new database or when trying to load a specific season or a range of seasons. 419 | ''', 420 | action='store_true') 421 | parser.add_argument( 422 | '--current-season-mode', 423 | help=''' 424 | Mode to refresh the current season. Use this mode on an existing database 425 | to update it with the latest data. 426 | ''', 427 | action='store_true') 428 | 429 | parser.add_argument( 430 | '--password', 431 | help="Database Password (Not Needed For SQLite)", 432 | default=None) 433 | 434 | valid_seasons = generate_valid_seasons() 435 | last_loadable_season = valid_seasons[-1] 436 | 437 | parser.add_argument( 438 | '--seasons', 439 | dest='seasons', 440 | default=[last_loadable_season], 441 | choices=valid_seasons, 442 | nargs="*", 443 | help=''' 444 | The seasons flag loads the database with the specified season. The format of the season 445 | should be in the form "YYYY-YY". The default behavior is loading the current season. 446 | ''') 447 | 448 | parser.add_argument( 449 | '--skip-tables', 450 | action='store', 451 | nargs="*", 452 | default='', 453 | choices=[ 454 | 'player_season', 455 | 'player_game_log', 456 | 'play_by_play', 457 | 'play_by_playv3', 458 | 'pgtt', 459 | 'shot_chart_detail', 460 | 'game', 461 | 'event_message_type', 462 | 'team', 463 | 'player', 464 | '' 465 | ], 466 | help='Use this option to skip loading certain tables.') 467 | 468 | args = parser.parse_args() 469 | 470 | main(args, False) 471 | -------------------------------------------------------------------------------- /stats/play_by_play.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayByPlay object requester and builder. 20 | """ 21 | 22 | import requests 23 | import urllib.parse 24 | 25 | from models import PlayByPlay 26 | from constants import headers 27 | from db_utils import insert_many 28 | 29 | 30 | class PlayByPlayRequester: 31 | 32 | url = 'https://stats.nba.com/stats/playbyplayv2' 33 | 34 | def __init__(self, settings): 35 | self.settings = settings 36 | self.settings.db.bind([PlayByPlay]) 37 | 38 | def create_ddl(self): 39 | """ 40 | Initialize the table schema. 41 | """ 42 | self.settings.db.create_tables([PlayByPlay], safe=True) 43 | 44 | def fetch_game(self, game_id): 45 | """ 46 | Build GET REST request to the NBA for a game, iterate over 47 | the results and return them. 48 | """ 49 | params = self.build_params(game_id) 50 | 51 | # Encode without safe '+', apparently the NBA likes unsafe url params. 52 | params_str = urllib.parse.urlencode(params, safe=':+') 53 | 54 | response = requests.get(url=self.url, headers=headers, params=params_str).json() 55 | 56 | # pulling just the data we want 57 | player_info = response['resultSets'][0]['rowSet'] 58 | 59 | rows = [] 60 | 61 | # looping over data to return. 62 | for row in player_info: 63 | new_row = { 64 | 'game_id': row[0], 65 | 'event_num': row[1], 66 | 'event_msg_type': row[2], 67 | 'event_msg_action_type': row[3], 68 | 'period': row[4], 69 | 'wc_time': row[5], 70 | 'home_description': row[7], 71 | 'neutral_description': row[8], 72 | 'visitor_description': row[9], 73 | 'score': row[10], 74 | 'score_margin': row[11], 75 | 'player1_id': self.get_null_id(row[13]), 76 | 'player1_team_id': self.get_null_id(row[15]), 77 | 'player2_id': self.get_null_id(row[20]), 78 | 'player2_team_id': self.get_null_id(row[22]), 79 | 'player3_id': self.get_null_id(row[27]), 80 | 'player3_team_id': self.get_null_id(row[29]) 81 | } 82 | 83 | rows.append(new_row) 84 | return rows 85 | 86 | def insert_batch(self, rows, player_id_set): 87 | """ 88 | Batch insertion of records. 89 | """ 90 | 91 | # It looks like the NBA API returns some bad data that 92 | # doesn't conform to their advertized schema: 93 | # (team_id in the player_id spot). 94 | # We can maybe get away with ignoring it. 95 | # Check if id is in player_id cache. 96 | # We need to preserve the row in general becuase it could still have 97 | # good data for the correctly returned players. 98 | 99 | for row in rows: 100 | for key in ['player1_id', 'player2_id', 'player3_id']: 101 | if row[key] is not None and row[key] not in player_id_set: 102 | row[key] is None 103 | insert_many(self.settings, PlayByPlay, rows) 104 | 105 | def build_params(self, game_id): 106 | """ 107 | Create required parameters dict for the request. 108 | """ 109 | return { 110 | 'EndPeriod': 6, 111 | 'GameId': game_id, 112 | 'StartPeriod': 1 113 | } 114 | 115 | def get_null_id(self, id): 116 | """ 117 | This endpoint will return a player's id or player's team id as 0 118 | sometimes. We will store 'null', as 0 breaks the foriegn key 119 | constraint. 120 | """ 121 | if id == 0: 122 | return None 123 | return id 124 | -------------------------------------------------------------------------------- /stats/play_by_playv3.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayByPlayV3 object requester and builder. 20 | 21 | This has a simpler schema than PlayByPlay 22 | """ 23 | 24 | import requests 25 | import urllib.parse 26 | 27 | from models import PlayByPlayV3 28 | from constants import headers 29 | from db_utils import insert_many 30 | 31 | 32 | class PlayByPlayV3Requester: 33 | 34 | url = 'https://stats.nba.com/stats/playbyplayv3' 35 | 36 | def __init__(self, settings): 37 | self.settings = settings 38 | self.settings.db.bind([PlayByPlayV3]) 39 | 40 | def create_ddl(self): 41 | """ 42 | Initialize the table schema. 43 | """ 44 | self.settings.db.create_tables([PlayByPlayV3], safe=True) 45 | 46 | def fetch_game(self, game_id): 47 | """ 48 | Build GET REST request to the NBA for a game, iterate over 49 | the results and return them. 50 | """ 51 | params = self.build_params(game_id) 52 | 53 | # Encode without safe '+', apparently the NBA likes unsafe url params. 54 | params_str = urllib.parse.urlencode(params, safe=':+') 55 | 56 | response = requests.get(url=self.url, headers=headers, params=params_str).json() 57 | 58 | # pulling just the data we want 59 | player_info = response['game']['actions'] 60 | game_id = response['game']['gameId'] 61 | 62 | rows = [] 63 | 64 | # looping over data to return. 65 | for row in player_info: 66 | new_row = { 67 | 'game_id': game_id, 68 | 'action_number': row['actionNumber'], 69 | 'clock': row['clock'], 70 | 'period': row['period'], 71 | 'team_id': self.get_null_id(row['teamId']), 72 | 'player_id': self.get_null_id(row['personId']), 73 | 'x_legacy': row['xLegacy'], 74 | 'y_legacy': row['yLegacy'], 75 | 'shot_distance': row['shotDistance'], 76 | 'shot_result': row['shotResult'], 77 | 'is_field_goal': row['isFieldGoal'], 78 | 'score_home': row['scoreHome'], 79 | 'score_away': row['scoreAway'], 80 | 'points_total': row['pointsTotal'], 81 | 'location': row['location'], 82 | 'description': row['description'], 83 | 'action_type': row['actionType'], 84 | 'sub_type': row['subType'] 85 | } 86 | rows.append(new_row) 87 | return rows 88 | 89 | def insert_batch(self, rows, player_id_set): 90 | """ 91 | Batch insertion of records. 92 | 93 | In this case the third arg is unused. 94 | """ 95 | insert_many(self.settings, PlayByPlayV3, rows) 96 | 97 | def build_params(self, game_id): 98 | """ 99 | Create required parameters dict for the request. 100 | """ 101 | return { 102 | 'EndPeriod': 6, 103 | 'GameId': game_id, 104 | 'StartPeriod': 1 105 | } 106 | 107 | def get_null_id(self, id): 108 | """ 109 | This endpoint will return a player's id or player's team id as 0 110 | sometimes. We will store 'null', as 0 breaks the foriegn key 111 | constraint. 112 | """ 113 | if id == 0: 114 | return None 115 | return id 116 | -------------------------------------------------------------------------------- /stats/player.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Player requester and builder. 20 | """ 21 | 22 | import urllib.parse 23 | 24 | from models import Player 25 | from general_requester import GenericRequester 26 | from db_utils import insert_many_on_conflict_ignore 27 | 28 | 29 | class PlayerRequester(GenericRequester): 30 | 31 | per_mode = 'Totals' 32 | player_info_url = 'http://stats.nba.com/stats/leaguedashplayerbiostats' 33 | 34 | def __init__(self, settings): 35 | """ 36 | Constructor. Pass on all relevant vars. 37 | """ 38 | super().__init__(settings, self.player_info_url, Player) 39 | 40 | def get_id_set(self): 41 | """ 42 | Gets a set of ids for caching. 43 | """ 44 | s = set() 45 | for player in Player.select(Player.player_id): 46 | s.add(player.player_id) 47 | return s 48 | 49 | def generate_rows(self, season_id): 50 | """ 51 | Build GET REST request to the NBA for a season. 52 | """ 53 | params = self.build_params(season_id) 54 | 55 | # Encode without safe '+', apparently the NBA likes unsafe url params. 56 | params_str = urllib.parse.urlencode(params, safe=':+') 57 | super().generate_rows(params_str) 58 | 59 | def populate(self): 60 | """ 61 | Store collected rows. Custom implementation for the on_conflict_ignore 62 | argument. 63 | """ 64 | insert_many_on_conflict_ignore(self.settings, Player, self.rows) 65 | 66 | def build_params(self, season_id): 67 | """ 68 | Create required parameters dict for the request. 69 | """ 70 | return { 71 | 'College': '', 72 | 'Conference': '', 73 | 'Country': '', 74 | 'DateFrom': '', 75 | 'DateTo': '', 76 | 'Division': '', 77 | 'DraftPick': '', 78 | 'DraftYear': '', 79 | 'GameScope': '', 80 | 'GameSegment': '', 81 | 'Height': '', 82 | 'LastNGames': '0', 83 | 'LeagueID': '00', 84 | 'Location': '', 85 | 'Month': '0', 86 | 'OpponentTeamID': '0', 87 | 'Outcome': '', 88 | 'PORound': '0', 89 | 'PerMode': self.per_mode, 90 | 'Period': '0', 91 | 'PlayerExperience': '', 92 | 'PlayerPosition': '', 93 | 'Season': season_id, 94 | 'SeasonSegment': '', 95 | 'SeasonType': 'Regular+Season', 96 | 'ShotClockRange': '', 97 | 'StarterBench': '', 98 | 'TeamID': '0', 99 | 'VsConference': '', 100 | 'VsDivision': '', 101 | 'Weight': '' 102 | } 103 | -------------------------------------------------------------------------------- /stats/player_game_log.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerGameLog requester and builder. Also takes care of the Game table. See 20 | class-level docstring. 21 | """ 22 | 23 | import requests 24 | import urllib.parse 25 | 26 | from db_utils import insert_many 27 | from utils import get_rowset_mapping, column_names_from_table, season_id_to_int 28 | from models import PlayerGameLog, PlayerGameLogTemp 29 | from game import GameEntry 30 | from general_requester import GenericRequester 31 | from constants import headers 32 | 33 | 34 | class PlayerGameLogRequester(GenericRequester): 35 | """ 36 | This class builds player game data from a season. 37 | As an optimization, we also build a set of game data. 38 | This is so we can build the game table without having to 39 | make a request for every game. We probably shouldn't 40 | do this, as this relys on data from the endpoint and if that 41 | changes this logic also must. But its fine for now... 42 | """ 43 | 44 | url = 'https://stats.nba.com/stats/playergamelogs' 45 | game_set = set() 46 | 47 | def __init__(self, settings): 48 | """ 49 | Constructor. 50 | """ 51 | super().__init__(settings, self.url, PlayerGameLog) 52 | # TODO: this conflicts with a fresh db. 53 | self.settings.db.bind([PlayerGameLogTemp]) 54 | self.settings.db.create_tables([PlayerGameLogTemp], safe=True) 55 | 56 | def create_ddl(self): 57 | """ 58 | Override method to setup temp table. 59 | """ 60 | super().create_ddl() 61 | 62 | def populate_temp(self): 63 | """ 64 | Bulk insert. 65 | """ 66 | insert_many(self.settings, PlayerGameLogTemp, self.rows) 67 | # TODO: should set rows to []? 68 | 69 | def get_game_set(self): 70 | """ 71 | Returns a the set of game ids. 72 | """ 73 | return self.game_set 74 | 75 | def get_rows(self): 76 | """ 77 | Returns the stored row list, to be inserted after the game data. 78 | """ 79 | return self.rows 80 | 81 | def set_game_set(self, set_new): 82 | """ 83 | Sets the game set. 84 | """ 85 | self.game_set = set_new 86 | 87 | def get_team_player_id_set(self, temp_table=False): 88 | """ 89 | Returns a set of team id and player ids, used for the shot_chart_detail api. 90 | """ 91 | s = set() 92 | 93 | if temp_table: 94 | table = PlayerGameLogTemp 95 | else: 96 | table = PlayerGameLog 97 | 98 | tid = table.team_id 99 | pid = table.player_id 100 | 101 | for player_game_log in table.select(tid, pid).group_by(tid, pid): 102 | s.add((player_game_log.team_id, player_game_log.player_id)) 103 | 104 | return s 105 | 106 | def temp_table_except_predicate(self): 107 | """ 108 | This runs an EXCEPT between the temp table and the non-temp table to find 109 | the new games. 110 | """ 111 | regular_query = PlayerGameLog.select(PlayerGameLog.game_id) 112 | temp_query = PlayerGameLogTemp.select(PlayerGameLogTemp.game_id) 113 | 114 | expt = temp_query - regular_query 115 | 116 | return expt.select_from(expt.c.game_id) 117 | 118 | def fetch_season(self, season_id, playoff_games): 119 | """ 120 | Build GET REST request to the NBA for a season, 121 | iterate over the results, store in the database. 122 | 123 | `playoff_games` is a boolean used to load regular or playoff games. 124 | """ 125 | params = self.build_params(season_id, playoff_games) 126 | 127 | # Encode without safe '+', apparently the NBA likes unsafe url params. 128 | params_str = urllib.parse.urlencode(params, safe=':+') 129 | 130 | response = requests.get(url=self.url, headers=headers, params=params_str).json() 131 | 132 | result_sets = response['resultSets'][0] 133 | rowset = result_sets['rowSet'] 134 | 135 | season_int = season_id_to_int(season_id) 136 | column_names = column_names_from_table(self.settings.db, self.table._meta.table_name) 137 | 138 | column_mapping = get_rowset_mapping(result_sets, column_names) 139 | 140 | rowset_mapping = get_rowset_mapping(result_sets, self.local_resultset_rows()) 141 | 142 | # looping over data to insert into table 143 | for row in rowset: 144 | 145 | matchup = row[rowset_mapping['MATCHUP']] 146 | wl = row[rowset_mapping['WL']] 147 | team_id = row[rowset_mapping['TEAM_ID']] 148 | game_date = row[rowset_mapping['GAME_DATE']] 149 | game_id = row[rowset_mapping['GAME_ID']] 150 | 151 | # Checking matchup for home team. 152 | if '@' in matchup: 153 | if wl == "W": 154 | winner = team_id 155 | loser = "" 156 | else: 157 | winner = "" 158 | loser = team_id 159 | self.game_set.add( 160 | GameEntry( 161 | season_id=season_int, 162 | game_id=game_id, 163 | game_date=game_date, 164 | matchup_in=matchup, 165 | winner=winner, 166 | playoff_game=playoff_games, 167 | loser=loser)) 168 | 169 | new_row = {column_name: row[row_index] for column_name, row_index in column_mapping.items()} 170 | new_row['season_id'] = season_int 171 | self.rows.append(new_row) 172 | 173 | def build_params(self, season_id, playoff_games): 174 | """ 175 | Create required parameters dict for the request. 176 | """ 177 | season_type = 'Regular Season' 178 | if playoff_games: 179 | season_type = 'Playoffs' 180 | return { 181 | 'DateFrom': '', 182 | 'DateTo': '', 183 | 'GameSegment': '', 184 | 'LastNGames': '', 185 | 'LeagueID': '00', 186 | 'Location': '', 187 | 'MeasureType': '', 188 | 'Month': '', 189 | 'OppTeamID': '', 190 | 'Outcome': '', 191 | 'PORound': '', 192 | 'PerMode': '', 193 | 'Period': '', 194 | 'PlayerID': '', 195 | 'Season': season_id, 196 | 'SeasonSegment': '', 197 | 'SeasonType': season_type, 198 | 'ShotClockRange': '', 199 | 'TeamID': '', 200 | 'VsConference': '', 201 | 'VsDivision': '' 202 | } 203 | 204 | def local_resultset_rows(self): 205 | """ 206 | Returns list of the specific rows that we want to pull from the request. 207 | """ 208 | 209 | return [ 210 | 'MATCHUP', 211 | 'WL', 212 | 'TEAM_ID', 213 | 'GAME_ID', 214 | 'GAME_DATE' 215 | ] 216 | 217 | def insert_from_temp_into_reg(self): 218 | """ 219 | Inserts values from the temp table into the regular table that don't exist 220 | in the regular table already. 221 | 222 | THERE HAS TO BE A BETTER WAY OF DEFINING ALL FIELDS. 223 | """ 224 | predicate = PlayerGameLog.select(PlayerGameLog.game_id) 225 | 226 | (PlayerGameLog.insert_from( 227 | PlayerGameLogTemp 228 | .select() 229 | .where(PlayerGameLogTemp.game_id.not_in(predicate)), 230 | 231 | fields=[ 232 | PlayerGameLog.player_id, 233 | PlayerGameLog.game_id, 234 | PlayerGameLog.team_id, 235 | PlayerGameLog.season_id, 236 | PlayerGameLog.wl, 237 | PlayerGameLog.min, 238 | PlayerGameLog.fgm, 239 | PlayerGameLog.fga, 240 | PlayerGameLog.fg_pct, 241 | PlayerGameLog.fg3m, 242 | PlayerGameLog.fg3a, 243 | PlayerGameLog.fg3_pct, 244 | PlayerGameLog.ftm, 245 | PlayerGameLog.fta, 246 | PlayerGameLog.ft_pct, 247 | PlayerGameLog.oreb, 248 | PlayerGameLog.dreb, 249 | PlayerGameLog.reb, 250 | PlayerGameLog.ast, 251 | PlayerGameLog.tov, 252 | PlayerGameLog.stl, 253 | PlayerGameLog.blk, 254 | PlayerGameLog.blka, 255 | PlayerGameLog.pf, 256 | PlayerGameLog.pfd, 257 | PlayerGameLog.pts, 258 | PlayerGameLog.plus_minus, 259 | PlayerGameLog.nba_fantasy_pts, 260 | PlayerGameLog.dd2, 261 | PlayerGameLog.td3 262 | ] 263 | )).execute() 264 | -------------------------------------------------------------------------------- /stats/player_general_traditional_total.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerGeneralTraditionalTotal builder and requester. 20 | """ 21 | 22 | import requests 23 | import urllib.parse 24 | 25 | from utils import get_rowset_mapping, column_names_from_table, season_id_to_int 26 | from models import PlayerGeneralTraditionalTotal 27 | from general_requester import GenericRequester 28 | from constants import headers 29 | 30 | 31 | class PlayerGeneralTraditionalTotalRequester(GenericRequester): 32 | 33 | player_info_url = 'https://stats.nba.com/stats/leaguedashplayerstats' 34 | per_mode = 'Totals' 35 | 36 | def __init__(self, settings): 37 | """ 38 | Constructor. Attach settings internally and bind the model to the 39 | database. 40 | """ 41 | super().__init__(settings, self.player_info_url, PlayerGeneralTraditionalTotal) 42 | 43 | def generate_rows(self, season_id): 44 | """ 45 | Build GET REST request to the NBA for a season. 46 | Also populate this table. 47 | We cannot rely on the base table's generic method, due to the `season_id` field. 48 | """ 49 | params = self.build_params(season_id) 50 | 51 | # Encode without safe '+', apparently the NBA likes unsafe url params. 52 | params_str = urllib.parse.urlencode(params, safe=':+') 53 | 54 | # json response 55 | response = requests.get(url=self.url, headers=headers, params=params_str).json() 56 | 57 | result_sets = response['resultSets'][0] 58 | rowset = result_sets['rowSet'] 59 | 60 | column_names = column_names_from_table(self.settings.db, self.table._meta.table_name) 61 | 62 | column_mapping = get_rowset_mapping(result_sets, column_names) 63 | 64 | season_id_int = season_id_to_int(season_id) 65 | for row in rowset: 66 | new_row = {} 67 | for column_name, row_index in column_mapping.items(): 68 | # None here represents a column that exists in the DB object but 69 | # not as a row in the response. See: [#97] 70 | if row_index is None: 71 | new_row[column_name] = None 72 | else: 73 | new_row[column_name] = row[row_index] 74 | new_row['season_id'] = season_id_int 75 | self.rows.append(new_row) 76 | 77 | super().populate() 78 | 79 | def build_params(self, season_id): 80 | """ 81 | Create required parameters dict for the request. 82 | """ 83 | return { 84 | 'College': '', 85 | 'Conference': '', 86 | 'Country': '', 87 | 'DateFrom': '', 88 | 'DateTo': '', 89 | 'Division': '', 90 | 'DraftPick': '', 91 | 'DraftYear': '', 92 | 'GameScope': '', 93 | 'GameSegment': '', 94 | 'Height': '', 95 | 'LastNGames': '0', 96 | 'LeagueID': '00', 97 | 'Location': '', 98 | 'MeasureType': 'Base', 99 | 'Month': '0', 100 | 'OpponentTeamID': '0', 101 | 'Outcome': '', 102 | 'PORound': '0', 103 | 'PaceAdjust': 'N', 104 | 'PerMode': self.per_mode, 105 | 'Period': '0', 106 | 'PlayerExperience': '', 107 | 'PlayerPosition': '', 108 | 'PlusMinus': 'N', 109 | 'Rank': 'N', 110 | 'Season': season_id, 111 | 'SeasonSegment': '', 112 | 'SeasonType': 'Regular+Season', 113 | 'ShotClockRange': '', 114 | 'StarterBench': '', 115 | 'TeamID': '0', 116 | 'TwoWay': '0', 117 | 'VsConference': '', 118 | 'VsDivision': '', 119 | 'Weight': '' 120 | } 121 | -------------------------------------------------------------------------------- /stats/player_season.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | PlayerSeason requester and builder. 20 | """ 21 | 22 | import requests 23 | import urllib.parse 24 | 25 | from utils import get_rowset_mapping, column_names_from_table, season_id_to_int 26 | from models import PlayerSeason 27 | from general_requester import GenericRequester 28 | from constants import headers 29 | 30 | 31 | class PlayerSeasonRequester(GenericRequester): 32 | 33 | per_mode = 'Totals' 34 | player_info_url = 'http://stats.nba.com/stats/leaguedashplayerbiostats' 35 | 36 | def __init__(self, settings): 37 | """ 38 | Constructor. Attach settings internally and bind the model to the 39 | database. 40 | """ 41 | super().__init__(settings, self.player_info_url, PlayerSeason) 42 | 43 | def populate_season(self, season_id): 44 | """ 45 | Build GET REST request to the NBA for a season, iterate over the 46 | results, store in the database. 47 | We cannot rely on the base table's generic method, due to the `season_id` field. 48 | """ 49 | params = self.build_params(season_id) 50 | 51 | # Encode without safe '+', apparently the NBA likes unsafe url params. 52 | params_str = urllib.parse.urlencode(params, safe=':+') 53 | 54 | # json response 55 | response = requests.get(url=self.url, headers=headers, params=params_str).json() 56 | 57 | result_sets = response['resultSets'][0] 58 | rowset = result_sets['rowSet'] 59 | 60 | column_names = column_names_from_table(self.settings.db, self.table._meta.table_name) 61 | 62 | column_mapping = get_rowset_mapping(result_sets, column_names) 63 | 64 | season_id_int = season_id_to_int(season_id) 65 | for row in rowset: 66 | new_row = {column_name: row[row_index] for column_name, row_index in column_mapping.items()} 67 | new_row['season_id'] = season_id_int 68 | self.rows.append(new_row) 69 | 70 | super().populate() 71 | 72 | def build_params(self, season_id): 73 | """ 74 | Create required parameters dict for the request. 75 | """ 76 | return { 77 | 'College': '', 78 | 'Conference': '', 79 | 'Country': '', 80 | 'DateFrom': '', 81 | 'DateTo': '', 82 | 'Division': '', 83 | 'DraftPick': '', 84 | 'DraftYear': '', 85 | 'GameScope': '', 86 | 'GameSegment': '', 87 | 'Height': '', 88 | 'LastNGames': '0', 89 | 'LeagueID': '00', 90 | 'Location': '', 91 | 'Month': '0', 92 | 'OpponentTeamID': '0', 93 | 'Outcome': '', 94 | 'PORound': '0', 95 | 'PerMode': self.per_mode, 96 | 'Period': '0', 97 | 'PlayerExperience': '', 98 | 'PlayerPosition': '', 99 | 'Season': season_id, 100 | 'SeasonSegment': '', 101 | 'SeasonType': 'Regular+Season', 102 | 'ShotClockRange': '', 103 | 'StarterBench': '', 104 | 'TeamID': '0', 105 | 'VsConference': '', 106 | 'VsDivision': '', 107 | 'Weight': '' 108 | } 109 | -------------------------------------------------------------------------------- /stats/season.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Season builder. 20 | """ 21 | 22 | from models import Season 23 | 24 | from db_utils import insert_many_on_conflict_ignore 25 | from utils import season_id_to_int 26 | from peewee import fn 27 | 28 | 29 | class SeasonBuilder: 30 | 31 | def __init__(self, settings): 32 | self.settings = settings 33 | self.settings.db.bind([Season]) 34 | 35 | def create_ddl(self): 36 | """ 37 | Creates the season table. 38 | """ 39 | self.settings.db.create_tables([Season], safe=True) 40 | 41 | def populate(self, seasons): 42 | """ 43 | Populates the season table from the passed seasons. Ignores previous seasons 44 | that were already loaded. 45 | """ 46 | 47 | season_ints = list(map(lambda p: self.season_to_row(p), seasons)) 48 | insert_many_on_conflict_ignore(self.settings, Season, season_ints) 49 | 50 | def current_season_loaded(self): 51 | rows = self.settings.db.execute_sql("SELECT MAX(season_id) FROM season;").fetchall() 52 | season_id = rows[0][0] 53 | return season_id 54 | 55 | def season_to_row(self, season): 56 | season_id = season_id_to_int(season) 57 | return {'season_id': season_id} 58 | -------------------------------------------------------------------------------- /stats/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Class for user defined settings. 20 | """ 21 | 22 | from peewee import PostgresqlDatabase, MySQLDatabase, SqliteDatabase 23 | 24 | import os 25 | from dotenv import load_dotenv 26 | load_dotenv() 27 | 28 | """ 29 | Singleton settings instance. 30 | """ 31 | 32 | DB_NAME = os.getenv('DB_NAME') 33 | DB_HOST = os.getenv('DB_HOST') 34 | DB_USER = os.getenv('DB_USER') 35 | DB_PASSWORD = os.getenv('DB_PASSWORD') 36 | 37 | 38 | class Settings: 39 | 40 | def __init__(self, database_type, database_name, 41 | database_user, database_password, database_host, 42 | batch_size, sqlite_path, quiet): 43 | 44 | self.user_agent = ( 45 | "Mozilla/5.0 (X11; Linux x86_64) " 46 | "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82" 47 | "Safari/537.36" 48 | ) 49 | 50 | self.db_type = database_type 51 | 52 | name = DB_NAME 53 | user = DB_USER 54 | password = DB_PASSWORD 55 | host = DB_HOST 56 | self.batch_size = batch_size 57 | 58 | if database_name is not None: 59 | name = database_name 60 | if database_user is not None: 61 | user = database_user 62 | if database_password is not None: 63 | password = database_password 64 | if database_host is not None: 65 | host = database_host 66 | 67 | if database_type == "postgres": 68 | if not quiet: 69 | print("Connecting to postgres database.") 70 | self.db = PostgresqlDatabase( 71 | name, 72 | host=host, 73 | user=user, 74 | password=password 75 | ) 76 | elif database_type == "sqlite": 77 | if not quiet: 78 | print("Initializing sqlite database.") 79 | self.db = SqliteDatabase(sqlite_path, pragmas={'journal_mode': 'wal'}) 80 | else: 81 | if not quiet: 82 | print("Connecting to mysql database.") 83 | self.db = MySQLDatabase( 84 | name, 85 | host=host, 86 | user=user, 87 | password=password, 88 | charset='utf8mb4' 89 | ) 90 | -------------------------------------------------------------------------------- /stats/shot_chart_detail.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | ShotChartDetail builder and requester. 20 | """ 21 | 22 | import urllib.parse 23 | 24 | from models import ShotChartDetail, ShotChartDetailTemp 25 | from general_requester import GenericRequester 26 | from db_utils import insert_many 27 | 28 | 29 | class ShotChartDetailRequester(GenericRequester): 30 | 31 | shot_chart_detail_url = "https://stats.nba.com/stats/shotchartdetail" 32 | 33 | def __init__(self, settings): 34 | """ 35 | Constructor. Pass on all relevant vars. 36 | """ 37 | super().__init__(settings, self.shot_chart_detail_url, ShotChartDetail) 38 | # TODO: this conflicts with a fresh db. 39 | self.settings.db.bind([ShotChartDetailTemp]) 40 | self.settings.db.create_tables([ShotChartDetailTemp], safe=True) 41 | 42 | def create_ddl(self): 43 | """ 44 | Override method to setup temp table. 45 | """ 46 | super().create_ddl() 47 | 48 | def temp_table_except_predicate(self): 49 | """ 50 | This runs an EXCEPT between the temp table and the non-temp table to find 51 | the new games. 52 | 53 | This is a silly way to do this, but I wanted a quick copy/paste to unblock 54 | something else. 55 | """ 56 | regular_query = ShotChartDetail.select(ShotChartDetail.game_id) 57 | temp_query = ShotChartDetailTemp.select(ShotChartDetailTemp.game_id) 58 | 59 | expt = temp_query - regular_query 60 | 61 | return expt.select_from(expt.c.game_id) 62 | 63 | def finalize(self, filter_predicate): 64 | """ 65 | This function finishes loading shot_chart_detail by inserting all valid 66 | records from the temp table into the main table. The temp table is 67 | dropped at the end of the session. 68 | 69 | This accepts a predicate to define what game_ids to filter on. 70 | Examples of usage: to only add rows for existing games, or for the 71 | EXCEPT for the player_game_log regular and temp tables. 72 | """ 73 | print('Inserting from shot_chart_detail temp table into main table.') 74 | with self.settings.db.atomic(): 75 | (ShotChartDetail.insert_from( 76 | ShotChartDetailTemp.select( 77 | ShotChartDetailTemp.game_id, 78 | ShotChartDetailTemp.player_id, 79 | ShotChartDetailTemp.team_id, 80 | ShotChartDetailTemp.game_event_id, 81 | ShotChartDetailTemp.period, 82 | ShotChartDetailTemp.minutes_remaining, 83 | ShotChartDetailTemp.seconds_remaining, 84 | ShotChartDetailTemp.event_type, 85 | ShotChartDetailTemp.action_type, 86 | ShotChartDetailTemp.shot_type, 87 | ShotChartDetailTemp.shot_zone_basic, 88 | ShotChartDetailTemp.shot_zone_area, 89 | ShotChartDetailTemp.shot_zone_range, 90 | ShotChartDetailTemp.shot_distance, 91 | ShotChartDetailTemp.loc_x, 92 | ShotChartDetailTemp.loc_y, 93 | ShotChartDetailTemp.shot_attempted_flag, 94 | ShotChartDetailTemp.shot_made_flag, 95 | ShotChartDetailTemp.htm, 96 | ShotChartDetailTemp.vtm 97 | ).where(ShotChartDetailTemp.game_id.in_(filter_predicate)), 98 | # TODO: Cleaner way to specify all fields but one? 99 | fields=[ 100 | ShotChartDetail.game_id, 101 | ShotChartDetail.player_id, 102 | ShotChartDetail.team_id, 103 | ShotChartDetail.game_event_id, 104 | ShotChartDetail.period, 105 | ShotChartDetail.minutes_remaining, 106 | ShotChartDetail.seconds_remaining, 107 | ShotChartDetail.event_type, 108 | ShotChartDetail.action_type, 109 | ShotChartDetail.shot_type, 110 | ShotChartDetail.shot_zone_basic, 111 | ShotChartDetail.shot_zone_area, 112 | ShotChartDetail.shot_zone_range, 113 | ShotChartDetail.shot_distance, 114 | ShotChartDetail.loc_x, 115 | ShotChartDetail.loc_y, 116 | ShotChartDetail.shot_attempted_flag, 117 | ShotChartDetail.shot_made_flag, 118 | ShotChartDetail.htm, 119 | ShotChartDetail.vtm 120 | ] 121 | )).execute() 122 | 123 | print('Insert finished.') 124 | 125 | def generate_rows(self, team_id, player_id): 126 | """ 127 | Build GET REST request to the NBA for a season. 128 | """ 129 | params = self.build_params(team_id, player_id) 130 | 131 | # Encode without safe '+', apparently the NBA likes unsafe url params. 132 | params_str = urllib.parse.urlencode(params, safe=':+') 133 | super().generate_rows(params_str) 134 | 135 | def populate(self): 136 | """ 137 | Store collected rows. Custom implementation to clear out the rows beteen 138 | populations and insert into the staging table. 139 | """ 140 | insert_many(self.settings, ShotChartDetailTemp, self.rows) 141 | self.rows = [] 142 | 143 | def build_params(self, team_id, player_id): 144 | """ 145 | Create required parameters dict for the request. 146 | """ 147 | params = self.base_params() 148 | params['PlayerID'] = player_id 149 | params['TeamID'] = team_id 150 | return params 151 | 152 | def base_params(self): 153 | """ 154 | The base params map. 155 | """ 156 | return { 157 | 'AheadBehind': '', 158 | 'ClutchTime': '', 159 | 'ContextFilter': '', 160 | 'ContextMeasure': 'FGA', 161 | 'DateFrom': '', 162 | 'DateTo': '', 163 | 'EndPeriod': '', 164 | 'EndRange': '', 165 | 'GameID': '', 166 | 'GameSegment': '', 167 | 'LastNGames': '0', 168 | 'LeagueID': '00', 169 | 'Location': '', 170 | 'Month': '0', 171 | 'OpponentTeamID': '0', 172 | 'Outcome': '', 173 | 'Period': '0', 174 | 'PlayerID': '', 175 | 'PlayerPosition': '', 176 | 'PointDiff': '', 177 | 'Position': '', 178 | 'RangeType': '', 179 | 'RookieYear': '', 180 | 'Season': '', 181 | 'SeasonSegment': '', 182 | 'SeasonType': 'Regular+Season', 183 | 'StartPeriod': '', 184 | 'StartRange': '', 185 | 'TeamID': '', 186 | 'VsConference': '', 187 | 'VsDivision': '' 188 | } 189 | -------------------------------------------------------------------------------- /stats/team.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Team requester and builder. 20 | """ 21 | 22 | from models import Team 23 | from general_requester import GenericRequester 24 | 25 | 26 | class TeamRequester(GenericRequester): 27 | 28 | team_details_url = 'https://stats.nba.com/stats/teamdetails' 29 | 30 | def __init__(self, settings): 31 | """ 32 | Constructor. Pass on all relevant vars. 33 | """ 34 | super().__init__(settings, self.team_details_url, Team) 35 | 36 | def generate_rows(self, team_id): 37 | """ 38 | Build GET Request for the team id. 39 | """ 40 | params = {'TeamID': team_id} 41 | super().generate_rows(params) 42 | -------------------------------------------------------------------------------- /stats/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------ 3 | Copyright 2023 Matthew Pope 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | ------------------------------------------------------------------------------ 17 | 18 | 19 | Misc utilities. 20 | """ 21 | 22 | from datetime import datetime 23 | 24 | 25 | def season_id_to_int(season_id): 26 | """ 27 | Util to convert a season_id to an int. 28 | """ 29 | return int(season_id[:4]) 30 | 31 | 32 | def get_rowset_mapping(result_sets, column_names): 33 | """ 34 | Returns a list of mapped fields to the passed headers. 35 | """ 36 | 37 | headers = result_sets['headers'] 38 | mapped = {} 39 | for column in column_names: 40 | column_upper = column.upper() 41 | if column_upper not in headers: 42 | mapped[column] = None 43 | else: 44 | mapped[column] = headers.index(column.upper()) 45 | 46 | return mapped 47 | 48 | 49 | def column_names_from_table(db, table_name): 50 | """ 51 | Gets the column names from a db table. 52 | """ 53 | 54 | columns = db.get_columns(table_name) 55 | mapped = [column.name for column in columns] 56 | 57 | # season_id is our construct, and isn't returned by any NBA endpoint. 58 | if 'season_id' in mapped: 59 | mapped.remove('season_id') 60 | 61 | # Ignore autogenerated id columns. 62 | if 'id' in mapped: 63 | mapped.remove('id') 64 | 65 | return mapped 66 | 67 | 68 | def chunk_list(in_list, n): 69 | """ 70 | Chunk list into lists of length n. 71 | """ 72 | return [in_list[i:i + n] for i in range(0, len(in_list), n)] 73 | 74 | 75 | def progress_bar(iterable, prefix='', suffix='', decimals=1, length=100, fill='█', printEnd="\r", quiet=False): 76 | """ 77 | https://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console 78 | Call in a loop to create terminal progress bar 79 | @params: 80 | iteration - Required : current iteration (Int) 81 | total - Required : total iterations (Int) 82 | prefix - Optional : prefix string (Str) 83 | suffix - Optional : suffix string (Str) 84 | decimals - Optional : number of decimals in percent complete (Int) 85 | length - Optional : character length of bar (Int) 86 | fill - Optional : bar fill character (Str) 87 | printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) 88 | quiet - Optoinal : should do printing. (Bool) 89 | """ 90 | total = 1 91 | if iterable: 92 | total = len(iterable) 93 | 94 | # Progress Bar Printing Function 95 | def printProgressBar(iteration): 96 | percent = ( 97 | ("{0:." + str(decimals) + "f}") 98 | .format(100 * (iteration / float(total))) 99 | ) 100 | filledLength = int(length * iteration // total) 101 | bar = fill * filledLength + '-' * (length - filledLength) 102 | if not quiet: 103 | print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=printEnd) 104 | # Initial Call 105 | printProgressBar(0) 106 | # Update Progress Bar 107 | for i, item in enumerate(iterable): 108 | yield item 109 | printProgressBar(i + 1) 110 | # Print New Line on Complete 111 | if not quiet: 112 | print() 113 | 114 | 115 | def generate_valid_seasons(): 116 | """ 117 | Genrate the valid seasons to choose from, starting at 1997 to the current season. 118 | This assumes that a season begins in October. 119 | """ 120 | 121 | valid_seasons = [] 122 | if datetime.now().month > 10: 123 | curr_season = datetime.now().year - 2000 + 1 124 | else: 125 | curr_season = datetime.now().year - 2000 126 | for x in range(1997, curr_season + 2000): 127 | valid_season = generate_valid_season(x) 128 | valid_seasons.append(valid_season) 129 | 130 | return valid_seasons 131 | 132 | 133 | def generate_valid_season(season): 134 | # Create string, reverse 135 | tmp = f"{(season % 100 + 1):02d}"[::-1] 136 | 137 | # Get last two vals of the last string, due to 1999's next season being 100 138 | next_year_rev = tmp[0:2] 139 | 140 | # re-reverse. 141 | next_year = next_year_rev[::-1] 142 | return f"{season}-{next_year}" 143 | --------------------------------------------------------------------------------