├── tests ├── __init__.py ├── scrape_game_tests.py ├── exception_decor_tests.py ├── visualize_game_tests.py ├── conftest.py ├── scrape_setup_tests.py ├── manipulate_tests.py └── test_schedule.py ├── scrapenhl2 ├── plot │ ├── app │ │ ├── team_page.py │ │ ├── __init__.py │ │ └── game_page.py │ ├── __init__.py │ ├── label_lines.py │ ├── team_lineup_cf.py │ ├── rolling_boxcars.py │ ├── team_score_state_toi.py │ ├── rolling_cf_gf.py │ └── team_score_shot_rate.py ├── __init__.py ├── manipulate │ └── __init__.py ├── scrape │ ├── __init__.py │ ├── check_game_data.py │ ├── events.py │ ├── games.py │ ├── organization.py │ ├── manipulate_schedules.py │ ├── autoupdate.py │ ├── scrape_pbp.py │ └── scrape_toi.py └── twitterbot │ └── instructions.txt ├── docs ├── build │ ├── _static │ │ ├── custom.css │ │ ├── up.png │ │ ├── down.png │ │ ├── file.png │ │ ├── minus.png │ │ ├── plus.png │ │ ├── comment.png │ │ ├── up-pressed.png │ │ ├── ajax-loader.gif │ │ ├── down-pressed.png │ │ ├── comment-bright.png │ │ ├── comment-close.png │ │ ├── pygments.css │ │ └── doctools.js │ ├── html │ │ ├── _static │ │ │ ├── custom.css │ │ │ ├── up.png │ │ │ ├── down.png │ │ │ ├── file.png │ │ │ ├── minus.png │ │ │ ├── plus.png │ │ │ ├── comment.png │ │ │ ├── up-pressed.png │ │ │ ├── ajax-loader.gif │ │ │ ├── down-pressed.png │ │ │ ├── comment-bright.png │ │ │ ├── comment-close.png │ │ │ ├── pygments.css │ │ │ └── doctools.js │ │ ├── objects.inv │ │ ├── _sources │ │ │ ├── modules.rst.txt │ │ │ ├── scrape_game.rst.txt │ │ │ ├── scrape_setup.rst.txt │ │ │ ├── scrapenhl2.rst.txt │ │ │ └── index.rst.txt │ │ ├── .buildinfo │ │ ├── searchindex.js │ │ ├── genindex.html │ │ ├── scrape_game.html │ │ ├── scrape_setup.html │ │ ├── search.html │ │ ├── modules.html │ │ ├── scrapenhl2.html │ │ └── index.html │ ├── objects.inv │ ├── doctrees │ │ ├── index.doctree │ │ ├── modules.doctree │ │ ├── environment.pickle │ │ ├── scrapenhl2.doctree │ │ ├── scrape_game.doctree │ │ └── scrape_setup.doctree │ ├── .doctrees │ │ ├── index.doctree │ │ ├── modules.doctree │ │ ├── environment.pickle │ │ ├── scrape_game.doctree │ │ ├── scrapenhl2.doctree │ │ └── scrape_setup.doctree │ ├── _sources │ │ ├── modules.rst.txt │ │ ├── scrape_game.rst.txt │ │ ├── scrape_setup.rst.txt │ │ ├── scrapenhl2.rst.txt │ │ └── index.rst.txt │ ├── .buildinfo │ ├── search.html │ ├── scrapenhl2.html │ ├── modules.html │ ├── scrape_game.html │ └── scrape_setup.html ├── source │ ├── _static │ │ ├── WSH-TOR_G6.png │ │ ├── example_h2h.png │ │ ├── Caps_d_pairs.png │ │ ├── Caps_lineup_cf.png │ │ ├── Oshie_boxcars.png │ │ ├── example_timeline.png │ │ ├── Ovechkin_rolling_cf.png │ │ ├── Score_states_2015.png │ │ ├── WSH-TOR_G6_timeline.png │ │ ├── game_page_screenshot.png │ │ ├── Caps_shot_score_parallel.png │ │ └── Caps_shot_rates_score_scatter.png │ ├── index.rst │ ├── manipulate.rst │ ├── support.rst │ ├── scrape.rst │ ├── plot.rst │ └── conf.py ├── Makefile └── make.bat ├── _static ├── WSH-TOR_G6.png ├── Ovechkin_rolling_cf.png └── WSH-TOR_G6_timeline.png ├── .travis.yml ├── pytest.ini ├── Makefile ├── requirements.txt ├── MANIFEST.in ├── scripts └── update_all.py ├── .idea └── runConfigurations │ └── Sphinx.xml ├── LICENSE.txt ├── .gitignore ├── setup.py └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /tests/scrape_game_tests.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapenhl2/plot/app/team_page.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/exception_decor_tests.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/visualize_game_tests.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/build/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* This file intentionally left blank. */ 2 | -------------------------------------------------------------------------------- /docs/build/html/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* This file intentionally left blank. */ 2 | -------------------------------------------------------------------------------- /_static/WSH-TOR_G6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/_static/WSH-TOR_G6.png -------------------------------------------------------------------------------- /docs/build/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/objects.inv -------------------------------------------------------------------------------- /scrapenhl2/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["scrape", 2 | "manipulate", 3 | "plot"] 4 | -------------------------------------------------------------------------------- /docs/build/_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/up.png -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import pytest 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/build/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/down.png -------------------------------------------------------------------------------- /docs/build/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/file.png -------------------------------------------------------------------------------- /docs/build/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/minus.png -------------------------------------------------------------------------------- /docs/build/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/plus.png -------------------------------------------------------------------------------- /docs/build/html/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/objects.inv -------------------------------------------------------------------------------- /scrapenhl2/plot/app/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['game_page', 2 | 'player_page', 3 | 'team_page'] -------------------------------------------------------------------------------- /docs/build/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/comment.png -------------------------------------------------------------------------------- /docs/build/html/_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/up.png -------------------------------------------------------------------------------- /scrapenhl2/manipulate/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['manipulate', 2 | 'add_onice_players', 3 | 'combos'] -------------------------------------------------------------------------------- /_static/Ovechkin_rolling_cf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/_static/Ovechkin_rolling_cf.png -------------------------------------------------------------------------------- /_static/WSH-TOR_G6_timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/_static/WSH-TOR_G6_timeline.png -------------------------------------------------------------------------------- /docs/build/_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/up-pressed.png -------------------------------------------------------------------------------- /docs/build/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/doctrees/index.doctree -------------------------------------------------------------------------------- /docs/build/html/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/down.png -------------------------------------------------------------------------------- /docs/build/html/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/file.png -------------------------------------------------------------------------------- /docs/build/html/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/minus.png -------------------------------------------------------------------------------- /docs/build/html/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/plus.png -------------------------------------------------------------------------------- /docs/build/.doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/.doctrees/index.doctree -------------------------------------------------------------------------------- /docs/build/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/build/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/build/doctrees/modules.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/doctrees/modules.doctree -------------------------------------------------------------------------------- /docs/build/html/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/comment.png -------------------------------------------------------------------------------- /docs/source/_static/WSH-TOR_G6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/WSH-TOR_G6.png -------------------------------------------------------------------------------- /docs/source/_static/example_h2h.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/example_h2h.png -------------------------------------------------------------------------------- /docs/build/.doctrees/modules.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/.doctrees/modules.doctree -------------------------------------------------------------------------------- /docs/build/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/build/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/_static/comment-close.png -------------------------------------------------------------------------------- /docs/build/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/build/doctrees/scrapenhl2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/doctrees/scrapenhl2.doctree -------------------------------------------------------------------------------- /docs/build/html/_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/up-pressed.png -------------------------------------------------------------------------------- /docs/source/_static/Caps_d_pairs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Caps_d_pairs.png -------------------------------------------------------------------------------- /docs/source/_static/Caps_lineup_cf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Caps_lineup_cf.png -------------------------------------------------------------------------------- /docs/source/_static/Oshie_boxcars.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Oshie_boxcars.png -------------------------------------------------------------------------------- /docs/build/.doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/.doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/build/.doctrees/scrape_game.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/.doctrees/scrape_game.doctree -------------------------------------------------------------------------------- /docs/build/.doctrees/scrapenhl2.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/.doctrees/scrapenhl2.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/scrape_game.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/doctrees/scrape_game.doctree -------------------------------------------------------------------------------- /docs/build/doctrees/scrape_setup.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/doctrees/scrape_setup.doctree -------------------------------------------------------------------------------- /docs/build/html/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/build/html/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/source/_static/example_timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/example_timeline.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.6" 4 | install: pip install -r requirements.txt 5 | 6 | script: python -m unittest 7 | -------------------------------------------------------------------------------- /docs/build/.doctrees/scrape_setup.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/.doctrees/scrape_setup.doctree -------------------------------------------------------------------------------- /docs/build/html/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/build/html/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/build/html/_static/comment-close.png -------------------------------------------------------------------------------- /docs/source/_static/Ovechkin_rolling_cf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Ovechkin_rolling_cf.png -------------------------------------------------------------------------------- /docs/source/_static/Score_states_2015.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Score_states_2015.png -------------------------------------------------------------------------------- /docs/source/_static/WSH-TOR_G6_timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/WSH-TOR_G6_timeline.png -------------------------------------------------------------------------------- /docs/build/_sources/modules.rst.txt: -------------------------------------------------------------------------------- 1 | scrape 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | scrape_game 8 | scrape_setup 9 | -------------------------------------------------------------------------------- /docs/build/html/_sources/modules.rst.txt: -------------------------------------------------------------------------------- 1 | scrape 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | scrape_game 8 | scrape_setup 9 | -------------------------------------------------------------------------------- /docs/source/_static/game_page_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/game_page_screenshot.png -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts=--junit-xml=results.xml --cov=scrapenhl2/scrape --cov-report term-missing --cov-report=xml:coverage.xml 3 | 4 | -------------------------------------------------------------------------------- /docs/source/_static/Caps_shot_score_parallel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Caps_shot_score_parallel.png -------------------------------------------------------------------------------- /docs/source/_static/Caps_shot_rates_score_scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muneebalam/scrapenhl2/HEAD/docs/source/_static/Caps_shot_rates_score_scatter.png -------------------------------------------------------------------------------- /docs/build/_sources/scrape_game.rst.txt: -------------------------------------------------------------------------------- 1 | scrape\_game module 2 | =================== 3 | 4 | .. automodule:: scrape_game 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/build/html/_sources/scrape_game.rst.txt: -------------------------------------------------------------------------------- 1 | scrape\_game module 2 | =================== 3 | 4 | .. automodule:: scrape_game 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/build/_sources/scrape_setup.rst.txt: -------------------------------------------------------------------------------- 1 | scrape\_setup module 2 | ==================== 3 | 4 | .. automodule:: scrape_setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/build/html/_sources/scrape_setup.rst.txt: -------------------------------------------------------------------------------- 1 | scrape\_setup module 2 | ==================== 3 | 4 | .. automodule:: scrape_setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/build/_sources/scrapenhl2.rst.txt: -------------------------------------------------------------------------------- 1 | scrapenhl2 package 2 | ================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: scrapenhl2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/html/_sources/scrapenhl2.rst.txt: -------------------------------------------------------------------------------- 1 | scrapenhl2 package 2 | ================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: scrapenhl2 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/build/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 690a2af23337d56ded0929534f7e30c6 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/build/html/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 4385351a0af84f7282472cd38b3768ae 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYTHON=python3 2 | MODULE_NAME=`$(PYTHON) -c "import constants as c; print(c.module_name)"` 3 | VERSION=`python setup.py --version` 4 | 5 | setup: 6 | pip install -e . 7 | 8 | clean: 9 | @rm -rf *.egg-info *.pyc __pycache__ 10 | @find . -regex "\(.*__pycache__.*\|*.py[co]\)" -delete 11 | 12 | test: 13 | py.test 14 | 15 | -------------------------------------------------------------------------------- /scrapenhl2/plot/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['game_h2h', 2 | 'game_timeline', 3 | 'visualization_helper', 4 | 'rolling_cf_gf', 5 | 'usage', 6 | 'team_score_state_toi', 7 | 'team_lineup_cf', 8 | 'rolling_boxcars', 9 | 'label_lines', 10 | 'defense_pairs', 11 | 'forward_trios', 12 | 'team_score_shot_rate'] 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy 3 | matplotlib 4 | seaborn 5 | pandas 6 | pyarrow 7 | feather-format 8 | halo 9 | scikit-learn 10 | flask 11 | python-Levenshtein 12 | fuzzywuzzy 13 | beautifulsoup4 14 | html-table-extractor 15 | plotly 16 | tqdm 17 | sphinx 18 | requests 19 | arrow 20 | dash 21 | dash-renderer 22 | dash-html-components 23 | dash-core-components 24 | pytest 25 | pytest-cov 26 | pytest-mock 27 | numba 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | include README.rst 3 | include tests/*.py 4 | include scrapenhl2/scrape 5 | include scrapenhl2/manipulate 6 | include scrapenhl2/plot 7 | include scrapenhl2/scrape/* 8 | include scrapenhl2/manipulate/* 9 | include scrapenhl2/plot/* 10 | include scrapenhl2/docs/source/* 11 | include scrapenhl2/docs/Makefile 12 | include scrapenhl2/docs/make.bat 13 | include scrapenhl2/scrape/data/other/PLAYER_INFO.feather 14 | include scrapenhl2/scrape/data/other/TEAM_INFO.feather 15 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Indicates to import all .py files 3 | """ 4 | __all__ = ['autoupdate', 5 | 'check_game_data', 6 | 'events', 7 | 'games', 8 | 'general_helpers', 9 | 'manipulate_schedules', 10 | 'organization', 11 | 'parse_pbp', 12 | 'parse_toi', 13 | 'players', 14 | 'schedules', 15 | 'scrape_pbp', 16 | 'scrape_toi', 17 | 'team_info', 18 | 'teams'] 19 | -------------------------------------------------------------------------------- /scripts/update_all.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | from scrapenhl2.scrape import autoupdate 6 | 7 | 8 | if __name__ == '__main__': 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument("-s", "--season", type=int, default=None) 11 | arguments = parser.parse_args() 12 | 13 | if arguments.season is not None and 2017 < arguments.season < 2005: 14 | print("Invalid season") 15 | 16 | autoupdate.autoupdate(season=arguments.season) 17 | 18 | -------------------------------------------------------------------------------- /scrapenhl2/twitterbot/instructions.txt: -------------------------------------------------------------------------------- 1 | Game charts 2 | ------------ 3 | Most recent game this season: @h2hbot WSH PIT 4 | Specific game: @h2hbot 2017 20001 5 | Playoff game: @h2hbot 2016 TOR WSH game 6 6 | 7 | Player Corsi charts 8 | -------------------- 9 | @h2hbot Alex Ovechkin cf 10 | 11 | Team abbreviations 12 | ------------------ 13 | Metropolitan: CAR CBJ NJD NYI NYR PHI PIT WSH 14 | Atlantic: BOS BUF DET FLA MTL OTT TBL TOR 15 | Central: CHI COL DAL MIN NSH STL WPG 16 | Pacific: ANA ARI CGY EDM LAK SJS VAN VGK 17 | 18 | -------------------------------------------------------------------------------- /docs/build/html/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. scrapenhl2 documentation master file, created by 2 | sphinx-quickstart on Sun Oct 1 17:47:07 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to scrapenhl2's documentation! 7 | ====================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/build/html/searchindex.js: -------------------------------------------------------------------------------- 1 | Search.setIndex({docnames:["index","modules","scrape_game","scrape_setup","scrapenhl2"],envversion:51,filenames:["index.rst","modules.rst","scrape_game.rst","scrape_setup.rst","scrapenhl2.rst"],objects:{},objnames:{},objtypes:{},terms:{content:[],index:0,modul:[0,1],packag:[],page:0,scrape_gam:1,scrape_setup:1,search:0},titles:["Welcome to scrapenhl2’s documentation!","scrape","scrape_game module","scrape_setup module","scrapenhl2 package"],titleterms:{content:4,document:0,indic:0,modul:[2,3,4],packag:4,scrape:1,scrape_gam:2,scrape_setup:3,scrapenhl2:[0,4],tabl:0,welcom:0}}) -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. scrapenhl2 documentation master file, created by Muneeb Alam 2 | 3 | Welcome to scrapenhl2's documentation! 4 | ====================================== 5 | 6 | Table of Contents 7 | ----------------- 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | 12 | Scraping 13 | Manipulating 14 | Plotting 15 | Support 16 | 17 | .. include:: ../../README.rst 18 | :start-after: inclusion-marker-for-sphinx 19 | 20 | 21 | Indices and tables 22 | ------------------ 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | -------------------------------------------------------------------------------- /docs/build/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. scrapenhl2 documentation master file, created by Muneeb Alam 2 | 3 | Welcome to scrapenhl2's documentation! 4 | ====================================== 5 | 6 | Table of Contents 7 | ------------------ 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | 12 | Scraping 13 | Manipulating 14 | Plotting 15 | Support 16 | 17 | .. include:: ../../README.rst 18 | :start-after: inclusion-marker-for-sphinx 19 | 20 | 21 | Indices and tables 22 | ------------------ 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = scrapenhl2 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=scrapenhl2 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /.idea/runConfigurations/Sphinx.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 19 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2017 Muneeb Alam 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /docs/source/manipulate.rst: -------------------------------------------------------------------------------- 1 | .. _manipulate: 2 | 3 | Manipulate 4 | ========== 5 | 6 | The scrapenhl2.manipulate module contains methods useful for scraping. 7 | 8 | Useful examples 9 | --------------- 10 | 11 | Add on-ice players to a file:: 12 | 13 | from scrapenhl.manipulate import add_onice_players as onice 14 | onice.add_players_to_file('/Users/muneebalam/Downloads/zone_entries.csv', 'WSH', time_format='elapsed') 15 | # Will output zone_entries_on-ice.csv in Downloads, with WSH players and opp players on-ice listed. 16 | 17 | :ref:`See documentation below ` for more information and additional arguments to add_players_to_file. 18 | 19 | Methods 20 | ------- 21 | 22 | General 23 | ~~~~~~~ 24 | 25 | .. automodule:: scrapenhl2.manipulate.manipulate 26 | :members: 27 | 28 | .. _addoniceplayers: 29 | 30 | Add on-ice players 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | .. automodule:: scrapenhl2.manipulate.add_onice_players 34 | :members: 35 | 36 | TOI and Corsi for combinations of players 37 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 38 | 39 | .. automodule:: scrapenhl2.manipulate.combos 40 | :members: 41 | -------------------------------------------------------------------------------- /docs/source/support.rst: -------------------------------------------------------------------------------- 1 | .. _support: 2 | 3 | Support 4 | ======= 5 | 6 | Feel free to contact me with questions or suggestions. 7 | 8 | Docs 9 | ---- 10 | `Read the Docs `_ 11 | 12 | Github 13 | ------ 14 | `Link `_. 15 | 16 | Contact 17 | ------- 18 | `Twitter 19 | `_, or 20 | `create an issue on GitHub `_. 21 | 22 | Collaboration 23 | ------------- 24 | 25 | I'm happy to partner with you in development efforts--just shoot me a message, or just get started by resolving a 26 | GitHub issue or suggested enhancement. 27 | Please also let me know if you'd like to alpha- or beta-test my code. 28 | 29 | Donations 30 | --------- 31 | If you would like to support my work, please donate money to a charity of your choice. Many large charities do 32 | great work all around the world (e.g. Médecins Sans Frontières), 33 | but don't forget that your support is often more critical for local/small charities. 34 | Also consider that small regular donations are sometimes better than one large donation. 35 | 36 | You can vet a charity you're targeting using a `charity rating website `_. 37 | 38 | If you do make a donation, make me happy `and leave a record here `_.. 39 | (It's anonymous.) -------------------------------------------------------------------------------- /tests/scrape_setup_tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | 5 | import scrapenhl2.scrape.scrape_setup as ss 6 | 7 | 8 | class SSTest_check_types(unittest.TestCase): 9 | """Tests for scrape_setup.check_types()""" 10 | 11 | def test_int(self): 12 | self.assertTrue(ss.check_types(8471214)) 13 | 14 | def test_str(self): 15 | self.assertTrue(ss.check_types('8471214')) 16 | 17 | def test_float(self): 18 | self.assertTrue(ss.check_types(8471214.0)) 19 | 20 | def test_npint64(self): 21 | self.assertTrue(ss.check_types(np.int64(8471214))) 22 | 23 | def test_npint32(self): 24 | self.assertTrue(ss.check_types(np.int32(8471214))) 25 | 26 | def test_list(self): 27 | self.assertFalse(ss.check_types([1, 2, 3])) 28 | 29 | 30 | class SSTest_infer_season_from_date(unittest.TestCase): 31 | """Tests for scrape_setup.infer_season_from_date()""" 32 | 33 | def test_jan(self): 34 | self.assertEquals(ss.infer_season_from_date('2017-01-01'), 2016) 35 | 36 | def test_jun(self): 37 | self.assertEquals(ss.infer_season_from_date('2017-06-01'), 2016) 38 | 39 | def test_sep(self): 40 | self.assertEquals(ss.infer_season_from_date('2017-08-01'), 2016) 41 | 42 | def test_dec(self): 43 | self.assertEquals(ss.infer_season_from_date('2017-12-01'), 2017) 44 | 45 | 46 | if __name__ == '__main__': 47 | unittest.main() 48 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/check_game_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | The purpose of this module is to check game data for integrity (e.g. TOI has at least 3600 rows). 3 | """ 4 | 5 | from scrapenhl2.scrape import parse_toi, parse_pbp, autoupdate, schedules, teams 6 | 7 | def check_game_pbp(season=None): 8 | """ 9 | Rescrapes gone-final games if they do not pass the following checks: 10 | - (TODO) 11 | 12 | :param season: int, the season 13 | 14 | :return: 15 | """ 16 | 17 | 18 | def check_game_toi(season=None): 19 | """ 20 | Rescrapes gone-final games if they do not pass the following checks: 21 | - (TODO) 22 | 23 | :param season: int, the season 24 | 25 | :return: 26 | """ 27 | if season is None: 28 | season = schedules.get_current_season() 29 | 30 | sch = schedules.get_season_schedule(season) 31 | finals = sch.query('Status == "Final" & TOIStatus == "Scraped" & Game >= 20001 & Game <= 30417').Game.values 32 | 33 | games_to_rescrape = [] 34 | 35 | for game in finals: 36 | try: 37 | toi = parse_toi.get_parsed_toi(season, game) 38 | 39 | assert len(toi) >= 3595 # At least 3600 seconds in game, approx 40 | 41 | # TODO add other checks 42 | 43 | except AssertionError as ae: 44 | print(ae, ae.args, len(toi)) 45 | 46 | games_to_rescrape.append(game) 47 | except IOError: 48 | games_to_rescrape.append(game) 49 | 50 | if len(games_to_rescrape) > 0: 51 | autoupdate.read_final_games(games_to_rescrape, season) 52 | teams.update_team_logs(season, force_games=games_to_rescrape) 53 | 54 | 55 | def check_team_toi(season=None): 56 | """ 57 | 58 | :param season: 59 | :return: 60 | """ 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Repo specific stuff 2 | scrapenhl2/data 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Local files 12 | *.db 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | results.xml 52 | *,cover 53 | .hypothesis/ 54 | *.prof 55 | *.lprof 56 | results.xml 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # dotenv 89 | .env 90 | 91 | # virtualenv 92 | .venv/ 93 | venv/ 94 | ENV/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | /scrapenhl2/plot/app/testing.ipynb 102 | 103 | # csv files 104 | *.csv 105 | /examples/example.py 106 | 107 | # passwords 108 | /scrapenhl2/twitterbot/auth.py 109 | -------------------------------------------------------------------------------- /docs/source/scrape.rst: -------------------------------------------------------------------------------- 1 | .. _scrape: 2 | 3 | Scrape 4 | ======= 5 | 6 | The scrapenhl2.scrape module contains methods useful for scraping. 7 | 8 | Useful examples 9 | ---------------- 10 | 11 | Updating data:: 12 | 13 | from scrapenhl2.scrape import autoupdate 14 | autoupdate.autoupdate() 15 | 16 | Get the season schedule:: 17 | 18 | from scrapenhl2.scrape import schedules 19 | schedules.get_season_schedule(2017) 20 | 21 | Convert between player ID and player name:: 22 | 23 | from scrapenhl2.scrape import players 24 | pname = 'Alex Ovechkin' 25 | players.player_as_id(pname) 26 | 27 | pid = 8471214 28 | players.player_as_str(pid) 29 | 30 | There's much more, and feel free to submit pull requests with whatever you find useful. 31 | 32 | Methods 33 | -------- 34 | 35 | The functions in these modules are organized pretty logically under the module names. 36 | 37 | Autoupdate 38 | ~~~~~~~~~~~ 39 | .. automodule:: scrapenhl2.scrape.autoupdate 40 | :members: 41 | 42 | Events 43 | ~~~~~~~ 44 | .. automodule:: scrapenhl2.scrape.events 45 | :members: 46 | 47 | Games 48 | ~~~~~~ 49 | .. automodule:: scrapenhl2.scrape.games 50 | :members: 51 | 52 | General helpers 53 | ~~~~~~~~~~~~~~~~ 54 | .. automodule:: scrapenhl2.scrape.general_helpers 55 | :members: 56 | 57 | Organization 58 | ~~~~~~~~~~~~~ 59 | .. automodule:: scrapenhl2.scrape.organization 60 | :members: 61 | 62 | Players 63 | ~~~~~~~~ 64 | .. automodule:: scrapenhl2.scrape.players 65 | :members: 66 | 67 | Schedules 68 | ~~~~~~~~~~ 69 | .. automodule:: scrapenhl2.scrape.schedules 70 | :members: 71 | 72 | Manipulate schedules 73 | ~~~~~~~~~~~~~~~~~~~~~ 74 | .. automodule:: scrapenhl2.scrape.manipulate_schedules 75 | :members: 76 | 77 | Scrape play by play 78 | ~~~~~~~~~~~~~~~~~~~~ 79 | .. automodule:: scrapenhl2.scrape.scrape_pbp 80 | :members: 81 | 82 | Parse play by play 83 | ~~~~~~~~~~~~~~~~~~~ 84 | .. automodule:: scrapenhl2.scrape.parse_pbp 85 | :members: 86 | 87 | Scrape TOI 88 | ~~~~~~~~~~~~ 89 | .. automodule:: scrapenhl2.scrape.scrape_toi 90 | :members: 91 | 92 | Parse TOI 93 | ~~~~~~~~~~~ 94 | .. automodule:: scrapenhl2.scrape.parse_toi 95 | :members: 96 | 97 | Team information 98 | ~~~~~~~~~~~~~~~~~ 99 | .. automodule:: scrapenhl2.scrape.team_info 100 | :members: 101 | 102 | Teams 103 | ~~~~~ 104 | .. automodule:: scrapenhl2.scrape.teams 105 | :members: 106 | 107 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/events.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods related to PBP events. 3 | """ 4 | 5 | import functools 6 | 7 | 8 | def convert_event(event): 9 | """ 10 | Converts to a more convenient, standardized name (see get_event_dictionary) 11 | 12 | :param event: str, the event name 13 | 14 | :return: str, shortened event name 15 | """ 16 | return get_event_dictionary()[event] # TODO the event dictionary is missing some 17 | 18 | 19 | def _get_event_dictionary(): 20 | """ 21 | Runs at startup to get a mapping of event name abbreviations to long versions. 22 | 23 | :return: a dictionary mapping, e.g., 'fo' to 'faceoff'. All lowercase. 24 | """ 25 | return {'fac': 'faceoff', 'faceoff': 'faceoff', 26 | 'shot': 'shot', 'sog': 'shot', 'save': 'shot', 27 | 'hit': 'hit', 28 | 'stop': 'stoppage', 'stoppage': 'stoppage', 29 | 'block': 'blocked shot', 'blocked shot': 'blocked shot', 30 | 'miss': 'missed shot', 'missed shot': 'missed shot', 31 | 'giveaway': 'giveaway', 'give': 'giveaway', 32 | 'takeaway': 'take', 'take': 'takeaway', 33 | 'penl': 'penalty', 'penalty': 'penalty', 34 | 'goal': 'goal', 35 | 'period end': 'period end', 36 | 'period official': 'period official', 37 | 'period ready': 'period ready', 38 | 'period start': 'period start', 39 | 'game scheduled': 'game scheduled', 40 | 'gend': 'game end', 41 | 'game end': 'game end', 42 | 'shootout complete': 'shootout complete', 43 | 'chal': 'official challenge', 'official challenge': 'official challenge'} 44 | 45 | 46 | def get_event_dictionary(): 47 | """ 48 | Returns the abbreviation: long name event mapping (in lowercase) 49 | 50 | :return: dict of str:str 51 | """ 52 | return _EVENT_DICT 53 | 54 | 55 | @functools.lru_cache(maxsize=10, typed=False) 56 | def get_event_longname(eventname): 57 | """ 58 | A method for translating event abbreviations to full names (for pbp matching) 59 | 60 | :param eventname: str, the event name 61 | 62 | :return: the non-abbreviated event name 63 | """ 64 | return get_event_dictionary()[eventname] 65 | 66 | 67 | def event_setup(): 68 | """ 69 | Loads event dictionary into memory 70 | 71 | :return: nothing 72 | """ 73 | global _EVENT_DICT 74 | _EVENT_DICT = _get_event_dictionary() 75 | 76 | 77 | event_setup() 78 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import setuptools 4 | 5 | 6 | # Utility function to read the README file. 7 | # Used for the long_description. It's nice, because now 1) we have a top level 8 | # README file and 2) it's easier to type in the README file than to put a raw 9 | # string in below ... 10 | def read(fname): 11 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 12 | 13 | setuptools.setup( 14 | name="scrapenhl2", 15 | version="0.4.1", 16 | author="Muneeb Alam", 17 | author_email="muneeb.alam@gmail.com", 18 | description=("scrapenhl2 is a python package for scraping and manipulating NHL data pulled from the NHL website."), 19 | license="MIT", 20 | keywords="nhl", 21 | url="https://github.com/muneebalam/scrapenhl2", 22 | packages=setuptools.find_packages(), 23 | install_requires=['numpy', # used by pandas 24 | 'scipy', # not currently used, but may be used for distribution fitting 25 | 'matplotlib', # graphing 26 | 'seaborn', # graphing; a little nicer than MPL 27 | 'pandas', # for handling and manipulating data 28 | 'pyarrow', # used by feather 29 | 'feather-format', # fast read-write format that plays nicely with R 30 | 'halo', # for spinners 31 | 'scikit-learn', # not currently used, but will be for machine learning 32 | 'flask', # for front end 33 | 'python-Levenshtein', # for fast fuzzy matching 34 | 'fuzzywuzzy', # for fuzzy string matching 35 | 'beautifulsoup4==4.5.3', # for html parsing 36 | 'html-table-extractor', # for html parsing 37 | 'plotly', # for interactive charts 38 | 'tqdm', # CLI progress bar 39 | 'tables', 40 | 'dash', 41 | 'dash-renderer', 42 | 'dash-html-components', 43 | 'dash-core-components', 44 | 'sphinx', 45 | 'requests', # Urllib for humans 46 | 'arrow', # Datetime for humans 47 | 'pytest', # Testrunner 48 | 'pytest-cov', # Coverage reports 49 | 'pytest-mock', # Test mocking framework 50 | 'numba' 51 | ], 52 | long_description=read('README.rst'), 53 | classifiers=[ 54 | "Development Status :: 3 - Alpha", 55 | 'Intended Audience :: Science/Research', 56 | "License :: OSI Approved :: MIT License", 57 | 'Programming Language :: Python :: 3' 58 | ], 59 | 60 | ) 61 | -------------------------------------------------------------------------------- /scrapenhl2/plot/label_lines.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is from `SO `_. 3 | It adds labels for lines on the lines themselves. 4 | """ 5 | 6 | from math import atan2,degrees 7 | import numpy as np 8 | 9 | def labelLine(line, x, label=None, align=True, **kwargs): 10 | """Labels line with line2D label data""" 11 | 12 | ax = line.get_axes() 13 | xdata = line.get_xdata() 14 | ydata = line.get_ydata() 15 | 16 | if (x < xdata[0]) or (x > xdata[-1]): 17 | print('x label location is outside data range!') 18 | return 19 | 20 | # Find corresponding y co-ordinate and angle of the line 21 | ip = 1 22 | for i in range(len(xdata)): 23 | if x < xdata[i]: 24 | ip = i 25 | break 26 | 27 | y = ydata[ip - 1] + (ydata[ip] - ydata[ip - 1]) * (x - xdata[ip - 1]) / (xdata[ip] - xdata[ip - 1]) 28 | 29 | if not label: 30 | label = line.get_label() 31 | 32 | if align: 33 | # Compute the slope 34 | dx = xdata[ip] - xdata[ip - 1] 35 | dy = ydata[ip] - ydata[ip - 1] 36 | ang = degrees(atan2(dy, dx)) 37 | 38 | # Transform to screen co-ordinates 39 | pt = np.array([x, y]).reshape((1, 2)) 40 | trans_angle = ax.transData.transform_angles(np.array((ang,)), pt)[0] 41 | 42 | else: 43 | trans_angle = 0 44 | 45 | # Set a bunch of keyword arguments 46 | if 'color' not in kwargs: 47 | kwargs['color'] = line.get_color() 48 | 49 | if ('horizontalalignment' not in kwargs) and ('ha' not in kwargs): 50 | kwargs['ha'] = 'center' 51 | 52 | if ('verticalalignment' not in kwargs) and ('va' not in kwargs): 53 | kwargs['va'] = 'center' 54 | 55 | if 'backgroundcolor' not in kwargs: 56 | kwargs['backgroundcolor'] = ax.get_facecolor() 57 | 58 | if 'clip_on' not in kwargs: 59 | kwargs['clip_on'] = True 60 | 61 | if 'zorder' not in kwargs: 62 | kwargs['zorder'] = 2.5 63 | 64 | ax.text(x, y, label, rotation=trans_angle, **kwargs) 65 | 66 | 67 | def labelLines(lines, align=True, xvals=None, **kwargs): 68 | """Labels lines in a line graph""" 69 | 70 | ax = lines[0].get_axes() 71 | labLines = [] 72 | labels = [] 73 | 74 | # Take only the lines which have labels other than the default ones 75 | for line in lines: 76 | label = line.get_label() 77 | if "_line" not in label and '_nolegend' not in label: 78 | labLines.append(line) 79 | labels.append(label) 80 | 81 | if xvals is None: 82 | xmin, xmax = ax.get_xlim() 83 | xvals = np.linspace(xmin, xmax, len(labLines)+2)[1:-1] 84 | 85 | for line, x, label in zip(labLines, xvals, labels): 86 | labelLine(line, x, label, align, **kwargs) -------------------------------------------------------------------------------- /scrapenhl2/scrape/games.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods related to scraping games. 3 | """ 4 | import os.path 5 | import re 6 | import datetime 7 | 8 | import scrapenhl2.scrape.organization as organization 9 | import scrapenhl2.scrape.schedules as schedules 10 | import scrapenhl2.scrape.team_info as team_info 11 | 12 | 13 | def most_recent_game_id(team1, team2): 14 | """ 15 | A convenience function to get the most recent game (this season) between two teams. 16 | 17 | :param team1: str, a team 18 | :param team2: str, a team 19 | 20 | :return: int, a game number 21 | """ 22 | return find_recent_games(team1, team2).Game.iloc[0] 23 | 24 | 25 | def find_recent_games(team1, team2=None, limit=1, season=None): 26 | """ 27 | A convenience function that lists the most recent in progress or final games for specified team(s) 28 | 29 | :param team1: str, a team 30 | :param team2: str, a team (optional) 31 | :param limit: How many games to return 32 | :param season: int, the season 33 | 34 | :return: df with relevant rows 35 | """ 36 | if season is None: 37 | season = schedules.get_current_season() 38 | sch = schedules.get_season_schedule(season) 39 | #sch = sch[sch.Status != "Scheduled"] # doesn't work if data hasn't been updated 40 | sch = sch[sch.Date <= datetime.datetime.now().strftime('%Y-%m-%d')] 41 | 42 | t1 = team_info.team_as_id(team1) 43 | sch = sch[(sch.Home == t1) | (sch.Road == t1)] 44 | if team2 is not None: 45 | t2 = team_info.team_as_id(team2) 46 | sch = sch[(sch.Home == t2) | (sch.Road == t2)] 47 | 48 | return sch.sort_values('Game', ascending=False).iloc[:limit, :] 49 | 50 | 51 | def find_playoff_game(searchstr): 52 | """ 53 | Finds playoff game id based on string specified 54 | :param searchstr: e.g. WSH PIT 2016 Game 5 55 | :return: (season, game) 56 | """ 57 | 58 | parts = searchstr.split(' ') 59 | teams = [] 60 | for part in parts: 61 | if re.match(r'^[A-z]{3}$', part.strip()): 62 | teams.append(part.upper()) 63 | if len(teams) != 2: 64 | return 65 | 66 | team1, team2 = teams[:2] 67 | 68 | searchstr += ' ' 69 | if re.search(r'\s\d{4}\s', searchstr) is not None: 70 | season = int(re.search(r'\s\d{4}\s', searchstr).group(0)) 71 | else: 72 | season = schedules.get_current_season() 73 | 74 | # Get game with a 5-digit regex 75 | if re.search(r'\s\d\s', searchstr) is not None: 76 | gamenum = int(re.search(r'\s\d\s', searchstr).group(0)) 77 | games = find_recent_games(team1, team2, limit=7, season=season) 78 | game = games[games.Game % 10 == gamenum].Game.iloc[0] 79 | else: 80 | raise ValueError 81 | 82 | return season, game 83 | 84 | 85 | 86 | def get_player_5v5_log_filename(season): 87 | """ 88 | Gets the filename for the season's player log file. Includes 5v5 CF, CA, TOI, and more. 89 | 90 | :param season: int, the season 91 | 92 | :return: str, /scrape/data/other/[season]_player_log.feather 93 | """ 94 | return os.path.join(organization.get_other_data_folder(), '{0:d}_player_5v5_log.feather'.format(season)) 95 | 96 | -------------------------------------------------------------------------------- /docs/build/html/genindex.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Index — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 | 46 |

Index

47 | 48 |
49 | 50 |
51 | 52 | 53 |
54 |
55 |
56 | 81 |
82 |
83 |
84 | ©2017, Muneeb Alam. 85 | 86 | | 87 | Powered by Sphinx 1.5.6 88 | & Alabaster 0.7.10 89 | 90 |
91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /tests/manipulate_tests.py: -------------------------------------------------------------------------------- 1 | # ! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from scrapenhl2 import scrape 5 | from scrapenhl2.manipulate.manipulate import ( 6 | time_to_mss, 7 | _filter_for_scores 8 | ) 9 | from unittest.mock import call, MagicMock 10 | from pytest_mock import mocker 11 | 12 | import pandas as pd 13 | 14 | def test_get_pbp_events(): 15 | pass 16 | 17 | def test_filter_for_event_types(): 18 | pass 19 | 20 | def test_filter_for_scores(mocker): 21 | mydf = pd.DataFrame({'TeamScore': [0, 0, 0, 1, 1, 1, 2, 2, 2], 22 | 'OppScore': [0, 1, 2, 0, 1, 2, 0, 1, 2], 23 | 'Row': list(range(1, 10))}) 24 | # No value 25 | assert _filter_for_scores(mydf, noscorekwarg=0).equals(mydf) is True 26 | # Single value 27 | assert _filter_for_scores(mydf, score_diff=0).equals(mydf.iloc[[0, 3, 6], :]) is True 28 | # Multiple values, including negative 29 | assert _filter_for_scores(mydf, score_diff=[-1, 1]) \ 30 | .sort_values('Row').equals(mydf.iloc[[2, 4, 6, 8], :]) is True 31 | 32 | def test_filter_for_strengths(): 33 | pass 34 | 35 | def test_filter_for_times(): 36 | pass 37 | 38 | def test_filter_for_games(): 39 | pass 40 | 41 | def test_filter_for_players(): 42 | pass 43 | 44 | def test_join_on_ice_players_to_pbp(): 45 | pass 46 | 47 | def test_filter_for_team(): 48 | pass 49 | 50 | def test_seasons_to_read(): 51 | pass 52 | 53 | def test_teams_to_read(): 54 | pass 55 | 56 | def test_get_5v5_player_game_toi(): 57 | pass 58 | 59 | def test_generate_player_toion_toioff(): 60 | pass 61 | 62 | def test_count_by_keys(): 63 | pass 64 | 65 | def test_get_5v5_player_game_boxcars(): 66 | pass 67 | 68 | def test_get_5v5_player_game_toicomp(): 69 | pass 70 | 71 | def test_long_on_player_and_opp(): 72 | pass 73 | 74 | def test_merge_toi60_position_calculate_sums(): 75 | pass 76 | 77 | def test_retrieve_start_end_times(): 78 | pass 79 | 80 | def test_get_5v5_player_game_shift_startend(): 81 | pass 82 | 83 | def test_get_directions_for_xy_for_season(): 84 | pass 85 | 86 | def test_get_directions_for_xy_for_game(): 87 | pass 88 | 89 | def test_infer_zones_for_faceoffs(): 90 | pass 91 | 92 | def test_generate_5v5_player_log(): 93 | pass 94 | 95 | def test_get_5v5_player_game_fa(): 96 | pass 97 | 98 | def test_merge_onto_all_team_games_and_zero_fill(): 99 | pass 100 | 101 | def test_convert_to_all_combos(): 102 | pass 103 | 104 | def test_get_player_toi(): 105 | pass 106 | 107 | def test_get_line_combos(): 108 | pass 109 | 110 | def test_get_pairings(): 111 | pass 112 | 113 | def test_get_game_h2h_toi(): 114 | pass 115 | 116 | def test_filter_for_event_types2(): 117 | pass 118 | 119 | def test_get_game_h2h_corsi(): 120 | pass 121 | 122 | def test_time_to_mss(mocker): 123 | # 0:00 124 | assert time_to_mss(0) == '0:00' 125 | # 10:01 126 | assert time_to_mss(601) == "10:01" 127 | # 10:10 128 | assert time_to_mss(610) == "10:10" 129 | 130 | def test_team_5v5_score_state_summary_by_game(): 131 | pass 132 | 133 | def test_team_5v5_shot_rates_by_score(): 134 | pass 135 | 136 | def test_add_score_adjustment_to_team_pbp(): 137 | pass 138 | -------------------------------------------------------------------------------- /docs/source/plot.rst: -------------------------------------------------------------------------------- 1 | .. _plot: 2 | 3 | Plot 4 | ==== 5 | 6 | The scrapenhl2.plot module contains methods useful for plotting. 7 | 8 | Useful examples 9 | --------------- 10 | 11 | First, import:: 12 | 13 | from scrapenhl2.plot import * 14 | 15 | Get the H2H for an in-progress game:: 16 | 17 | live_h2h('WSH', 'EDM') 18 | 19 | .. image:: _static/example_h2h.png 20 | 21 | Get the Corsi timeline as well, but don't update data this time:: 22 | 23 | live_timeline('WSH', 'EDM', update=False) 24 | 25 | .. image:: _static/example_timeline.png 26 | 27 | Save the timeline of a memorable game to file:: 28 | 29 | game_timeline(2016, 30136, save_file='/Users/muneebalam/Desktop/WSH_TOR_G6.png') 30 | 31 | More methods being added regularly. 32 | 33 | App 34 | ---- 35 | 36 | This package contains a lightweight app for browsing charts and doing some data manipulations. 37 | 38 | Launch using:: 39 | 40 | import scrapenhl2.plot.app as app 41 | app.browse_game_charts() 42 | # app.browse_player_charts() 43 | # app.browse_team_charts() 44 | 45 | It will print a link in your terminal--follow it. The page looks something like this: 46 | 47 | .. image:: _static/game_page_screenshot.png 48 | 49 | The dropdowns also allow you to search--just start typing. 50 | 51 | Methods (games) 52 | ----------------- 53 | 54 | Game H2H 55 | ~~~~~~~~ 56 | .. image:: _static/WSH-TOR_G6.png 57 | :width: 50% 58 | :align: right 59 | 60 | .. automodule:: scrapenhl2.plot.game_h2h 61 | :members: 62 | 63 | Corsi timeline 64 | ~~~~~~~~~~~~~~ 65 | .. image:: _static/WSH-TOR_G6_timeline.png 66 | :width: 50% 67 | :align: right 68 | 69 | .. automodule:: scrapenhl2.plot.game_timeline 70 | :members: 71 | 72 | Methods (teams) 73 | --------------- 74 | 75 | Team TOI by score 76 | ~~~~~~~~~~~~~~~~~ 77 | .. image:: _static/Score_states_2015.png 78 | :width: 50% 79 | :align: right 80 | 81 | .. automodule:: scrapenhl2.plot.team_score_state_toi 82 | :members: 83 | 84 | Team lineup CF% 85 | ~~~~~~~~~~~~~~~ 86 | .. image:: _static/Caps_lineup_cf.png 87 | :width: 50% 88 | :align: right 89 | 90 | .. automodule:: scrapenhl2.plot.team_lineup_cf 91 | :members: 92 | 93 | Team shot rates by score 94 | ~~~~~~~~~~~~~~~~~~~~~~~~ 95 | 96 | .. image:: _static/Caps_shot_score_parallel.png 97 | :width: 49% 98 | .. image:: _static/Caps_shot_rates_score_scatter.png 99 | :width: 49% 100 | 101 | 102 | .. automodule:: scrapenhl2.plot.team_score_shot_rate 103 | :members: 104 | 105 | Methods (individuals) 106 | ---------------------- 107 | 108 | Player rolling CF and GF 109 | ~~~~~~~~~~~~~~~~~~~~~~~~ 110 | .. image:: _static/Ovechkin_rolling_cf.png 111 | :width: 50% 112 | :align: right 113 | 114 | .. automodule:: scrapenhl2.plot.rolling_cf_gf 115 | :members: 116 | 117 | Player rolling boxcars 118 | ~~~~~~~~~~~~~~~~~~~~~~ 119 | .. image:: _static/Oshie_boxcars.png 120 | :width: 50% 121 | :align: right 122 | 123 | .. automodule:: scrapenhl2.plot.rolling_boxcars 124 | :members: 125 | 126 | Methods (individual comparisons) 127 | -------------------------------- 128 | 129 | Team D-pair shot rates 130 | ~~~~~~~~~~~~~~~~~~~~~~ 131 | 132 | .. image:: _static/Caps_d_pairs.png 133 | :width: 50% 134 | :align: right 135 | 136 | .. automodule:: scrapenhl2.plot.defense_pairs 137 | :members: 138 | 139 | Usage 140 | ~~~~~ 141 | .. automodule:: scrapenhl2.plot.usage 142 | :members: 143 | 144 | Helper methods 145 | -------------- 146 | 147 | .. automodule:: scrapenhl2.plot.visualization_helper 148 | :members: 149 | 150 | .. automodule:: scrapenhl2.plot.label_lines 151 | :members: 152 | 153 | -------------------------------------------------------------------------------- /tests/test_schedule.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from scrapenhl2.scrape.schedules import ( 5 | _get_current_season, 6 | schedule_setup, 7 | get_season_schedule_filename, 8 | get_season_schedule, 9 | get_team_schedule, 10 | write_season_schedule, 11 | get_game_data_from_schedule, 12 | _CURRENT_SEASON, 13 | _SCHEDULES, 14 | ) 15 | from unittest.mock import call, MagicMock 16 | from pytest_mock import mocker 17 | 18 | def test_get_current_season(mocker): 19 | 20 | now_mock = mocker.patch("arrow.now") 21 | 22 | date_mock = now_mock.return_value 23 | date_mock.year = 2017 24 | date_mock.month = 8 25 | 26 | assert _get_current_season() == 2016 27 | date_mock.month = 9 28 | assert _get_current_season() == 2017 29 | date_mock.month = 10 30 | assert _get_current_season() == 2017 31 | 32 | 33 | def test_get_season_schedule_filename(mocker): 34 | 35 | organization_mock = mocker.patch( 36 | "scrapenhl2.scrape.schedules.organization" 37 | ) 38 | organization_mock.get_other_data_folder.return_value = "/tmp" 39 | 40 | assert get_season_schedule_filename(2017) == "/tmp/2017_schedule.feather" 41 | 42 | 43 | def test_schedule_setup(mocker): 44 | 45 | current_season_mock = mocker.patch( 46 | "scrapenhl2.scrape.schedules._get_current_season" 47 | ) 48 | current_season_mock.return_value = 2006 49 | get_season_schedule_mock = mocker.patch( 50 | "scrapenhl2.scrape.schedules.get_season_schedule_filename" 51 | ) 52 | get_season_schedule_mock.side_effect = ["tmp/2005", "tmp/2006"] 53 | path_exists_mock = mocker.patch( 54 | "os.path.exists" 55 | ) 56 | path_exists_mock.side_effect = [True, False] 57 | gen_schedule_file_mock = mocker.patch( 58 | "scrapenhl2.scrape.schedules.generate_season_schedule_file" 59 | ) 60 | season_schedule_mock = mocker.patch( 61 | "scrapenhl2.scrape.schedules._get_season_schedule" 62 | ) 63 | 64 | schedule_setup() 65 | get_season_schedule_mock.assert_has_calls([call(2005), call(2006)]) 66 | path_exists_mock.assert_has_calls(get_season_schedule_mock.return_value) 67 | gen_schedule_file_mock.assert_has_calls([call(2006)]) 68 | season_schedule_mock.assert_has_calls([call(2005), call(2006)]) 69 | 70 | 71 | def test_write_season_schedule(mocker): 72 | 73 | feather_mock = mocker.patch("scrapenhl2.scrape.schedules.feather") 74 | dataframe_mock = MagicMock() 75 | 76 | ret = write_season_schedule(dataframe_mock, 2017, True) 77 | 78 | feather_mock.write_dataframe.assert_called_once_with( 79 | dataframe_mock, get_season_schedule_filename(2017) 80 | ) 81 | 82 | get_season_schedule_mock = mocker.patch( 83 | "scrapenhl2.scrape.schedules.get_season_schedule" 84 | ) 85 | panda_mock = mocker.patch("scrapenhl2.scrape.schedules.pd") 86 | 87 | ret = write_season_schedule(dataframe_mock, 2017, False) 88 | 89 | assert get_season_schedule_mock().query.called_once_with("Status != Final") 90 | assert panda_mock.concat.called_once_with( 91 | get_season_schedule_mock().query() 92 | ) 93 | 94 | def test_get_game_data_from_schedule(mocker): 95 | 96 | get_season_schedule_mock = mocker.patch( 97 | "scrapenhl2.scrape.schedules.get_season_schedule" 98 | ) 99 | 100 | get_game_data_from_schedule(2017, 1234) 101 | get_season_schedule_mock.assert_called_once_with(2017) 102 | get_season_schedule_mock().query.assert_called_once_with('Game == 1234') 103 | get_season_schedule_mock().query().to_dict.assert_called_once_with(orient='series') 104 | 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /docs/build/html/scrape_game.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrape_game module — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 |
46 |

scrape_game module

48 |
49 | 50 | 51 |
52 |
53 |
54 | 84 |
85 |
86 |
87 | ©2017, Muneeb Alam. 88 | 89 | | 90 | Powered by Sphinx 1.5.6 91 | & Alabaster 0.7.10 92 | 93 | | 94 | Page source 96 |
97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /docs/build/html/scrape_setup.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrape_setup module — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 |
46 |

scrape_setup module

48 |
49 | 50 | 51 |
52 |
53 |
54 | 84 |
85 |
86 |
87 | ©2017, Muneeb Alam. 88 | 89 | | 90 | Powered by Sphinx 1.5.6 91 | & Alabaster 0.7.10 92 | 93 | | 94 | Page source 96 |
97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /scrapenhl2/plot/team_lineup_cf.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods to generate a graph showing player CF%. 18 little graphs, 1 for each of 18 players. 3 | """ 4 | 5 | import matplotlib.pyplot as plt 6 | import pandas as pd 7 | 8 | import scrapenhl2.scrape.players as players 9 | import scrapenhl2.plot.visualization_helper as vhelper 10 | import scrapenhl2.plot.rolling_cf_gf as rolling_cfgf 11 | 12 | 13 | def team_lineup_cf_graph(team, **kwargs): 14 | """ 15 | This method builds a 4x5 matrix of rolling CF% line graphs. The left 4x3 matrix are forward lines and the top-right 16 | 3x2 are defense pairs. 17 | 18 | :param team: str or id, team to build this graph for 19 | :param kwargs: need to specify the following as iterables of names: l1, l2, l3, l4, p1, p2, p3. 20 | Three players for each of the 'l's and two for each of the 'p's. 21 | 22 | :return: figure, or nothing 23 | """ 24 | allplayers = [] 25 | if 'l1' in kwargs and 'l2' in kwargs and 'l3' in kwargs and 'l4' in kwargs and \ 26 | 'p1' in kwargs and 'p2' in kwargs and 'p3' in kwargs: 27 | # Change all to IDs 28 | # Go on this strange order because it'll be the order of the plots below 29 | for key in ['l1', 'p1', 'l2', 'p2', 'l3', 'p3', 'l4']: 30 | kwargs[key] = [players.player_as_id(x) for x in kwargs[key]] 31 | allplayers += kwargs[key] 32 | else: 33 | # TODO Find most common lines 34 | # Edit get_line_combos etc from manip, and the method to get player order from game_h2h, to work at team level 35 | pass 36 | 37 | # Get data 38 | kwargs['add_missing_games'] = True 39 | kwargs['team'] = team 40 | kwargs['players'] = allplayers 41 | if 'roll_len' not in kwargs: 42 | kwargs['roll_len'] = 25 43 | data = vhelper.get_and_filter_5v5_log(**kwargs) 44 | df = pd.concat([data[['Season', 'Game', 'PlayerID']], rolling_cfgf._calculate_f_rates(data, 'C')], axis=1) 45 | col_dict = {col[col.index(' ') + 1:]: col for col in df.columns if '%' in col} 46 | 47 | # Set up figure to share x and y 48 | fig, axes = plt.subplots(4, 5, sharex=True, sharey=True, figsize=[12, 8]) 49 | 50 | # Make chart for each player 51 | gamenums = df[['Season', 'Game']].drop_duplicates().assign(GameNum=1) 52 | gamenums.loc[:, 'GameNum'] = gamenums.GameNum.cumsum() 53 | df = df.merge(gamenums, how='left', on=['Season', 'Game']) 54 | 55 | axes = axes.flatten() 56 | for i in range(len(allplayers)): 57 | ax = axes[i] 58 | ax.set_title(players.player_as_str(allplayers[i]), fontsize=10) 59 | temp = df.query('PlayerID == {0:d}'.format(int(allplayers[i]))) 60 | x = temp.GameNum.values 61 | y1 = temp[col_dict['CF%']].values 62 | y2 = temp[col_dict['CF% Off']].values 63 | ax.fill_between(x, y1, y2, where=y1 > y2, alpha=0.5) 64 | ax.fill_between(x, y1, y2, where=y2 > y1, alpha=0.5) 65 | ax.plot(x, y1) 66 | ax.plot(x, y2, ls='--') 67 | ax.plot(x, [0.5 for _ in range(len(x))], color='k') 68 | 69 | for i, ax in enumerate(axes): 70 | for direction in ['right', 'top', 'bottom', 'left']: 71 | ax.spines[direction].set_visible(False) 72 | ax.xaxis.set_ticks_position('none') 73 | ax.yaxis.set_ticks_position('none') 74 | 75 | # Set title and axis labels 76 | axes[0].set_ylim(0.35, 0.65) 77 | axes[0].set_yticks([0.4, 0.5, 0.6]) 78 | axes[0].set_yticklabels(['40%', '50%', '60%']) 79 | axes[0].set_xlim(1, df.GameNum.max()) 80 | 81 | plt.annotate('Game', xy=(0.5, 0.05), ha='center', va='top', xycoords='figure fraction') 82 | 83 | fig.suptitle(_team_lineup_cf_graph_title(**kwargs), fontsize=16, y=0.95) 84 | 85 | # Return 86 | return vhelper.savefilehelper(**kwargs) 87 | 88 | 89 | def _team_lineup_cf_graph_title(**kwargs): 90 | return ', '.join(vhelper.generic_5v5_log_graph_title('Lineup CF%', **kwargs)) 91 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/organization.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains paths to folders. 3 | """ 4 | 5 | import os.path 6 | 7 | 8 | def check_create_folder(*args): 9 | """ 10 | A helper method to create a folder if it doesn't exist already 11 | 12 | :param args: list of str, the parts of the filepath. These are joined together with the base directory 13 | 14 | :return: nothing 15 | """ 16 | path = os.path.join(get_base_dir(), *args) 17 | if not os.path.exists(path): 18 | os.makedirs(path) 19 | 20 | 21 | def get_base_dir(): 22 | """ 23 | Returns the base directory of this package (one directory up from this file) 24 | 25 | :return: str, the base directory 26 | """ 27 | return os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 28 | 29 | 30 | def get_raw_data_folder(): 31 | """ 32 | Returns the folder containing raw data 33 | 34 | :return: str, /scrape/data/raw/ 35 | """ 36 | return os.path.join(get_base_dir(), 'data', 'raw') 37 | 38 | 39 | def get_parsed_data_folder(): 40 | """ 41 | Returns the folder containing parsed data 42 | 43 | :return: str, /scrape/data/parsed/ 44 | """ 45 | return os.path.join(get_base_dir(), 'data', 'parsed') 46 | 47 | 48 | def get_team_data_folder(): 49 | """ 50 | Returns the folder containing team log data 51 | 52 | :return: str, /scrape/data/teams/ 53 | """ 54 | return os.path.join(get_base_dir(), 'data', 'teams') 55 | 56 | 57 | def get_other_data_folder(): 58 | """ 59 | Returns the folder containing other data 60 | 61 | :return: str, /scrape/data/other/ 62 | """ 63 | return os.path.join(get_base_dir(), 'data', 'other') 64 | 65 | 66 | def get_season_raw_pbp_folder(season): 67 | """ 68 | Returns the folder containing raw pbp for given season 69 | 70 | :param season: int, current season 71 | 72 | :return: str, /scrape/data/raw/pbp/[season]/ 73 | """ 74 | return os.path.join(get_raw_data_folder(), 'pbp', str(season)) 75 | 76 | 77 | def get_season_raw_toi_folder(season): 78 | """ 79 | Returns the folder containing raw toi for given season 80 | 81 | :param season: int, current season 82 | 83 | :return: str, /scrape/data/raw/toi/[season]/ 84 | """ 85 | return os.path.join(get_raw_data_folder(), 'toi', str(season)) 86 | 87 | 88 | def get_season_parsed_pbp_folder(season): 89 | """ 90 | Returns the folder containing parsed pbp for given season 91 | 92 | :param season: int, current season 93 | 94 | :return: str, /scrape/data/parsed/pbp/[season]/ 95 | """ 96 | return os.path.join(get_parsed_data_folder(), 'pbp', str(season)) 97 | 98 | 99 | def get_season_parsed_toi_folder(season): 100 | """ 101 | Returns the folder containing parsed toi for given season 102 | 103 | :param season: int, current season 104 | 105 | :return: str, /scrape/data/raw/toi/[season]/ 106 | """ 107 | return os.path.join(get_parsed_data_folder(), 'toi', str(season)) 108 | 109 | 110 | def get_season_team_pbp_folder(season): 111 | """ 112 | Returns the folder containing team pbp logs for given season 113 | 114 | :param season: int, current season 115 | 116 | :return: str, /scrape/data/teams/pbp/[season]/ 117 | """ 118 | return os.path.join(get_team_data_folder(), 'pbp', str(season)) 119 | 120 | 121 | def get_season_team_toi_folder(season): 122 | """ 123 | Returns the folder containing team toi logs for given season 124 | 125 | :param season: int, current season 126 | 127 | :return: str, /scrape/data/teams/toi/[season]/ 128 | """ 129 | return os.path.join(get_team_data_folder(), 'toi', str(season)) 130 | 131 | 132 | def organization_setup(): 133 | """ 134 | Creates other folder if need be 135 | 136 | :return: nothing 137 | """ 138 | check_create_folder(get_other_data_folder()) 139 | 140 | 141 | organization_setup() 142 | -------------------------------------------------------------------------------- /docs/build/html/search.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Search — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
50 |
51 |
52 |
53 | 54 |

Search

55 |
56 | 57 |

58 | Please activate JavaScript to enable the search 59 | functionality. 60 |

61 |
62 |

63 | From here you can search these documents. Enter your search 64 | words into the box below and click "search". Note that the search 65 | function will automatically search for all of the words. Pages 66 | containing fewer words won't appear in the result list. 67 |

68 |
69 | 70 | 71 | 72 |
73 | 74 |
75 | 76 |
77 | 78 |
79 |
80 |
81 | 94 |
95 |
96 |
97 | ©2017, Muneeb Alam. 98 | 99 | | 100 | Powered by Sphinx 1.5.6 101 | & Alabaster 0.7.10 102 | 103 |
104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /docs/build/search.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Search — scrapenhl2 0.3.9 documentation 10 | 11 | 12 | 13 | 14 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 51 | 52 |
53 |
54 |
55 |
56 | 57 |

Search

58 |
59 | 60 |

61 | Please activate JavaScript to enable the search 62 | functionality. 63 |

64 |
65 |

66 | From here you can search these documents. Enter your search 67 | words into the box below and click "search". Note that the search 68 | function will automatically search for all of the words. Pages 69 | containing fewer words won't appear in the result list. 70 |

71 |
72 | 73 | 74 | 75 |
76 | 77 |
78 | 79 |
80 | 81 |
82 |
83 |
84 | 88 |
89 |
90 | 102 | 106 | 107 | -------------------------------------------------------------------------------- /docs/build/html/modules.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrape — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 |
46 |

scrape

47 |
48 | 54 |
55 |
56 | 57 | 58 |
59 |
60 |
61 | 91 |
92 |
93 |
94 | ©2017, Muneeb Alam. 95 | 96 | | 97 | Powered by Sphinx 1.5.6 98 | & Alabaster 0.7.10 99 | 100 | | 101 | Page source 103 |
104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /docs/build/scrapenhl2.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrapenhl2 package — scrapenhl2 0.3.8 documentation 10 | 11 | 12 | 13 | 14 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 43 | 44 |
45 |
46 |
47 |
48 | 49 |
50 |

scrapenhl2 package

51 |
52 |

Module contents

53 |
54 |
55 | 56 | 57 |
58 |
59 |
60 | 89 |
90 |
91 | 103 | 107 | 108 | -------------------------------------------------------------------------------- /docs/build/html/scrapenhl2.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrapenhl2 package — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 |
46 |

scrapenhl2 package

48 |
49 |

Module contents

51 |
52 |
53 | 54 | 55 |
56 |
57 |
58 | 96 |
97 |
98 |
99 | ©2017, Muneeb Alam. 100 | 101 | | 102 | Powered by Sphinx 1.5.6 103 | & Alabaster 0.7.10 104 | 105 | | 106 | Page source 108 |
109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /docs/build/_static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ 8 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 9 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 10 | .highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ 11 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 12 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 13 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 14 | .highlight .ge { font-style: italic } /* Generic.Emph */ 15 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 16 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 17 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 18 | .highlight .go { color: #333333 } /* Generic.Output */ 19 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 20 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 21 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 22 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 23 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 24 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 25 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 26 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 27 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 28 | .highlight .kt { color: #902000 } /* Keyword.Type */ 29 | .highlight .m { color: #208050 } /* Literal.Number */ 30 | .highlight .s { color: #4070a0 } /* Literal.String */ 31 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 32 | .highlight .nb { color: #007020 } /* Name.Builtin */ 33 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 34 | .highlight .no { color: #60add5 } /* Name.Constant */ 35 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 36 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 37 | .highlight .ne { color: #007020 } /* Name.Exception */ 38 | .highlight .nf { color: #06287e } /* Name.Function */ 39 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 40 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 41 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 42 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 43 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 44 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 45 | .highlight .mb { color: #208050 } /* Literal.Number.Bin */ 46 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 47 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 48 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 49 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 50 | .highlight .sa { color: #4070a0 } /* Literal.String.Affix */ 51 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 52 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 53 | .highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ 54 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 55 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 56 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 57 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 58 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 59 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 60 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 61 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 62 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 63 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 64 | .highlight .fm { color: #06287e } /* Name.Function.Magic */ 65 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 66 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 67 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 68 | .highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ 69 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /docs/build/html/_static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ 8 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 9 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 10 | .highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ 11 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 12 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 13 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 14 | .highlight .ge { font-style: italic } /* Generic.Emph */ 15 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 16 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 17 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 18 | .highlight .go { color: #333333 } /* Generic.Output */ 19 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 20 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 21 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 22 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 23 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 24 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 25 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 26 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 27 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 28 | .highlight .kt { color: #902000 } /* Keyword.Type */ 29 | .highlight .m { color: #208050 } /* Literal.Number */ 30 | .highlight .s { color: #4070a0 } /* Literal.String */ 31 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 32 | .highlight .nb { color: #007020 } /* Name.Builtin */ 33 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 34 | .highlight .no { color: #60add5 } /* Name.Constant */ 35 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 36 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 37 | .highlight .ne { color: #007020 } /* Name.Exception */ 38 | .highlight .nf { color: #06287e } /* Name.Function */ 39 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 40 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 41 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 42 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 43 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 44 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 45 | .highlight .mb { color: #208050 } /* Literal.Number.Bin */ 46 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 47 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 48 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 49 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 50 | .highlight .sa { color: #4070a0 } /* Literal.String.Affix */ 51 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 52 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 53 | .highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ 54 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 55 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 56 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 57 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 58 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 59 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 60 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 61 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 62 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 63 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 64 | .highlight .fm { color: #06287e } /* Name.Function.Magic */ 65 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 66 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 67 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 68 | .highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ 69 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /docs/build/modules.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrape — scrapenhl2 0.3.8 documentation 10 | 11 | 12 | 13 | 14 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 44 | 45 |
46 |
47 |
48 |
49 | 50 |
51 |

scrape

52 |
53 |
54 | 55 | 56 | 57 | 58 | 59 | 60 | 64 | 65 | 66 |
maxdepth:

4

61 |

scrape_game 62 | scrape_setup

63 |
67 |
68 | 69 | 70 |
71 |
72 |
73 | 94 |
95 |
96 | 109 | 113 | 114 | -------------------------------------------------------------------------------- /scrapenhl2/plot/rolling_boxcars.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods for creating the rolling boxcars stacked area graph. 3 | """ 4 | 5 | import pandas as pd 6 | import matplotlib.pyplot as plt 7 | 8 | from scrapenhl2.plot import visualization_helper as vhelper 9 | from scrapenhl2.scrape import players 10 | 11 | def rolling_player_boxcars(player, **kwargs): 12 | """ 13 | A method to generate the rolling boxcars graph. 14 | 15 | :param player: str or int, player to generate for 16 | :param kwargs: other filters. See scrapenhl2.plot.visualization_helper.get_and_filter_5v5_log for more information. 17 | 18 | :return: nothing, or figure 19 | """ 20 | 21 | kwargs['player'] = player 22 | if 'roll_len' not in kwargs: 23 | kwargs['roll_len'] = 25 24 | boxcars = vhelper.get_and_filter_5v5_log(**kwargs) 25 | 26 | boxcars = pd.concat([boxcars[['Season', 'Game']], calculate_boxcar_rates(boxcars)], axis=1) 27 | 28 | col_dict = {col[col.index(' ') + 1:col.index('/') ]: col for col in boxcars.columns if col[-3:] == '/60'} 29 | 30 | plt.clf() 31 | 32 | # Set an index 33 | # TODO allow for datetime index 34 | boxcars.loc[:, 'Game Number'] = 1 35 | boxcars.loc[:, 'Game Number'] = boxcars['Game Number'].cumsum() 36 | boxcars.set_index('Game Number', inplace=True) 37 | plt.fill_between(boxcars.index, 0, boxcars[col_dict['iG']], label='G', color='k') 38 | plt.fill_between(boxcars.index, boxcars[col_dict['iG']], boxcars[col_dict['iP1']], label='A1', color='b') 39 | plt.fill_between(boxcars.index, boxcars[col_dict['iP1']], boxcars[col_dict['iP']], label='A2', color='dodgerblue') 40 | plt.fill_between(boxcars.index, boxcars[col_dict['iP']], boxcars[col_dict['GFON']], 41 | label='Other\nGFON', color='c', alpha=0.3) 42 | 43 | plt.xlabel('Game') 44 | plt.ylabel('Per 60') 45 | plt.xlim(0, len(boxcars)) 46 | plt.ylim(0, 4) 47 | 48 | position = players.get_player_position(player) 49 | if position == 'D': 50 | ypos = [0.17, 0.84, 2.5] 51 | ytext = ['P1\nG', 'P1\nP', 'P1\nGF'] 52 | elif position in {'C', 'R', 'L', 'F'}: 53 | ypos = [0.85, 1.94, 2.7] 54 | ytext = ['L1\nG', 'L1\nP', 'L1\nGF'] 55 | 56 | xlimits = plt.xlim() 57 | tempaxis = plt.twinx() 58 | tempaxis.tick_params(axis='y', which='major', pad=2) 59 | tempaxis.set_yticks(ypos) 60 | tempaxis.set_yticklabels(ytext, fontsize=8) 61 | tempaxis.grid(b=False) 62 | tempaxis.plot(xlimits, [ypos[0], ypos[0]], color='k', ls=':') 63 | tempaxis.plot(xlimits, [ypos[1], ypos[1]], color='dodgerblue', ls=':') 64 | tempaxis.plot(xlimits, [ypos[2], ypos[2]], color='c', ls=':') 65 | 66 | plt.legend(loc=2, bbox_to_anchor=(1.05, 1), fontsize=10) 67 | tempaxis.set_ylim(0, 4) 68 | plt.xlim(0, len(boxcars)) 69 | plt.ylim(0, 4) 70 | 71 | plt.title(_get_rolling_boxcars_title(**kwargs)) 72 | 73 | return vhelper.savefilehelper(**kwargs) 74 | 75 | 76 | def _get_rolling_boxcars_title(**kwargs): 77 | """ 78 | Returns suggested chart title for rolling boxcar graph given these keyword arguments 79 | 80 | :param kwargs: 81 | 82 | :return: str 83 | """ 84 | 85 | title = 'Rolling {0:d}-game boxcar rates for {1:s}'.format(kwargs['roll_len'], 86 | players.player_as_str(kwargs['player'])) 87 | title += '\n{0:s} to {1:s}'.format(*(str(x) for x in vhelper.get_startdate_enddate_from_kwargs(**kwargs))) 88 | return title 89 | 90 | 91 | def calculate_boxcar_rates(df): 92 | """ 93 | Takes the given dataframe and makes the following calculations: 94 | 95 | - Divides col ending in GFON, iA2, iA1, and iG by one ending in TOI 96 | - Adds iG to iA1, calls result iP1 97 | - Adds iG and iA1 to iA2, calls result iP 98 | - Adds /60 to ends of iG, iA1, iP1, iA2, iP, and GFON 99 | 100 | :param df: dataframe 101 | 102 | :return: dataframe with columns changed as specified, and only those mentioned above selected. 103 | """ 104 | 105 | # Select columns 106 | boxcars = df.filter(regex='\d{2}-game') 107 | cols_wanted = {'GFON', 'iA1', 'iA2', 'iG', 'TOION'} 108 | boxcars = boxcars.select(lambda colname: colname[colname.index(' ') + 1:] in cols_wanted, axis=1) 109 | 110 | # This is to help me select columns 111 | col_dict = {col[col.index(' ') + 1:]: col for col in boxcars.columns} 112 | 113 | # Transform 114 | for col in {'iG', 'iA1', 'iA2', 'GFON'}: 115 | boxcars.loc[:, col_dict[col]] = boxcars[col_dict[col]] / boxcars[col_dict['TOION']] 116 | 117 | prefix = col_dict['iG'][:col_dict['iG'].index(' ')] # e.g. 25-game 118 | boxcars.loc[:, prefix + ' iP1'] = boxcars[col_dict['iG']] + boxcars[col_dict['iA1']] 119 | boxcars.loc[:, prefix + ' iP'] = boxcars[prefix + ' iP1'] + boxcars[col_dict['iA2']] 120 | 121 | # Rename 122 | renaming = {col: col + '/60' for col in boxcars.columns if col[-5:] != 'TOION'} 123 | boxcars.rename(columns=renaming, inplace=True) 124 | 125 | return boxcars 126 | 127 | -------------------------------------------------------------------------------- /docs/build/html/index.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Welcome to scrapenhl2’s documentation! — scrapenhl2 0.0.11 documentation 10 | 11 | 12 | 13 | 14 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 |
42 |
43 |
44 | 45 |
46 |

Welcome to scrapenhl2’s documentation!

49 |
50 |
51 |
52 |
53 |

Indices and tables

55 | 63 |
64 | 65 | 66 |
67 |
68 |
69 | 104 |
105 |
106 |
107 | ©2017, Muneeb Alam. 108 | 109 | | 110 | Powered by Sphinx 1.5.6 111 | & Alabaster 0.7.10 112 | 113 | | 114 | Page source 116 |
117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # scrapenhl2 documentation build configuration file, created by 5 | # sphinx-quickstart on Sun Oct 1 17:47:07 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import sys 21 | 22 | sys.path.insert(0, '../../') 23 | sys.path.insert(0, '../') 24 | sys.path.insert(0, './') 25 | sys.path.insert(0, '../../scrapenhl2/') 26 | sys.path.insert(0, '../../scrapenhl2/scrape/') 27 | sys.path.insert(0, '../../scrapenhl2/manipulate/') 28 | sys.path.insert(0, '../../scrapenhl2/plot/') 29 | 30 | 31 | # -- General configuration ------------------------------------------------ 32 | 33 | # If your documentation needs a minimal Sphinx version, state it here. 34 | # 35 | # needs_sphinx = '1.0' 36 | 37 | # Add any Sphinx extension module names here, as strings. They can be 38 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 39 | # ones. 40 | extensions = ['sphinx.ext.autodoc', 41 | 'sphinx.ext.todo'] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ['_templates'] 45 | 46 | # The suffix(es) of source filenames. 47 | # You can specify multiple suffix as a list of string: 48 | # 49 | # source_suffix = ['.rst', '.md'] 50 | source_suffix = '.rst' 51 | 52 | # The master toctree document. 53 | master_doc = 'index' 54 | 55 | # General information about the project. 56 | project = 'scrapenhl2' 57 | copyright = '2017, Muneeb Alam' 58 | author = 'Muneeb Alam' 59 | 60 | # The version info for the project you're documenting, acts as replacement for 61 | # |version| and |release|, also used in various other places throughout the 62 | # built documents. 63 | # 64 | # The short X.Y version. 65 | version = '0.4.1' 66 | # The full version, including alpha/beta/rc tags. 67 | release = version 68 | 69 | # The language for content autogenerated by Sphinx. Refer to documentation 70 | # for a list of supported languages. 71 | # 72 | # This is also used if you do content translation via gettext catalogs. 73 | # Usually you set "language" from the command line for these cases. 74 | language = None 75 | 76 | # List of patterns, relative to source directory, that match files and 77 | # directories to ignore when looking for source files. 78 | # This patterns also effect to html_static_path and html_extra_path 79 | exclude_patterns = [] 80 | 81 | # The name of the Pygments (syntax highlighting) style to use. 82 | pygments_style = 'sphinx' 83 | 84 | # If true, `todo` and `todoList` produce output, else they produce nothing. 85 | todo_include_todos = True 86 | 87 | # -- Options for HTML output ---------------------------------------------- 88 | 89 | # The theme to use for HTML and HTML Help pages. See the documentation for 90 | # a list of builtin themes. 91 | # 92 | html_theme = 'nature' 93 | 94 | # Theme options are theme-specific and customize the look and feel of a theme 95 | # further. For a list of options available for each theme, see the 96 | # documentation. 97 | # 98 | # html_theme_options = {} 99 | 100 | # Add any paths that contain custom static files (such as style sheets) here, 101 | # relative to this directory. They are copied after the builtin static files, 102 | # so a file named "default.css" will overwrite the builtin "default.css". 103 | html_static_path = ['_static'] 104 | 105 | # -- Options for HTMLHelp output ------------------------------------------ 106 | 107 | # Output file base name for HTML help builder. 108 | htmlhelp_basename = 'scrapenhl2doc' 109 | 110 | # -- Options for LaTeX output --------------------------------------------- 111 | 112 | latex_elements = { 113 | # The paper size ('letterpaper' or 'a4paper'). 114 | # 115 | # 'papersize': 'letterpaper', 116 | 117 | # The font size ('10pt', '11pt' or '12pt'). 118 | # 119 | # 'pointsize': '10pt', 120 | 121 | # Additional stuff for the LaTeX preamble. 122 | # 123 | # 'preamble': '', 124 | 125 | # Latex figure (float) alignment 126 | # 127 | # 'figure_align': 'htbp', 128 | } 129 | 130 | # Grouping the document tree into LaTeX files. List of tuples 131 | # (source start file, target name, title, 132 | # author, documentclass [howto, manual, or own class]). 133 | latex_documents = [ 134 | (master_doc, 'scrapenhl2.tex', 'scrapenhl2 Documentation', 135 | 'Muneeb Alam', 'manual'), 136 | ] 137 | 138 | # -- Options for manual page output --------------------------------------- 139 | 140 | # One entry per manual page. List of tuples 141 | # (source start file, name, description, authors, manual section). 142 | man_pages = [ 143 | (master_doc, 'scrapenhl2', 'scrapenhl2 Documentation', 144 | [author], 1) 145 | ] 146 | 147 | # -- Options for Texinfo output ------------------------------------------- 148 | 149 | # Grouping the document tree into Texinfo files. List of tuples 150 | # (source start file, target name, title, author, 151 | # dir menu entry, description, category) 152 | texinfo_documents = [ 153 | (master_doc, 'scrapenhl2', 'scrapenhl2 Documentation', 154 | author, 'scrapenhl2', 'One line description of project.', 155 | 'Miscellaneous'), 156 | ] 157 | -------------------------------------------------------------------------------- /scrapenhl2/plot/team_score_state_toi.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods for making a stacked bar graph indicating how much TOI each team spends in score states. 3 | """ 4 | 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | import scrapenhl2.manipulate.manipulate as manip 9 | import scrapenhl2.scrape.team_info as team_info 10 | import scrapenhl2.plot.visualization_helper as vhelper 11 | 12 | def score_state_graph(season): 13 | """ 14 | Generates a horizontal stacked bar graph showing how much 5v5 TOI each team has played in each score state 15 | for given season. 16 | 17 | :param season: int, the season 18 | 19 | :return: 20 | """ 21 | #TODO make kwargs match other methods: startseason, startdate, etc 22 | 23 | state_toi = manip.team_5v5_score_state_summary_by_game(season) \ 24 | .drop('Game', axis=1) \ 25 | .groupby(['Team', 'ScoreState'], as_index=False).sum() 26 | 27 | bar_positions = _score_state_graph_bar_positions(state_toi) 28 | bar_positions.loc[:, 'Team'] = bar_positions.Team.apply(lambda x: team_info.team_as_str(x)) 29 | 30 | plt.clf() 31 | tiedcolor, leadcolor, trailcolor = plt.rcParams['axes.prop_cycle'].by_key()['color'][:3] 32 | colors = {0: tiedcolor, 1: leadcolor, -1: trailcolor} 33 | for i in range(2, 4): 34 | colors[i] = vhelper.make_color_lighter(colors[i - 1]) 35 | colors[-1 * i] = vhelper.make_color_lighter(colors[-1 * i + 1]) 36 | for score in (-3, -2, -1, 0, 1, 2, 3): # bar_positions.ScoreState.unique(): 37 | score = int(score) 38 | if score == 3: 39 | label = 'Up 3+' 40 | elif score > 0: 41 | label = 'Up {0:d}'.format(score) 42 | elif score == 0: 43 | label = 'Tied' 44 | elif score == -3: 45 | label = 'Trail 3+' 46 | else: 47 | label = 'Trail {0:d}'.format(-1 * score) 48 | 49 | temp = bar_positions.query('ScoreState == {0:d}'.format(score)) 50 | alpha = 0.5 51 | plt.barh(bottom=temp.Y.values, width=temp.Width.values, left=temp.Left.values, label=label, alpha=alpha, 52 | color=colors[score]) 53 | 54 | for _, y, team in bar_positions[['Y', 'Team']].drop_duplicates().itertuples(): 55 | plt.annotate(team, xy=(0, y), ha='center', va='center', fontsize=6) 56 | 57 | plt.ylim(-1, len(bar_positions.Team.unique())) 58 | plt.legend(loc='lower right', fontsize=8) 59 | plt.yticks([]) 60 | for spine in ['right', 'left', 'top', 'bottom']: 61 | plt.gca().spines[spine].set_visible(False) 62 | plt.title(get_score_state_graph_title(season)) 63 | 64 | lst = list(np.arange(-0.6, 0.61, 0.2)) 65 | plt.xticks(lst, ['{0:d}%'.format(abs(int(round(100 * x)))) for x in lst]) 66 | plt.show() 67 | 68 | 69 | def _order_for_score_state_graph(toidf): 70 | """ 71 | Want to arrange teams so top team has most time leading minus trailing. 72 | 73 | This method sums over lead/trail, sorts, and arranges so the team with the largest (lead-trail) has the highest Y. 74 | 75 | :param toidf: dataframe, unique on team and score state 76 | 77 | :return: dataframe with team and Y 78 | """ 79 | temp = toidf.assign(LeadTrail=toidf.ScoreState.apply(lambda x: 'Lead' if x > 0 else 'Trail')) \ 80 | .query("ScoreState != 0") \ 81 | [['Team', 'LeadTrail', 'Secs']] \ 82 | .groupby(['Team', 'LeadTrail'], as_index=False) \ 83 | .sum() \ 84 | .pivot(index='Team', columns='LeadTrail', values='Secs') \ 85 | .reset_index() 86 | temp = temp.assign(Diff=temp.Lead - temp.Trail).sort_values('Diff').assign(Y=1) 87 | temp.loc[:, 'Y'] = temp.Y.cumsum() - 1 88 | return temp[['Team', 'Y']] 89 | 90 | 91 | def _score_state_graph_bar_positions(toidf): 92 | """ 93 | Figures out where bars should start and stop so that the y-axis bisects the "tied" bar. 94 | 95 | :param toidf: 96 | 97 | :return: 98 | """ 99 | 100 | totaltoi = toidf[['Team', 'Secs']].groupby('Team', as_index=False).sum().rename(columns={'Secs': 'TotalTOI'}) 101 | 102 | # Trim score states to -3 to 3 103 | toidf.loc[:, 'ScoreState'] = toidf.ScoreState.apply(lambda x: max(-3, min(3, x))) 104 | toidf = toidf.groupby(['Team', 'ScoreState'], as_index=False).sum() 105 | 106 | # Change numbers to fractions of 100% 107 | df = toidf.merge(totaltoi, how='left', on='Team') 108 | df = df.assign(FracTOI=df.Secs / df.TotalTOI) \ 109 | .drop({'Secs', 'TotalTOI'}, axis=1) \ 110 | .rename(columns={'FracTOI': 'Width'}) \ 111 | .sort_values('ScoreState') 112 | 113 | # Take cumsums for the left in a barh 114 | df.loc[:, 'Left'] = df[['Team', 'Width']].groupby('Team', as_index=False).cumsum().Width 115 | df.loc[:, 'Left'] = df.Left - df.Width # because cumsum is inclusive, no remove it 116 | 117 | # Shift them over so the center of the tied bar is at zero 118 | zeroes = df.query('ScoreState == 0') 119 | zeroes = zeroes.assign(Shift=zeroes.Left + zeroes.Width / 2)[['Team', 'Shift']] 120 | 121 | # Shift 122 | df = df.merge(zeroes, how='left', on='Team') 123 | df.loc[:, 'Left'] = df.Left - df.Shift 124 | df = df.drop('Shift', axis=1) 125 | 126 | # Check that zeroes are centered 127 | tempdf = df.query('ScoreState == 0') 128 | tempdf = tempdf.assign(Diff=tempdf.Left * 2 + tempdf.Width) 129 | assert np.isclose(0, tempdf.Diff.sum()) # sometimes have little float nonzeroes, like 1e-16 130 | 131 | return df.merge(_order_for_score_state_graph(toidf), how='left', on='Team').sort_values('Y') 132 | 133 | 134 | def get_score_state_graph_title(season): 135 | """ 136 | 137 | :param season: int, the season 138 | 139 | :return: 140 | """ 141 | return 'Team 5v5 TOI by score state in {0:d}-{1:s}'.format(season, str(season + 1)[2:]) -------------------------------------------------------------------------------- /docs/build/scrape_game.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrape_game module — scrapenhl2 0.4.0 documentation 10 | 11 | 12 | 13 | 14 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 44 | 45 |
46 |
47 |
48 |
49 | 50 |
51 | 52 |

scrape_game module

54 | 55 | 56 | 57 | 58 | 59 | 60 | 79 | 80 | 81 |
members: 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 |
undoc-members:
show-inheritance:
 
78 |
82 |
83 | 84 | 85 |
86 |
87 |
88 | 109 |
110 |
111 | 124 | 128 | 129 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/manipulate_schedules.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods related to generating and manipulating schedules. 3 | """ 4 | 5 | import scrapenhl2.scrape.general_helpers as helpers 6 | import scrapenhl2.scrape.schedules as schedules 7 | 8 | 9 | def update_schedule_with_result(season, game, result): 10 | """ 11 | Updates the season schedule file with game result (which are listed 'N/A' at schedule generation) 12 | 13 | :param season: int, the season 14 | :param game: int, the game 15 | :param result: str, the result from home team perspective 16 | 17 | :return: 18 | """ 19 | 20 | # Replace coaches with N/A if None b/c feather has trouble with mixed datatypes. Need str here. 21 | if result is None: 22 | result = 'N/A' 23 | 24 | # Edit relevant schedule files 25 | df = schedules.get_season_schedule(season) 26 | df.loc[df.Game == game, 'Result'] = result 27 | 28 | # Write to file and refresh schedule in memory 29 | schedules.write_season_schedule(df, season, True) 30 | 31 | 32 | def _update_schedule_with_coaches(season, game, homecoach, roadcoach): 33 | """ 34 | Updates the season schedule file with given coaches' names (which are listed 'N/A' at schedule generation) 35 | 36 | :param season: int, the season 37 | :param game: int, the game 38 | :param homecoach: str, the home coach name 39 | :param roadcoach: str, the road coach name 40 | 41 | :return: 42 | """ 43 | 44 | # Replace coaches with N/A if None b/c feather has trouble with mixed datatypes. Need str here. 45 | if homecoach is None: 46 | homecoach = 'N/A' 47 | if roadcoach is None: 48 | roadcoach = 'N/A' 49 | 50 | # Edit relevant schedule files 51 | df = schedules.get_season_schedule(season) 52 | df.loc[df.Game == game, 'HomeCoach'] = homecoach 53 | df.loc[df.Game == game, 'RoadCoach'] = roadcoach 54 | 55 | # Write to file and refresh schedule in memory 56 | schedules.write_season_schedule(df, season, True) 57 | 58 | 59 | def update_schedule_with_pbp_scrape(season, game): 60 | """ 61 | Updates the schedule file saying that specified game's pbp has been scraped. 62 | 63 | :param season: int, the season 64 | :param game: int, the game, or list of ints 65 | 66 | :return: updated schedule 67 | """ 68 | df = schedules.get_season_schedule(season) 69 | if helpers.check_types(game): 70 | df.loc[df.Game == game, "PBPStatus"] = "Scraped" 71 | else: 72 | df.loc[df.Game.isin(game), "PBPStatus"] = "Scraped" 73 | schedules.write_season_schedule(df, season, True) 74 | return schedules.get_season_schedule(season) 75 | 76 | 77 | def update_schedule_with_toi_scrape(season, game): 78 | """ 79 | Updates the schedule file saying that specified game's toi has been scraped. 80 | 81 | :param season: int, the season 82 | :param game: int, the game, or list of int 83 | 84 | :return: nothing 85 | """ 86 | df = schedules.get_season_schedule(season) 87 | if helpers.check_types(game): 88 | df.loc[df.Game == game, "TOIStatus"] = "Scraped" 89 | else: 90 | df.loc[df.Game.isin(game), "TOIStatus"] = "Scraped" 91 | schedules.write_season_schedule(df, season, True) 92 | return schedules.get_season_schedule(season) 93 | 94 | 95 | def update_schedule_with_result_using_pbp(pbp, season, game): 96 | """ 97 | Uses the PbP to update results for this game. 98 | 99 | :param pbp: json, the pbp for this game 100 | :param season: int, the season 101 | :param game: int, the game 102 | 103 | :return: nothing 104 | """ 105 | 106 | gameinfo = schedules.get_game_data_from_schedule(season, game) 107 | result = None # In case they have the same score. Like 2006 10009 has incomplete data, shows 0-0 108 | 109 | # If game is not final yet, don't do anything 110 | if gameinfo['Status'] != 'Final': 111 | return False 112 | 113 | # If one team one by at least two, we know it was a regulation win 114 | if gameinfo['HomeScore'] >= gameinfo['RoadScore'] + 2: 115 | result = 'W' 116 | elif gameinfo['RoadScore'] >= gameinfo['HomeScore'] + 2: 117 | result = 'L' 118 | else: 119 | # Check for the final period 120 | finalplayperiod = helpers.try_to_access_dict(pbp, 'liveData', 'linescore', 'currentPeriodOrdinal') 121 | 122 | # Identify SO vs OT vs regulation 123 | if finalplayperiod is None: 124 | pass 125 | elif finalplayperiod == 'SO': 126 | if gameinfo['HomeScore'] > gameinfo['RoadScore']: 127 | result = 'SOW' 128 | elif gameinfo['RoadScore'] > gameinfo['HomeScore']: 129 | result = 'SOL' 130 | elif finalplayperiod[-2:] == 'OT': 131 | if gameinfo['HomeScore'] > gameinfo['RoadScore']: 132 | result = 'OTW' 133 | elif gameinfo['RoadScore'] > gameinfo['HomeScore']: 134 | result = 'OTL' 135 | else: 136 | if gameinfo['HomeScore'] > gameinfo['RoadScore']: 137 | result = 'W' 138 | elif gameinfo['RoadScore'] > gameinfo['HomeScore']: 139 | result = 'L' 140 | 141 | update_schedule_with_result(season, game, result) 142 | 143 | 144 | def update_schedule_with_coaches(pbp, season, game): 145 | """ 146 | Uses the PbP to update coach info for this game. 147 | 148 | :param pbp: json, the pbp for this game 149 | :param season: int, the season 150 | :param game: int, the game 151 | 152 | :return: nothing 153 | """ 154 | 155 | homecoach = helpers.try_to_access_dict(pbp, 'liveData', 'boxscore', 'teams', 'home', 'coaches', 0, 'person', 156 | 'fullName') 157 | roadcoach = helpers.try_to_access_dict(pbp, 'liveData', 'boxscore', 'teams', 'away', 'coaches', 0, 'person', 158 | 'fullName') 159 | _update_schedule_with_coaches(season, game, homecoach, roadcoach) 160 | -------------------------------------------------------------------------------- /docs/build/scrape_setup.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | scrape_setup module — scrapenhl2 0.4.0 documentation 10 | 11 | 12 | 13 | 14 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 44 | 45 |
46 |
47 |
48 |
49 | 50 |
51 | 52 |

scrape_setup module

54 |

This module contains several helpful methods for accessing files. 55 | At import, this module creates folders for data storage if need be.

56 |

It also creates a team ID mapping and schedule files from 2005 through the current season (if the 57 | files do not exist).

58 | 59 | 60 | 61 | 62 | 63 | 64 | 83 | 84 | 85 |
members: 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 |
undoc-members:
show-inheritance:
 
82 |
86 |
87 | 88 | 89 |
90 |
91 |
92 | 113 |
114 |
115 | 128 | 132 | 133 | -------------------------------------------------------------------------------- /scrapenhl2/plot/app/game_page.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains the information needed to create the game pages in the app. 3 | 4 | The page has a dropdown for the season, dropdown for the game, a radio button to select chart type, 5 | and a button to update. 6 | """ 7 | import os 8 | 9 | import dash 10 | import dash_core_components as dcc 11 | import dash_html_components as html 12 | from dash.dependencies import Input, Output 13 | import flask 14 | 15 | import scrapenhl2.scrape.schedules as schedules 16 | import scrapenhl2.scrape.team_info as team_info 17 | import scrapenhl2.scrape.organization as organization 18 | import scrapenhl2.plot.game_h2h as game_h2h 19 | import scrapenhl2.plot.game_timeline as game_timeline 20 | 21 | def get_images_url(): 22 | """Returns /static/""" 23 | return '/static/' 24 | 25 | 26 | def get_game_images_url(): 27 | """Returns /static/game/""" 28 | return '{0:s}{1:s}'.format(get_images_url(), 'game/') 29 | 30 | 31 | def get_game_image_url(season, game, charttype): 32 | """Returns /static/game/2017/20001/H2H.png for example""" 33 | return '{0:s}{1:d}/{2:d}/{3:s}.png'.format(get_game_images_url(), season, game, charttype) 34 | 35 | 36 | def get_images_folder(): 37 | """Returns scrapenhl2/plot/app/_static/""" 38 | return os.path.join(organization.get_base_dir(), 'plot', 'app', '_static') 39 | 40 | 41 | def clean_images_folder(): 42 | """Removes all files in scrapenhl2/plot/app/_static/""" 43 | filelist = [os.path.join(get_images_folder(), file) for file in os.listdir(get_images_folder())] 44 | for file in filelist: 45 | os.unlink(file) 46 | 47 | 48 | def get_game_image_filename(season, game, charttype): 49 | """Returns e.g. scrapenhl2/plot/app/_static/2017-20001-H2H.png""" 50 | return os.path.join(get_images_folder(), '{0:d}-{1:d}-{2:s}.png'.format(season, game, charttype)) 51 | 52 | 53 | def generate_table(dataframe): 54 | """Transforms a pandas dataframe into an HTML table""" 55 | return html.Table( 56 | # Header 57 | [html.Tr([html.Th(col) for col in dataframe.columns])] + 58 | 59 | # Body 60 | [html.Tr([html.Td(dataframe.iloc[i][col]) for col in dataframe.columns]) for i in range(len(dataframe))]) 61 | 62 | 63 | def reduced_schedule_dataframe(season): 64 | """Returns schedule[Date, Game, Road, Home, Status]""" 65 | sch = schedules.get_season_schedule(season).drop({'Season', 'PBPStatus', 'TOIStatus'}, axis=1) 66 | sch.loc[:, 'Home'] = sch.Home.apply(team_info.team_as_str) 67 | sch.loc[:, 'Road'] = sch.Road.apply(team_info.team_as_str) 68 | sch = sch[['Date', 'Game', 'Road', 'Home', 'Status']].query('Game >= 20001 & Game <= 30417') 69 | return sch 70 | 71 | def get_season_dropdown_options(): 72 | """Use for options in season dropdown""" 73 | options = [{'label': '{0:d}-{1:s}'.format(yr, str(yr + 1)[2:]), 74 | 'value': yr} for yr in range(2010, schedules.get_current_season()+1)] 75 | return options 76 | 77 | def get_game_dropdown_options_for_season(season): 78 | """Use for options in game dropdown""" 79 | sch = reduced_schedule_dataframe(season) 80 | options = [{'label': '{0:s}: {1:d} {2:s}@{3:s} ({4:s})'.format(date, game, road, home, status), 81 | 'value': game} for index, date, game, road, home, status in sch.itertuples()] 82 | return options 83 | 84 | def get_game_graph_types(): 85 | """Update this with more chart types for single games""" 86 | options = [{'label': 'Head-to-head', 'value': 'H2H'}, 87 | {'label': 'Game timeline', 'value': 'TL'}] 88 | return options 89 | 90 | #sch = reduced_schedule_dataframe(schedules.get_current_season()) 91 | clean_images_folder() 92 | 93 | app = dash.Dash() 94 | 95 | app.layout = html.Div(children=[html.H1(children='Welcome to the app for scrapenhl2'), 96 | html.Label('Select season'), 97 | dcc.Dropdown( 98 | options=get_season_dropdown_options(), 99 | value=schedules.get_current_season(), 100 | id='season-dropdown'), 101 | html.Label('Select game'), 102 | dcc.Dropdown( 103 | options=get_game_dropdown_options_for_season(schedules.get_current_season()), 104 | value=20001, 105 | id='game-dropdown'), 106 | html.Label('Select graph type'), 107 | dcc.RadioItems( 108 | id='game-graph-radio', 109 | options=get_game_graph_types(), 110 | value='H2H'), 111 | html.Img(id='image', width=800) 112 | ]) 113 | 114 | @app.callback(Output('game-dropdown', 'options'), [Input('season-dropdown', 'value')]) 115 | def update_game_dropdown_options_for_season(selected_season): 116 | return get_game_dropdown_options_for_season(selected_season) 117 | 118 | @app.callback(Output('image', 'src'), [Input('season-dropdown', 'value'), 119 | Input('game-dropdown', 'value'), 120 | Input('game-graph-radio', 'value')]) 121 | def update_game_graph(selected_season, selected_game, selected_chart): 122 | if selected_chart == 'H2H': 123 | game_h2h.game_h2h(selected_season, selected_game, 124 | save_file=get_game_image_filename(selected_season, selected_game, selected_chart)) 125 | elif selected_chart == 'TL': 126 | game_timeline.game_timeline(selected_season, selected_game, 127 | save_file=get_game_image_filename(selected_season, selected_game, selected_chart)) 128 | return get_game_image_url(selected_season, selected_game, selected_chart) 129 | 130 | 131 | @app.server.route('{0:s}//.png'.format(get_game_images_url())) 132 | def serve_game_image(season, game, charttype): 133 | fname = get_game_image_filename(int(season), int(game), charttype) 134 | fname = fname[fname.rfind('/') + 1:] 135 | return flask.send_from_directory(get_images_folder(), fname) 136 | 137 | 138 | def browse_game_charts(): 139 | print('Go to http://127.0.0.1:8050/') 140 | app.run_server(debug=True) 141 | -------------------------------------------------------------------------------- /scrapenhl2/plot/rolling_cf_gf.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module creates rolling CF% and GF% charts 3 | """ 4 | 5 | import matplotlib.pyplot as plt 6 | import matplotlib.dates as mdates 7 | import numpy as np # standard scientific python stack 8 | import pandas as pd # standard scientific python stack 9 | 10 | from scrapenhl2.scrape import players, schedules 11 | from scrapenhl2.plot import visualization_helper as vhelper 12 | 13 | def rolling_player_gf(player, **kwargs): 14 | """ 15 | Creates a graph with GF% and GF% off. Defaults to roll_len of 40. 16 | 17 | :param player: str or int, player to generate for 18 | :param kwargs: other filters. See scrapenhl2.plot.visualization_helper.get_and_filter_5v5_log for more information. 19 | 20 | :return: nothing, or figure 21 | """ 22 | if 'roll_len' not in kwargs: 23 | kwargs['roll_len'] = 40 24 | _rolling_player_f(player, 'G', **kwargs) 25 | 26 | 27 | def rolling_player_cf(player, **kwargs): 28 | """ 29 | Creates a graph with CF% and CF% off. Defaults to roll_len of 25. 30 | 31 | :param player: str or int, player to generate for 32 | :param kwargs: other filters. See scrapenhl2.plot.visualization_helper.get_and_filter_5v5_log for more information. 33 | 34 | :return: nothing, or figure 35 | """ 36 | if 'roll_len' not in kwargs: 37 | kwargs['roll_len'] = 25 38 | _rolling_player_f(player, 'C', **kwargs) 39 | 40 | 41 | def _rolling_player_f(player, gfcf, **kwargs): 42 | """ 43 | Creates a graph with CF% or GF% (on plus off). Use gfcf to indicate which one. 44 | 45 | :param player: str or int, player to generate for 46 | :param gfcf: str. Use 'G' for GF% and GF% Off and 'C' for CF% and CF% Off 47 | :param kwargs: other filters. See scrapenhl2.plot.visualization_helper.get_and_filter_5v5_log for more information. 48 | Use x='Date' to index on date instead of game number 49 | 50 | :return: nothing, or figure 51 | """ 52 | 53 | kwargs['player'] = player 54 | fa = vhelper.get_and_filter_5v5_log(**kwargs) 55 | 56 | df = pd.concat([fa[['Season', 'Game']], _calculate_f_rates(fa, gfcf)], axis=1) 57 | col_dict = {col[col.index(' ') + 1:]: col for col in df.columns if '%' in col} 58 | 59 | plt.close('all') 60 | 61 | df.loc[:, 'Game Number'] = 1 62 | df.loc[:, 'Game Number'] = df['Game Number'].cumsum() 63 | df = df.set_index('Game Number', drop=False) 64 | 65 | if 'x' in kwargs and kwargs['x'] == 'Date': 66 | df = schedules.attach_game_dates_to_dateframe(df) 67 | df.loc[:, 'Date'] = pd.to_datetime(df.Date) 68 | #df.loc[:, 'Date'] = pd.to_datetime(df.Date).dt.strftime('%b/%y') 69 | df = df.set_index(pd.DatetimeIndex(df['Date'])) 70 | plt.gca().xaxis_date() 71 | plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b\'%y')) 72 | plt.xlabel('Date') 73 | else: 74 | plt.xlabel('Game') 75 | kwargs['x'] = 'Game Number' 76 | 77 | series = gfcf + 'F%' 78 | series2 = gfcf + 'F% Off' 79 | 80 | # Avoid the long lines in offseason by setting first value in each season to None 81 | df.loc[:, 'PrevSeason'] = df.Season.shift(1) 82 | df.loc[:, 'PrevSeason'] = df.PrevSeason.fillna(df.Season - 1) 83 | df.loc[df.Season != df.PrevSeason, col_dict[series]] = None 84 | df.loc[df.Season != df.PrevSeason, col_dict[series2]] = None 85 | 86 | # Add YY-YY for top axis 87 | df.loc[:, 'TopLabel'] = df.Season.apply(lambda x: '{0:d}-{1:s} -->'.format(x, str(x+1)[2:])) 88 | 89 | plt.plot(df.index, df[col_dict[series]].values, label=series) 90 | plt.plot(df.index, df[col_dict[series2]].values, label=series2, ls='--') 91 | 92 | plt.legend(loc=1, fontsize=10) 93 | 94 | # Add seasons at top 95 | ax1 = plt.gca() 96 | ax2 = ax1.twiny() 97 | ax2.set_xlim(*ax1.get_xlim()) 98 | temp = df[df.Season != df.PrevSeason][[kwargs['x'], 'TopLabel']] 99 | ax2.tick_params(length=0, labelsize=8) 100 | ax2.set_xticks(temp.iloc[:, 0].values) 101 | ax2.set_xticklabels(temp.iloc[:, 1].values) 102 | for label in ax2.xaxis.get_majorticklabels(): 103 | label.set_horizontalalignment('left') 104 | for tick in ax2.xaxis.get_major_ticks(): 105 | tick.set_pad(-10) 106 | 107 | plt.title(_get_rolling_f_title(gfcf, **kwargs)) 108 | 109 | # axes 110 | 111 | plt.ylabel(gfcf + 'F%') 112 | plt.ylim(0.3, 0.7) 113 | plt.xlim(df.index.min(), df.index.max()) 114 | ticks = list(np.arange(0.3, 0.71, 0.05)) 115 | plt.yticks(ticks, ['{0:.0f}%'.format(100 * tick) for tick in ticks]) 116 | 117 | return vhelper.savefilehelper(**kwargs) 118 | 119 | 120 | def _calculate_f_rates(df, gfcf): 121 | """ 122 | Calculates GF% or CF% (plus off) 123 | 124 | :param dataframe: dataframe 125 | :param gfcf: str. Use 'G' for GF% and GF% Off and 'C' for CF% and CF% Off 126 | 127 | :return: dataframe 128 | """ 129 | 130 | # Select columns 131 | fa = df.filter(regex='\d+-game') 132 | cols_wanted = {gfcf + x for x in {'FON', 'FOFF', 'AON', 'AOFF'}} 133 | fa = fa.select(lambda colname: colname[colname.index(' ') + 1:] in cols_wanted, axis=1) 134 | 135 | # This is to help me select columns 136 | col_dict = {col[col.index(' ') + 1:]: col for col in fa.columns} 137 | 138 | # Transform 139 | prefix = col_dict[gfcf + 'FON'][:col_dict[gfcf + 'FON'].index(' ')] # e.g. 25-game 140 | fa.loc[:, '{0:s} {1:s}F%'.format(prefix, gfcf)] = fa[col_dict[gfcf + 'FON']] / \ 141 | (fa[col_dict[gfcf + 'FON']] + fa[col_dict[gfcf + 'AON']]) 142 | fa.loc[:, '{0:s} {1:s}F% Off'.format(prefix, gfcf)] = fa[col_dict[gfcf + 'FOFF']] / \ 143 | (fa[col_dict[gfcf + 'FOFF']] + 144 | fa[col_dict[gfcf + 'AOFF']]) 145 | 146 | # Keep only those columns 147 | fa = fa[['{0:s} {1:s}F%'.format(prefix, gfcf), '{0:s} {1:s}F% Off'.format(prefix, gfcf)]] 148 | 149 | return fa 150 | 151 | 152 | def _get_rolling_f_title(gfcf, **kwargs): 153 | """ 154 | Returns default title for this type of graph 155 | 156 | :param gfcf: str. Use 'G' for GF% and GF% Off and 'C' for CF% and CF% Off 157 | :param kwargs: 158 | 159 | :return: str, the title 160 | """ 161 | 162 | title = 'Rolling {0:d}-game rolling {1:s}F% for {2:s}'.format(kwargs['roll_len'], gfcf, 163 | players.player_as_str(kwargs['player'])) 164 | title += '\n{0:s} to {1:s}'.format(*(str(x) for x in vhelper.get_startdate_enddate_from_kwargs(**kwargs))) 165 | return title -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/muneebalam/scrapenhl2.svg?branch=master 2 | :target: https://travis-ci.org/muneebalam/scrapenhl2 3 | .. image:: https://coveralls.io/repos/github/muneebalam/scrapenhl2/badge.svg?branch=master 4 | :target: https://coveralls.io/github/muneebalam/scrapenhl2?branch=master 5 | .. image:: https://landscape.io/github/muneebalam/scrapenhl2/master/landscape.svg?style=flat 6 | :target: https://landscape.io/github/muneebalam/scrapenhl2/master 7 | :alt: Code Health 8 | .. image:: https://badge.fury.io/py/scrapenhl2.svg 9 | :target: https://badge.fury.io/py/scrapenhl2 10 | .. image:: https://api.codeclimate.com/v1/badges/63e04a03b3aab131e262/maintainability 11 | :target: https://codeclimate.com/github/muneebalam/scrapenhl2/maintainability 12 | :alt: Maintainability 13 | .. image:: https://readthedocs.org/projects/scrapenhl2/badge/?version=latest 14 | :target: https://readthedocs.org/projects/scrapenhl2/?badge=latest 15 | :alt: Documentation Status 16 | 17 | .. inclusion-marker-for-sphinx 18 | 19 | Introduction 20 | ------------ 21 | 22 | scrapenhl2 is a python package for scraping and manipulating NHL data pulled from the NHL website. 23 | 24 | Installation 25 | ------------ 26 | You need python3 and the python scientific stack (e.g. numpy, matplotlib, pandas, etc). 27 | Easiest way is to simply use `Anaconda `_. 28 | To be safe, make sure you have python 3.5+, matplotlib 2.0+, and pandas 0.20+. 29 | 30 | Next, if you are on Windows, you need to get python-Levenshtein. 31 | `You can find it here `_. Download the appropriate .whl 32 | file--connect your version of python with the "cp" you see and use the one with "amd64" if you have an AMD 33 | 64-bit processor--and navigate to your downloads folder in command line. For example:: 34 | 35 | cd 36 | cd muneebalam 37 | cd Downloads 38 | 39 | Next, install the whl file using pip:: 40 | 41 | pip install [insert filename here].whl 42 | 43 | (Sometimes, this errors out and says you need Visual Studio C++ tools. You can download and install the 2015 version 44 | `from here `_.) 45 | 46 | Now, all users can open up terminal or command line and enter:: 47 | 48 | pip install scrapenhl2 49 | 50 | (If you have multiple versions of python installed, you may need to alter that command slightly.) 51 | 52 | For now, installation should be pretty quick, but in the future it may take awhile 53 | (depending on how many past years' files I make part of the package). 54 | 55 | As far as coding environments go, I recommend jupyter notebook or 56 | `Pycharm Community `_. 57 | Some folks also like the PyDev plugin in Eclipse. The latter two are full-scale applications, while the former 58 | launches in your browser. Open up terminal or command line and run:: 59 | 60 | jupyter notebook 61 | 62 | Then navigate to your coding folder, start a new Python file, and you're good to go. 63 | 64 | Use 65 | --- 66 | 67 | *Note that because this is in pre-alpha/alpha, syntax and use may be buggy and subject to change.* 68 | 69 | On startup, when you have an internet connection and some games have gone final since you last used the package, 70 | open up your python environment and update:: 71 | 72 | from scrapenhl2.scrape import autoupdate 73 | autoupdate.autoupdate() 74 | 75 | Autoupdate should update you regularly on its progress; be patient. 76 | 77 | To get a game H2H, use:: 78 | 79 | from scrapenhl2.plot import game_h2h 80 | season = 2016 81 | game = 30136 82 | game_h2h.game_h2h(season, game) 83 | 84 | .. image:: _static/WSH-TOR_G6.png 85 | 86 | To get a game timeline, use:: 87 | 88 | from scrapenhl2.plot import game_timeline 89 | season = 2016 90 | game = 30136 91 | game_timeline.game_timeline(season, game) 92 | 93 | .. image:: _static/WSH-TOR_G6_timeline.png 94 | 95 | To get a player rolling CF% graph, use:: 96 | 97 | from scrapenhl2.plot import rolling_cf_gf 98 | player = 'Ovechkin' 99 | rolling_games = 25 100 | start_year = 2015 101 | end_year = 2017 102 | rolling_cf_gf.rolling_player_cf(player, rolling_games, start_year, end_year) 103 | 104 | .. image:: _static/Ovechkin_rolling_cf.png 105 | 106 | This package is targeted for script use, so I recommend familiarizing yourself with python. 107 | (This is not intended to be a replacement for a site like Corsica.) 108 | 109 | Look through the documentation at `Read the Docs `_ and the 110 | `examples on Github `_. 111 | Also always feel free to contact me with questions or suggestions. 112 | 113 | Contact 114 | ------- 115 | `Twitter 116 | `_. 117 | 118 | Collaboration 119 | ------------- 120 | 121 | I'm happy to partner with you in development efforts--just shoot me a message or submit a pull request. 122 | Please also let me know if you'd like to alpha- or beta-test my code. 123 | 124 | Donations 125 | --------- 126 | If you would like to support my work, please donate money to a charity of your choice. Many large charities do 127 | great work all around the world (e.g. Médecins Sans Frontières), 128 | but don't forget that your support is often more critical for local/small charities. 129 | Also consider that small regular donations are sometimes better than one large donation. 130 | 131 | You can vet a charity you're targeting using a `charity rating website `_. 132 | 133 | If you do make a donation, make me happy `and leave a record here `_.. 134 | (It's anonymous.) 135 | 136 | Change log 137 | ---------- 138 | 139 | 1/13/18: Various bug fixes, some charts added. 140 | 141 | 11/10/17: Switched from Flask to Dash, bug fixes. 142 | 143 | 11/5/17: Bug fixes and method to add on-ice players to file. More refactoring. 144 | 145 | 10/28/17: Major refactoring. Docs up and running. 146 | 147 | 10/21/17: Added basic front end. Committed early versions of 2017 logs. 148 | 149 | 10/16/17: Added initial versions of game timelines, player rolling corsi, and game H2H graphs. 150 | 151 | 10/10/17: Bug fixes on scraping and team logs. Started methods to aggregate 5v5 game-by-game data for players. 152 | 153 | 10/7/17: Committed code to scrape 2010 onward and create team logs; still bugs to fix. 154 | 155 | 9/24/17: Committed minimal structure. 156 | 157 | Major outstanding to-dos 158 | ------------------------ 159 | 160 | * Bring in old play by play and shifts from HTML 161 | * More examples 162 | * More graphs 163 | * More graphs in Dash app 164 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/autoupdate.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods for automatically scraping and parsing games. 3 | """ 4 | 5 | import os 6 | import os.path 7 | import requests 8 | from tqdm import tqdm 9 | from numba import jit 10 | 11 | import scrapenhl2.scrape.manipulate_schedules as manipulate_schedules 12 | import scrapenhl2.scrape.parse_pbp as parse_pbp 13 | import scrapenhl2.scrape.parse_toi as parse_toi 14 | import scrapenhl2.scrape.schedules as schedules 15 | import scrapenhl2.scrape.scrape_pbp as scrape_pbp 16 | import scrapenhl2.scrape.scrape_toi as scrape_toi 17 | import scrapenhl2.scrape.teams as teams 18 | 19 | 20 | def delete_game_html(season, game): 21 | """ 22 | Deletes html files. HTML files are used for live game charts, but deleted in favor of JSONs when games go final. 23 | 24 | :param season: int, the season 25 | 26 | :param game: int, the game 27 | 28 | :return: nothing 29 | """ 30 | 31 | for fun in (scrape_pbp.get_game_pbplog_filename, 32 | scrape_toi.get_home_shiftlog_filename, 33 | scrape_toi.get_road_shiftlog_filename): 34 | filename = fun(season, game) 35 | if os.path.exists(filename): 36 | os.remove(filename) 37 | 38 | 39 | def autoupdate(season=None, update_team_logs=True): 40 | """ 41 | Run this method to update local data. It reads the schedule file for given season and scrapes and parses 42 | previously unscraped games that have gone final or are in progress. Use this for 2010 or later. 43 | 44 | :param season: int, the season. If None (default), will do current season 45 | :param update_team_logs: bool, update team logs too? Faster if False. 46 | 47 | :return: nothing 48 | """ 49 | # TODO: why does sometimes the schedule have the wrong game-team pairs, but when I regenerate, it's all ok? 50 | # TODO: this does not work quite right. Doesn't seem to know it needs to re-scrape TOI for previously scraped 51 | # TODO: in-progress games after they go final 52 | 53 | if season is None: 54 | season = schedules.get_current_season() 55 | 56 | sch = schedules.get_season_schedule(season) 57 | 58 | # First, for all games that were in progress during last scrape, delete html charts 59 | inprogress = sch.query('Status == "In Progress"') 60 | inprogressgames = inprogress.Game.values 61 | inprogressgames.sort() 62 | for game in inprogressgames: 63 | delete_game_html(season, game) 64 | 65 | # Now keep tabs on old final games 66 | old_final_games = set(sch.query('Status == "Final" & Result != "N/A"').Game.values) 67 | 68 | # Update schedule to get current status 69 | schedules.generate_season_schedule_file(season) 70 | 71 | # For games done previously, set pbp and toi status to scraped 72 | manipulate_schedules.update_schedule_with_pbp_scrape(season, old_final_games) 73 | manipulate_schedules.update_schedule_with_toi_scrape(season, old_final_games) 74 | sch = schedules.get_season_schedule(season) 75 | 76 | # Now, for games currently in progress, scrape. 77 | # But no need to force-overwrite. We handled games previously in progress above. 78 | # Games newly in progress will be written to file here. 79 | 80 | inprogressgames = sch.query('Status == "In Progress"') 81 | inprogressgames = inprogressgames.Game.values 82 | inprogressgames.sort() 83 | print("Updating in-progress games") 84 | read_inprogress_games(inprogressgames, season) 85 | 86 | # Now, for any games that are final, scrape and parse if not previously done 87 | games = sch.query('Status == "Final" & Result == "N/A"') 88 | games = games.Game.values 89 | games.sort() 90 | print('Updating final games') 91 | read_final_games(games, season) 92 | 93 | if update_team_logs: 94 | try: 95 | teams.update_team_logs(season, force_overwrite=False) 96 | except Exception as e: 97 | pass # ed.print_and_log("Error with team logs in {0:d}: {1:s}".format(season, str(e)), 'warn') 98 | 99 | 100 | def read_final_games(games, season): 101 | """ 102 | 103 | :param games: 104 | :param season: 105 | 106 | :return: 107 | """ 108 | for game in tqdm(games, desc="Parsing Games"): 109 | try: 110 | scrape_pbp.scrape_game_pbp(season, game, True) 111 | manipulate_schedules.update_schedule_with_pbp_scrape(season, game) 112 | parse_pbp.parse_game_pbp(season, game, True) 113 | except requests.exceptions.HTTPError as he: 114 | print('Could not access pbp url for {0:d} {1:d}'.format(season, game)) 115 | print(str(he)) 116 | except requests.exceptions.ConnectionError as ue: 117 | print('Could not access pbp url for {0:d} {1:d}'.format(season, game)) 118 | print(str(ue)) 119 | except Exception as e: 120 | print(str(e)) 121 | try: 122 | # TODO update only a couple of days later from json and delete html and don't update with toi scrape until then 123 | if season < 2010: 124 | scrape_toi.scrape_game_toi_from_html(season, game, True) 125 | manipulate_schedules.update_schedule_with_toi_scrape(season, game) 126 | parse_toi.parse_game_toi_from_html(season, game, True) 127 | else: 128 | scrape_toi.scrape_game_toi(season, game, True) 129 | manipulate_schedules.update_schedule_with_toi_scrape(season, game) 130 | parse_toi.parse_game_toi(season, game, True) 131 | 132 | # If you scrape soon after a game the json only has like the first period for example. 133 | # If I don't have the full game, use html 134 | if len(parse_toi.get_parsed_toi(season, game)) < 3600: 135 | print('Not enough rows in json for {0:d} {1:d}; reading from html'.format(int(season), int(game))) 136 | scrape_toi.scrape_game_toi_from_html(season, game, True) 137 | parse_toi.parse_game_toi_from_html(season, game, True) 138 | except ( 139 | requests.exceptions.HTTPError, 140 | requests.exceptions.ReadTimeout, 141 | ) as he: 142 | print('Could not access toi url for {0:d} {1:d}'.format(season, game)) 143 | print(str(he)) 144 | except Exception as e: 145 | print(str(e)) 146 | 147 | print('Done with {0:d} {1:d} (final)'.format(season, game)) 148 | 149 | 150 | def read_inprogress_games(inprogressgames, season): 151 | """ 152 | Saves these games to file via html (for toi) and json (for pbp) 153 | 154 | :param inprogressgames: list of int 155 | 156 | :return: 157 | """ 158 | 159 | for game in inprogressgames: 160 | # scrape_game_pbp_from_html(season, game, False) 161 | # parse_game_pbp_from_html(season, game, False) 162 | # PBP JSON updates live, so I can just use that, as before 163 | scrape_pbp.scrape_game_pbp(season, game, True) 164 | scrape_toi.scrape_game_toi_from_html(season, game, True) 165 | parse_pbp.parse_game_pbp(season, game, True) 166 | parse_toi.parse_game_toi_from_html(season, game, True) 167 | print('Done with {0:d} {1:d} (in progress)'.format(season, game)) 168 | -------------------------------------------------------------------------------- /scrapenhl2/plot/team_score_shot_rate.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module creates a scatterplot for specified team with shot attempt rates versus league median from down 3 to up 3. 3 | """ 4 | 5 | import matplotlib.pyplot as plt 6 | import math 7 | import pandas as pd 8 | 9 | import scrapenhl2.scrape.team_info as team_info 10 | import scrapenhl2.manipulate.manipulate as manip 11 | import scrapenhl2.plot.visualization_helper as vhelper 12 | 13 | def team_score_shot_rate_parallel(team, startseason, endseason=None, save_file=None): 14 | """ 15 | 16 | :param team: 17 | :param startseason: 18 | :param endseason: 19 | :param save_file: 20 | :return: 21 | """ 22 | if endseason is None: 23 | endseason = startseason 24 | 25 | df = pd.concat([manip.team_5v5_shot_rates_by_score(season) for season in range(startseason, endseason + 1)]) 26 | 27 | df.loc[:, 'ScoreState'] = df.ScoreState.apply(lambda x: max(min(3, x), -3)) # reduce to +/- 3 28 | df = df.drop('Game', axis=1) \ 29 | .groupby(['Team', 'ScoreState'], as_index=False) \ 30 | .sum() 31 | df.loc[:, 'CF%'] = df.CF / (df.CF + df.CA) 32 | df = df[['Team', 'ScoreState', 'CF%']] \ 33 | .sort_values('ScoreState') 34 | 35 | statelabels = {x: 'Lead{0:d}'.format(x) if x >= 1 else 'Trail{0:d}'.format(abs(x)) for x in range(-3, 4)} 36 | statelabels[0] = 'Tied' 37 | df.loc[:, 'ScoreState'] = df.ScoreState.apply(lambda x: statelabels[x]) 38 | 39 | # Go to wide 40 | df = df.pivot_table(index='Team', columns='ScoreState', values='CF%').reset_index() 41 | 42 | # Reorder columns 43 | df = df[['Team', 'Trail3', 'Trail2', 'Trail1', 'Tied', 'Lead1', 'Lead2', 'Lead3']] 44 | 45 | # Teams to strings 46 | df.loc[:, 'Team'] = df.Team.apply(team_info.team_as_str) 47 | 48 | # filter for own team 49 | teamdf = df.query('Team == "{0:s}"'.format(team_info.team_as_str(team))) 50 | 51 | # Make parallel coords 52 | vhelper.parallel_coords(df, teamdf, 'Team') 53 | 54 | # Set yticklabels 55 | ys = (0.4, 0.5, 0.6) 56 | plt.yticks(ys, ['{0:d}%'.format(int(y * 100)) for y in ys]) 57 | plt.ylim(0.35, 0.65) 58 | 59 | plt.title(_team_score_shot_rate_parallel_title(team, startseason, endseason)) 60 | 61 | for direction in ['right', 'top', 'bottom', 'left']: 62 | plt.gca().spines[direction].set_visible(False) 63 | 64 | if save_file is None: 65 | plt.show() 66 | else: 67 | plt.savefig(save_file) 68 | 69 | 70 | def team_score_shot_rate_scatter(team, startseason, endseason=None, save_file=None): 71 | """ 72 | 73 | :param team: str or int, team 74 | :param startseason: int, the starting season (inclusive) 75 | :param endseason: int, the ending season (inclusive) 76 | 77 | :return: nothing 78 | """ 79 | 80 | if endseason is None: 81 | endseason = startseason 82 | 83 | df = pd.concat([manip.team_5v5_shot_rates_by_score(season) for season in range(startseason, endseason + 1)]) 84 | 85 | df.loc[:, 'ScoreState'] = df.ScoreState.apply(lambda x: max(min(3, x), -3)) # reduce to +/- 3 86 | df = df.drop('Game', axis=1) \ 87 | .groupby(['Team', 'ScoreState'], as_index=False) \ 88 | .sum() 89 | df.loc[:, 'CF60'] = df.CF * 3600 / df.Secs 90 | df.loc[:, 'CA60'] = df.CA * 3600 / df.Secs 91 | 92 | # get medians 93 | medians = df[['ScoreState', 'CF60', 'CA60', 'Secs']].groupby('ScoreState', as_index=False).median() 94 | 95 | # filter for own team 96 | teamdf = df.query('Team == {0:d}'.format(int(team_info.team_as_id(team)))) 97 | 98 | statelabels = {x: 'Lead {0:d}'.format(x) if x >= 1 else 'Trail {0:d}'.format(abs(x)) for x in range(-3, 4)} 99 | statelabels[0] = 'Tied' 100 | for state in range(-3, 4): 101 | teamxy = teamdf.query('ScoreState == {0:d}'.format(state)) 102 | teamx = teamxy.CF60.iloc[0] 103 | teamy = teamxy.CA60.iloc[0] 104 | 105 | leaguexy = medians.query('ScoreState == {0:d}'.format(state)) 106 | leaguex = leaguexy.CF60.iloc[0] 107 | leaguey = leaguexy.CA60.iloc[0] 108 | 109 | midx = (leaguex + teamx) / 2 110 | midy = (leaguey + teamy) / 2 111 | 112 | rot = _calculate_label_rotation(leaguex, leaguey, teamx, teamy) 113 | 114 | plt.annotate('', xy=(teamx, teamy), xytext=(leaguex, leaguey), xycoords='data', 115 | arrowprops={'arrowstyle': '-|>'}) 116 | plt.annotate(statelabels[state], xy=(midx, midy), ha="center", va="center", xycoords='data', size=8, 117 | rotation=rot, bbox=dict(boxstyle="round", fc="w", alpha=0.9)) 118 | 119 | plt.scatter(medians.CF60.values, medians.CA60.values, s=100, color='w') 120 | plt.scatter(teamdf.CF60.values, teamdf.CA60.values, s=100, color='w') 121 | 122 | #bbox_props = dict(boxstyle="round", fc="w", ec="0.5", alpha=0.9) 123 | #plt.annotate('Fast', xy=(0.95, 0.95), xycoords='axes fraction', bbox=bbox_props, ha='center', va='center') 124 | #plt.annotate('Slow', xy=(0.05, 0.05), xycoords='axes fraction', bbox=bbox_props, ha='center', va='center') 125 | #plt.annotate('Good', xy=(0.95, 0.05), xycoords='axes fraction', bbox=bbox_props, ha='center', va='center') 126 | #plt.annotate('Bad', xy=(0.05, 0.95), xycoords='axes fraction', bbox=bbox_props, ha='center', va='center') 127 | vhelper.add_good_bad_fast_slow() 128 | 129 | plt.xlabel('CF60') 130 | plt.ylabel('CA60') 131 | 132 | plt.title(_team_score_shot_rate_scatter_title(team, startseason, endseason)) 133 | 134 | if save_file is None: 135 | plt.show() 136 | else: 137 | plt.savefig(save_file) 138 | 139 | 140 | def _team_score_shot_rate_scatter_title(team, startseason, endseason): 141 | """ 142 | 143 | :param team: 144 | :param startseason: 145 | :param endseason: 146 | :return: 147 | """ 148 | return '{0:s} shot rate by score state, {1:s} to {2:s}'.format(team_info.team_as_str(team), 149 | *vhelper.get_startdate_enddate_from_kwargs( 150 | startseason=startseason, 151 | endseason=endseason)) 152 | 153 | 154 | def _team_score_shot_rate_parallel_title(team, startseason, endseason): 155 | """ 156 | 157 | :param team: 158 | :param startseason: 159 | :param endseason: 160 | :return: 161 | """ 162 | return '{0:s} CF% by score state\n{1:s} to {2:s}'.format(team_info.team_as_str(team), 163 | *vhelper.get_startdate_enddate_from_kwargs( 164 | startseason=startseason, 165 | endseason=endseason)) 166 | 167 | def _calculate_label_rotation(startx, starty, endx, endy): 168 | """ 169 | Calculates the appropriate rotation angle for a label on an arrow (matches line, is between -90 and 90 degrees) 170 | 171 | :param startx: start of arrow (x) 172 | :param starty: start of arrow (y) 173 | :param endx: end of arrow (x) 174 | :param endy: end of arrow (y) 175 | 176 | :return: rotation angle. 177 | """ 178 | return math.degrees(math.atan((endy - starty)/(endx - startx))) 179 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/scrape_pbp.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods for scraping pbp. 3 | """ 4 | 5 | import json 6 | import os.path 7 | import urllib.request 8 | import zlib 9 | from time import sleep 10 | 11 | from scrapenhl2.scrape import organization, schedules, general_helpers as helpers, manipulate_schedules, parse_pbp 12 | 13 | 14 | def scrape_game_pbp_from_html(season, game, force_overwrite=True): 15 | """ 16 | This method scrapes the html pbp for the given game. Use for live games. 17 | 18 | :param season: int, the season 19 | :param game: int, the game 20 | :param force_overwrite: bool. If file exists already, won't scrape again 21 | 22 | :return: bool, False if not scraped, else True 23 | """ 24 | filename = get_game_pbplog_filename(season, game) 25 | if not force_overwrite and os.path.exists(filename): 26 | return False 27 | 28 | page = get_game_from_url(season, game) 29 | save_raw_html_pbp(page, season, game) 30 | # ed.print_and_log('Scraped html pbp for {0:d} {1:d}'.format(season, game)) 31 | sleep(1) # Don't want to overload NHL servers 32 | 33 | # It's most efficient to parse with page in memory, but for sake of simplicity will do it later 34 | # pbp = read_pbp_events_from_page(page) 35 | # update_team_logs(pbp, season, schedule_item['Home']) 36 | return True 37 | 38 | 39 | def scrape_game_pbp(season, game, force_overwrite=False): 40 | """ 41 | This method scrapes the pbp for the given game. 42 | 43 | :param season: int, the season 44 | :param game: int, the game 45 | :param force_overwrite: bool. If file exists already, won't scrape again 46 | 47 | :return: bool, False if not scraped, else True 48 | """ 49 | filename = get_game_raw_pbp_filename(season, game) 50 | if not force_overwrite and os.path.exists(filename): 51 | return False 52 | 53 | # Use the season schedule file to get the home and road team names 54 | # schedule_item = get_files.get_season_schedule(season) \ 55 | # .query('Game == {0:d}'.format(game)) \ 56 | # .to_dict(orient = 'series') 57 | # The output format of above was {colname: np.array[vals]}. Change to {colname: val} 58 | # schedule_item = {k: v.values[0] for k, v in schedule_item.items()} 59 | 60 | page = get_game_from_url(season, game) 61 | save_raw_pbp(page, season, game) 62 | # ed.print_and_log('Scraped pbp for {0:d} {1:d}'.format(season, game)) 63 | sleep(1) # Don't want to overload NHL servers 64 | 65 | # It's most efficient to parse with page in memory, but for sake of simplicity will do it later 66 | # pbp = read_pbp_events_from_page(page) 67 | # update_team_logs(pbp, season, schedule_item['Home']) 68 | return True 69 | 70 | 71 | def save_raw_html_pbp(page, season, game): 72 | """ 73 | Takes the bytes page containing html pbp information and saves as such 74 | 75 | :param page: bytes 76 | :param season: int, the season 77 | :param game: int, the game 78 | 79 | :return: nothing 80 | """ 81 | filename = get_game_pbplog_filename(season, game) 82 | w = open(filename, 'w') 83 | w.write(page) 84 | w.close() 85 | 86 | 87 | def save_raw_pbp(page, season, game): 88 | """ 89 | Takes the bytes page containing pbp information and saves to disk as a compressed zlib. 90 | 91 | :param page: bytes. str(page) would yield a string version of the json pbp 92 | :param season: int, the season 93 | :param game: int, the game 94 | 95 | :return: nothing 96 | """ 97 | try: 98 | page2 = zlib.compress(page.encode('latin-1'), level=9) 99 | except TypeError: 100 | # No level kwarg before Python 3.6 101 | page2 = zlib.compress(page.encode('latin-1')) 102 | filename = get_game_raw_pbp_filename(season, game) 103 | w = open(filename, 'wb') 104 | w.write(page2) 105 | w.close() 106 | 107 | 108 | def get_raw_pbp(season, game): 109 | """ 110 | Loads the compressed json file containing this game's play by play from disk. 111 | 112 | :param season: int, the season 113 | :param game: int, the game 114 | 115 | :return: json, the json pbp 116 | """ 117 | with open(get_game_raw_pbp_filename(season, game), 'rb') as reader: 118 | page = reader.read() 119 | return json.loads(str(zlib.decompress(page).decode('latin-1'))) 120 | 121 | 122 | def get_raw_html_pbp(season, game): 123 | """ 124 | Loads the html file containing this game's play by play from disk. 125 | 126 | :param season: int, the season 127 | :param game: int, the game 128 | 129 | :return: str, the html pbp 130 | """ 131 | with open(get_game_pbplog_filename(season, game), 'r') as reader: 132 | page = reader.read() 133 | return page 134 | 135 | 136 | def get_game_from_url(season, game): 137 | """ 138 | Gets the page containing information for specified game from NHL API. 139 | 140 | :param season: int, the season 141 | :param game: int, the game 142 | 143 | :return: str, the page at the url 144 | """ 145 | 146 | return helpers.try_url_n_times(get_game_url(season, game)) 147 | 148 | 149 | def get_game_pbplog_url(season, game): 150 | """ 151 | Gets the url for a page containing pbp information for specified game from HTML tables. 152 | 153 | :param season: int, the season 154 | :param game: int, the game 155 | 156 | :return : str, e.g. http://www.nhl.com/scores/htmlreports/20072008/PL020001.HTM 157 | """ 158 | return 'http://www.nhl.com/scores/htmlreports/{0:d}{1:d}/PL0{2:d}.HTM'.format(season, season + 1, game) 159 | 160 | 161 | def get_game_url(season, game): 162 | """ 163 | Gets the url for a page containing information for specified game from NHL API. 164 | 165 | :param season: int, the season 166 | :param game: int, the game 167 | 168 | :return: str, https://statsapi.web.nhl.com/api/v1/game/[season]0[game]/feed/live 169 | """ 170 | return 'https://statsapi.web.nhl.com/api/v1/game/{0:d}0{1:d}/feed/live'.format(season, game) 171 | 172 | 173 | def get_game_raw_pbp_filename(season, game): 174 | """ 175 | Returns the filename of the raw pbp folder 176 | 177 | :param season: int, current season 178 | :param game: int, game 179 | 180 | :return: str, /scrape/data/raw/pbp/[season]/[game].zlib 181 | """ 182 | return os.path.join(organization.get_season_raw_pbp_folder(season), str(game) + '.zlib') 183 | 184 | 185 | def get_game_pbplog_filename(season, game): 186 | """ 187 | Returns the filename of the parsed pbp html game pbp 188 | 189 | :param season: int, current season 190 | :param game: int, game 191 | 192 | :return: str, /scrape/data/raw/pbp/[season]/[game].html 193 | """ 194 | return os.path.join(organization.get_season_raw_pbp_folder(season), str(game) + '.html') 195 | 196 | 197 | def scrape_season_pbp(season, force_overwrite=False): 198 | """ 199 | Scrapes and parses pbp from the given season. 200 | 201 | :param season: int, the season 202 | :param force_overwrite: bool. If true, rescrapes all games. If false, only previously unscraped ones 203 | 204 | :return: nothing 205 | """ 206 | if season is None: 207 | season = schedules.get_current_season() 208 | 209 | sch = schedules.get_season_schedule(season) 210 | games = sch[sch.Status == "Final"].Game.values 211 | games.sort() 212 | intervals = helpers.intervals(games) 213 | interval_j = 0 214 | for i, game in enumerate(games): 215 | try: 216 | scrape_game_pbp(season, game, force_overwrite) 217 | manipulate_schedules.update_schedule_with_pbp_scrape(season, game) 218 | parse_pbp.parse_game_pbp(season, game, True) 219 | except Exception as e: 220 | pass # ed.print_and_log('{0:d} {1:d} {2:s}'.format(season, game, str(e)), 'warn') 221 | if interval_j < len(intervals): 222 | if i == intervals[interval_j][0]: 223 | print('Done scraping through {0:d} {1:d} ({2:d}%)'.format( 224 | season, game, round(intervals[interval_j][0] / len(games) * 100))) 225 | interval_j += 1 226 | 227 | 228 | def scrape_pbp_setup(): 229 | """ 230 | Creates raw pbp folders if need be 231 | 232 | :return: 233 | """ 234 | for season in range(2005, schedules.get_current_season() + 1): 235 | organization.check_create_folder(organization.get_season_raw_pbp_folder(season)) 236 | 237 | 238 | scrape_pbp_setup() 239 | -------------------------------------------------------------------------------- /scrapenhl2/scrape/scrape_toi.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains methods for scraping TOI. 3 | """ 4 | 5 | import json 6 | import os.path 7 | import urllib.request 8 | import zlib 9 | from time import sleep 10 | 11 | from scrapenhl2.scrape import organization, schedules, manipulate_schedules, general_helpers as helpers, parse_toi 12 | 13 | 14 | def scrape_game_toi(season, game, force_overwrite=False): 15 | """ 16 | This method scrapes the toi for the given game. 17 | 18 | :param season: int, the season 19 | :param game: int, the game 20 | :param force_overwrite: bool. If file exists already, won't scrape again 21 | 22 | :return: nothing 23 | """ 24 | filename = get_game_raw_toi_filename(season, game) 25 | if not force_overwrite and os.path.exists(filename): 26 | return False 27 | 28 | page = helpers.try_url_n_times(get_shift_url(season, game)) 29 | save_raw_toi(page, season, game) 30 | # ed.print_and_log('Scraped toi for {0:d} {1:d}'.format(season, game)) 31 | sleep(1) # Don't want to overload NHL servers 32 | 33 | # It's most efficient to parse with page in memory, but for sake of simplicity will do it later 34 | # toi = read_toi_from_page(page) 35 | return True 36 | 37 | 38 | def get_home_shiftlog_filename(season, game): 39 | """ 40 | Returns the filename of the parsed toi html home shifts 41 | 42 | :param season: int, the season 43 | :param game: int, the game 44 | 45 | :return: str, /scrape/data/raw/pbp/[season]/[game]H.html 46 | """ 47 | return os.path.join(organization.get_season_raw_toi_folder(season), str(game) + 'H.html') 48 | 49 | 50 | def get_road_shiftlog_filename(season, game): 51 | """ 52 | Returns the filename of the parsed toi html road shifts 53 | 54 | :param season: int, current season 55 | :param game: int, game 56 | :return: str, /scrape/data/raw/pbp/[season]/[game]H.html 57 | """ 58 | return os.path.join(organization.get_season_raw_toi_folder(season), str(game) + 'R.html') 59 | 60 | 61 | def scrape_game_toi_from_html(season, game, force_overwrite=True): 62 | """ 63 | This method scrapes the toi html logs for the given game. 64 | 65 | :param season: int, the season 66 | :param game: int, the game 67 | :param force_overwrite: bool. If file exists already, won't scrape again 68 | 69 | :return: nothing 70 | """ 71 | filenames = (get_home_shiftlog_filename(season, game), get_road_shiftlog_filename(season, game)) 72 | urls = (get_home_shiftlog_url(season, game), get_road_shiftlog_url(season, game)) 73 | filetypes = ('H', 'R') 74 | for i in range(2): 75 | filename = filenames[i] 76 | if not force_overwrite and os.path.exists(filename): 77 | pass 78 | 79 | page = helpers.try_url_n_times(urls[i]) 80 | save_raw_toi_from_html(page, season, game, filetypes[i]) 81 | sleep(1) # Don't want to overload NHL servers 82 | print('Scraped html toi for {0:d} {1:d}'.format(season, game)) 83 | 84 | 85 | def save_raw_toi(page, season, game): 86 | """ 87 | Takes the bytes page containing shift information and saves to disk as a compressed zlib. 88 | 89 | :param page: bytes. str(page) would yield a string version of the json shifts 90 | :param season: int, the season 91 | :param game: int, the game 92 | 93 | :return: nothing 94 | """ 95 | try: 96 | page2 = zlib.compress(page.encode('latin-1'), level=9) 97 | except TypeError: 98 | # No level kwarg before Python 3.6 99 | page2 = zlib.compress(page.encode('latin-1')) 100 | filename = get_game_raw_toi_filename(season, game) 101 | w = open(filename, 'wb') 102 | w.write(page2) 103 | w.close() 104 | 105 | 106 | def save_raw_toi_from_html(page, season, game, homeroad): 107 | """ 108 | Takes the bytes page containing shift information and saves to disk as html. 109 | 110 | :param page: bytes. str(page) would yield a string version of the json shifts 111 | :param season: int, he season 112 | :param game: int, the game 113 | :param homeroad: str, 'H' or 'R' 114 | 115 | :return: nothing 116 | """ 117 | if homeroad == 'H': 118 | filename = get_home_shiftlog_filename(season, game) 119 | elif homeroad == 'R': 120 | filename = get_road_shiftlog_filename(season, game) 121 | w = open(filename, 'w') 122 | if type(page) != str: 123 | page = page.decode('latin-1') 124 | w.write(page) 125 | w.close() 126 | 127 | 128 | def get_raw_html_toi(season, game, homeroad): 129 | """ 130 | Loads the html file containing this game's toi from disk. 131 | 132 | :param season: int, the season 133 | :param game: int, the game 134 | :param homeroad: str, 'H' for home or 'R' for road 135 | 136 | :return: str, the html toi 137 | """ 138 | if homeroad == 'H': 139 | filename = get_home_shiftlog_filename(season, game) 140 | elif homeroad == 'R': 141 | filename = get_road_shiftlog_filename(season, game) 142 | with open(filename, 'r') as reader: 143 | page = reader.read() 144 | return page 145 | 146 | 147 | def get_raw_toi(season, game): 148 | """ 149 | Loads the compressed json file containing this game's shifts from disk. 150 | 151 | :param season: int, the season 152 | :param game: int, the game 153 | 154 | :return: dict, the json shifts 155 | """ 156 | with open(get_game_raw_toi_filename(season, game), 'rb') as reader: 157 | page = reader.read() 158 | return json.loads(str(zlib.decompress(page).decode('latin-1'))) 159 | 160 | 161 | def get_home_shiftlog_url(season, game): 162 | """ 163 | Gets the url for a page containing shift information for specified game from HTML tables for home team. 164 | 165 | :param season: int, the season 166 | :param game: int, the game 167 | 168 | :return : str, e.g. http://www.nhl.com/scores/htmlreports/20072008/TH020001.HTM 169 | """ 170 | return 'http://www.nhl.com/scores/htmlreports/{0:d}{1:d}/TH0{2:d}.HTM'.format(season, season + 1, game) 171 | 172 | 173 | def get_road_shiftlog_url(season, game): 174 | """ 175 | Gets the url for a page containing shift information for specified game from HTML tables for road team. 176 | 177 | :param season: int, the season 178 | :param game: int, the game 179 | 180 | :return : str, e.g. http://www.nhl.com/scores/htmlreports/20072008/TV020001.HTM 181 | """ 182 | return 'http://www.nhl.com/scores/htmlreports/{0:d}{1:d}/TV0{2:d}.HTM'.format(season, season + 1, game) 183 | 184 | 185 | def get_shift_url(season, game): 186 | """ 187 | Gets the url for a page containing shift information for specified game from NHL API. 188 | 189 | :param season: int, the season 190 | :param game: int, the game 191 | 192 | :return : str, http://www.nhl.com/stats/rest/shiftcharts?cayenneExp=gameId=[season]0[game] 193 | """ 194 | return 'http://www.nhl.com/stats/rest/shiftcharts?cayenneExp=gameId={0:d}0{1:d}'.format(season, game) 195 | 196 | 197 | def get_game_raw_toi_filename(season, game): 198 | """ 199 | Returns the filename of the raw toi folder 200 | 201 | :param season: int, current season 202 | :param game: int, game 203 | 204 | :return: str, /scrape/data/raw/toi/[season]/[game].zlib 205 | """ 206 | return os.path.join(organization.get_season_raw_toi_folder(season), str(game) + '.zlib') 207 | 208 | 209 | def scrape_season_toi(season, force_overwrite=False): 210 | """ 211 | Scrapes and parses toi from the given season. 212 | 213 | :param season: int, the season 214 | :param force_overwrite: bool. If true, rescrapes all games. If false, only previously unscraped ones 215 | 216 | :return: nothing 217 | """ 218 | if season is None: 219 | season = schedules.get_current_season() 220 | 221 | sch = schedules.get_season_schedule(season) 222 | games = sch[sch.Status == "Final"].Game.values 223 | games.sort() 224 | intervals = helpers.intervals(games) 225 | interval_j = 0 226 | for i, game in enumerate(games): 227 | try: 228 | scrape_game_toi(season, game, force_overwrite) 229 | manipulate_schedules.update_schedule_with_pbp_scrape(season, game) 230 | parse_toi.parse_game_pbp(season, game, True) 231 | if len(parse_toi.get_parsed_toi(season, game)) < 3600: 232 | scrape_game_toi_from_html(season, game, True) 233 | parse_toi.parse_game_toi_from_html(season, game, True) 234 | except Exception as e: 235 | pass # ed.print_and_log('{0:d} {1:d} {2:s}'.format(season, game, str(e)), 'warn') 236 | if interval_j < len(intervals): 237 | if i == intervals[interval_j][0]: 238 | print('Done scraping through {0:d} {1:d} ({2:d}%)'.format( 239 | season, game, round(intervals[interval_j][0] / len(games) * 100))) 240 | interval_j += 1 241 | 242 | 243 | def scrape_toi_setup(): 244 | """ 245 | Creates raw toi folders if need be 246 | 247 | :return: 248 | """ 249 | for season in range(2005, schedules.get_current_season() + 1): 250 | organization.check_create_folder(organization.get_season_raw_toi_folder(season)) 251 | 252 | 253 | scrape_toi_setup() 254 | -------------------------------------------------------------------------------- /docs/build/_static/doctools.js: -------------------------------------------------------------------------------- 1 | /* 2 | * doctools.js 3 | * ~~~~~~~~~~~ 4 | * 5 | * Sphinx JavaScript utilities for all documentation. 6 | * 7 | * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /** 13 | * select a different prefix for underscore 14 | */ 15 | $u = _.noConflict(); 16 | 17 | /** 18 | * make the code below compatible with browsers without 19 | * an installed firebug like debugger 20 | if (!window.console || !console.firebug) { 21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir", 22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", 23 | "profile", "profileEnd"]; 24 | window.console = {}; 25 | for (var i = 0; i < names.length; ++i) 26 | window.console[names[i]] = function() {}; 27 | } 28 | */ 29 | 30 | /** 31 | * small helper function to urldecode strings 32 | */ 33 | jQuery.urldecode = function(x) { 34 | return decodeURIComponent(x).replace(/\+/g, ' '); 35 | }; 36 | 37 | /** 38 | * small helper function to urlencode strings 39 | */ 40 | jQuery.urlencode = encodeURIComponent; 41 | 42 | /** 43 | * This function returns the parsed url parameters of the 44 | * current request. Multiple values per key are supported, 45 | * it will always return arrays of strings for the value parts. 46 | */ 47 | jQuery.getQueryParameters = function(s) { 48 | if (typeof s == 'undefined') 49 | s = document.location.search; 50 | var parts = s.substr(s.indexOf('?') + 1).split('&'); 51 | var result = {}; 52 | for (var i = 0; i < parts.length; i++) { 53 | var tmp = parts[i].split('=', 2); 54 | var key = jQuery.urldecode(tmp[0]); 55 | var value = jQuery.urldecode(tmp[1]); 56 | if (key in result) 57 | result[key].push(value); 58 | else 59 | result[key] = [value]; 60 | } 61 | return result; 62 | }; 63 | 64 | /** 65 | * highlight a given string on a jquery object by wrapping it in 66 | * span elements with the given class name. 67 | */ 68 | jQuery.fn.highlightText = function(text, className) { 69 | function highlight(node) { 70 | if (node.nodeType == 3) { 71 | var val = node.nodeValue; 72 | var pos = val.toLowerCase().indexOf(text); 73 | if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { 74 | var span = document.createElement("span"); 75 | span.className = className; 76 | span.appendChild(document.createTextNode(val.substr(pos, text.length))); 77 | node.parentNode.insertBefore(span, node.parentNode.insertBefore( 78 | document.createTextNode(val.substr(pos + text.length)), 79 | node.nextSibling)); 80 | node.nodeValue = val.substr(0, pos); 81 | } 82 | } 83 | else if (!jQuery(node).is("button, select, textarea")) { 84 | jQuery.each(node.childNodes, function() { 85 | highlight(this); 86 | }); 87 | } 88 | } 89 | return this.each(function() { 90 | highlight(this); 91 | }); 92 | }; 93 | 94 | /* 95 | * backward compatibility for jQuery.browser 96 | * This will be supported until firefox bug is fixed. 97 | */ 98 | if (!jQuery.browser) { 99 | jQuery.uaMatch = function(ua) { 100 | ua = ua.toLowerCase(); 101 | 102 | var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || 103 | /(webkit)[ \/]([\w.]+)/.exec(ua) || 104 | /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || 105 | /(msie) ([\w.]+)/.exec(ua) || 106 | ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || 107 | []; 108 | 109 | return { 110 | browser: match[ 1 ] || "", 111 | version: match[ 2 ] || "0" 112 | }; 113 | }; 114 | jQuery.browser = {}; 115 | jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; 116 | } 117 | 118 | /** 119 | * Small JavaScript module for the documentation. 120 | */ 121 | var Documentation = { 122 | 123 | init : function() { 124 | this.fixFirefoxAnchorBug(); 125 | this.highlightSearchWords(); 126 | this.initIndexTable(); 127 | 128 | }, 129 | 130 | /** 131 | * i18n support 132 | */ 133 | TRANSLATIONS : {}, 134 | PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, 135 | LOCALE : 'unknown', 136 | 137 | // gettext and ngettext don't access this so that the functions 138 | // can safely bound to a different name (_ = Documentation.gettext) 139 | gettext : function(string) { 140 | var translated = Documentation.TRANSLATIONS[string]; 141 | if (typeof translated == 'undefined') 142 | return string; 143 | return (typeof translated == 'string') ? translated : translated[0]; 144 | }, 145 | 146 | ngettext : function(singular, plural, n) { 147 | var translated = Documentation.TRANSLATIONS[singular]; 148 | if (typeof translated == 'undefined') 149 | return (n == 1) ? singular : plural; 150 | return translated[Documentation.PLURALEXPR(n)]; 151 | }, 152 | 153 | addTranslations : function(catalog) { 154 | for (var key in catalog.messages) 155 | this.TRANSLATIONS[key] = catalog.messages[key]; 156 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); 157 | this.LOCALE = catalog.locale; 158 | }, 159 | 160 | /** 161 | * add context elements like header anchor links 162 | */ 163 | addContextElements : function() { 164 | $('div[id] > :header:first').each(function() { 165 | $('\u00B6'). 166 | attr('href', '#' + this.id). 167 | attr('title', _('Permalink to this headline')). 168 | appendTo(this); 169 | }); 170 | $('dt[id]').each(function() { 171 | $('\u00B6'). 172 | attr('href', '#' + this.id). 173 | attr('title', _('Permalink to this definition')). 174 | appendTo(this); 175 | }); 176 | }, 177 | 178 | /** 179 | * workaround a firefox stupidity 180 | * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075 181 | */ 182 | fixFirefoxAnchorBug : function() { 183 | if (document.location.hash) 184 | window.setTimeout(function() { 185 | document.location.href += ''; 186 | }, 10); 187 | }, 188 | 189 | /** 190 | * highlight the search words provided in the url in the text 191 | */ 192 | highlightSearchWords : function() { 193 | var params = $.getQueryParameters(); 194 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; 195 | if (terms.length) { 196 | var body = $('div.body'); 197 | if (!body.length) { 198 | body = $('body'); 199 | } 200 | window.setTimeout(function() { 201 | $.each(terms, function() { 202 | body.highlightText(this.toLowerCase(), 'highlighted'); 203 | }); 204 | }, 10); 205 | $('

' + _('Hide Search Matches') + '

') 207 | .appendTo($('#searchbox')); 208 | } 209 | }, 210 | 211 | /** 212 | * init the domain index toggle buttons 213 | */ 214 | initIndexTable : function() { 215 | var togglers = $('img.toggler').click(function() { 216 | var src = $(this).attr('src'); 217 | var idnum = $(this).attr('id').substr(7); 218 | $('tr.cg-' + idnum).toggle(); 219 | if (src.substr(-9) == 'minus.png') 220 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 221 | else 222 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 223 | }).css('display', ''); 224 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 225 | togglers.click(); 226 | } 227 | }, 228 | 229 | /** 230 | * helper function to hide the search marks again 231 | */ 232 | hideSearchWords : function() { 233 | $('#searchbox .highlight-link').fadeOut(300); 234 | $('span.highlighted').removeClass('highlighted'); 235 | }, 236 | 237 | /** 238 | * make the url absolute 239 | */ 240 | makeURL : function(relativeURL) { 241 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 242 | }, 243 | 244 | /** 245 | * get the current relative url 246 | */ 247 | getCurrentURL : function() { 248 | var path = document.location.pathname; 249 | var parts = path.split(/\//); 250 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 251 | if (this == '..') 252 | parts.pop(); 253 | }); 254 | var url = parts.join('/'); 255 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 256 | }, 257 | 258 | initOnKeyListeners: function() { 259 | $(document).keyup(function(event) { 260 | var activeElementType = document.activeElement.tagName; 261 | // don't navigate when in search box or textarea 262 | if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') { 263 | switch (event.keyCode) { 264 | case 37: // left 265 | var prevHref = $('link[rel="prev"]').prop('href'); 266 | if (prevHref) { 267 | window.location.href = prevHref; 268 | return false; 269 | } 270 | case 39: // right 271 | var nextHref = $('link[rel="next"]').prop('href'); 272 | if (nextHref) { 273 | window.location.href = nextHref; 274 | return false; 275 | } 276 | } 277 | } 278 | }); 279 | } 280 | }; 281 | 282 | // quick alias for translations 283 | _ = Documentation.gettext; 284 | 285 | $(document).ready(function() { 286 | Documentation.init(); 287 | }); -------------------------------------------------------------------------------- /docs/build/html/_static/doctools.js: -------------------------------------------------------------------------------- 1 | /* 2 | * doctools.js 3 | * ~~~~~~~~~~~ 4 | * 5 | * Sphinx JavaScript utilities for all documentation. 6 | * 7 | * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /** 13 | * select a different prefix for underscore 14 | */ 15 | $u = _.noConflict(); 16 | 17 | /** 18 | * make the code below compatible with browsers without 19 | * an installed firebug like debugger 20 | if (!window.console || !console.firebug) { 21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir", 22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", 23 | "profile", "profileEnd"]; 24 | window.console = {}; 25 | for (var i = 0; i < names.length; ++i) 26 | window.console[names[i]] = function() {}; 27 | } 28 | */ 29 | 30 | /** 31 | * small helper function to urldecode strings 32 | */ 33 | jQuery.urldecode = function(x) { 34 | return decodeURIComponent(x).replace(/\+/g, ' '); 35 | }; 36 | 37 | /** 38 | * small helper function to urlencode strings 39 | */ 40 | jQuery.urlencode = encodeURIComponent; 41 | 42 | /** 43 | * This function returns the parsed url parameters of the 44 | * current request. Multiple values per key are supported, 45 | * it will always return arrays of strings for the value parts. 46 | */ 47 | jQuery.getQueryParameters = function(s) { 48 | if (typeof s == 'undefined') 49 | s = document.location.search; 50 | var parts = s.substr(s.indexOf('?') + 1).split('&'); 51 | var result = {}; 52 | for (var i = 0; i < parts.length; i++) { 53 | var tmp = parts[i].split('=', 2); 54 | var key = jQuery.urldecode(tmp[0]); 55 | var value = jQuery.urldecode(tmp[1]); 56 | if (key in result) 57 | result[key].push(value); 58 | else 59 | result[key] = [value]; 60 | } 61 | return result; 62 | }; 63 | 64 | /** 65 | * highlight a given string on a jquery object by wrapping it in 66 | * span elements with the given class name. 67 | */ 68 | jQuery.fn.highlightText = function(text, className) { 69 | function highlight(node) { 70 | if (node.nodeType == 3) { 71 | var val = node.nodeValue; 72 | var pos = val.toLowerCase().indexOf(text); 73 | if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { 74 | var span = document.createElement("span"); 75 | span.className = className; 76 | span.appendChild(document.createTextNode(val.substr(pos, text.length))); 77 | node.parentNode.insertBefore(span, node.parentNode.insertBefore( 78 | document.createTextNode(val.substr(pos + text.length)), 79 | node.nextSibling)); 80 | node.nodeValue = val.substr(0, pos); 81 | } 82 | } 83 | else if (!jQuery(node).is("button, select, textarea")) { 84 | jQuery.each(node.childNodes, function() { 85 | highlight(this); 86 | }); 87 | } 88 | } 89 | return this.each(function() { 90 | highlight(this); 91 | }); 92 | }; 93 | 94 | /* 95 | * backward compatibility for jQuery.browser 96 | * This will be supported until firefox bug is fixed. 97 | */ 98 | if (!jQuery.browser) { 99 | jQuery.uaMatch = function(ua) { 100 | ua = ua.toLowerCase(); 101 | 102 | var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || 103 | /(webkit)[ \/]([\w.]+)/.exec(ua) || 104 | /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || 105 | /(msie) ([\w.]+)/.exec(ua) || 106 | ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || 107 | []; 108 | 109 | return { 110 | browser: match[ 1 ] || "", 111 | version: match[ 2 ] || "0" 112 | }; 113 | }; 114 | jQuery.browser = {}; 115 | jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; 116 | } 117 | 118 | /** 119 | * Small JavaScript module for the documentation. 120 | */ 121 | var Documentation = { 122 | 123 | init : function() { 124 | this.fixFirefoxAnchorBug(); 125 | this.highlightSearchWords(); 126 | this.initIndexTable(); 127 | 128 | }, 129 | 130 | /** 131 | * i18n support 132 | */ 133 | TRANSLATIONS : {}, 134 | PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, 135 | LOCALE : 'unknown', 136 | 137 | // gettext and ngettext don't access this so that the functions 138 | // can safely bound to a different name (_ = Documentation.gettext) 139 | gettext : function(string) { 140 | var translated = Documentation.TRANSLATIONS[string]; 141 | if (typeof translated == 'undefined') 142 | return string; 143 | return (typeof translated == 'string') ? translated : translated[0]; 144 | }, 145 | 146 | ngettext : function(singular, plural, n) { 147 | var translated = Documentation.TRANSLATIONS[singular]; 148 | if (typeof translated == 'undefined') 149 | return (n == 1) ? singular : plural; 150 | return translated[Documentation.PLURALEXPR(n)]; 151 | }, 152 | 153 | addTranslations : function(catalog) { 154 | for (var key in catalog.messages) 155 | this.TRANSLATIONS[key] = catalog.messages[key]; 156 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); 157 | this.LOCALE = catalog.locale; 158 | }, 159 | 160 | /** 161 | * add context elements like header anchor links 162 | */ 163 | addContextElements : function() { 164 | $('div[id] > :header:first').each(function() { 165 | $('\u00B6'). 166 | attr('href', '#' + this.id). 167 | attr('title', _('Permalink to this headline')). 168 | appendTo(this); 169 | }); 170 | $('dt[id]').each(function() { 171 | $('\u00B6'). 172 | attr('href', '#' + this.id). 173 | attr('title', _('Permalink to this definition')). 174 | appendTo(this); 175 | }); 176 | }, 177 | 178 | /** 179 | * workaround a firefox stupidity 180 | * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075 181 | */ 182 | fixFirefoxAnchorBug : function() { 183 | if (document.location.hash) 184 | window.setTimeout(function() { 185 | document.location.href += ''; 186 | }, 10); 187 | }, 188 | 189 | /** 190 | * highlight the search words provided in the url in the text 191 | */ 192 | highlightSearchWords : function() { 193 | var params = $.getQueryParameters(); 194 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; 195 | if (terms.length) { 196 | var body = $('div.body'); 197 | if (!body.length) { 198 | body = $('body'); 199 | } 200 | window.setTimeout(function() { 201 | $.each(terms, function() { 202 | body.highlightText(this.toLowerCase(), 'highlighted'); 203 | }); 204 | }, 10); 205 | $('

' + _('Hide Search Matches') + '

') 207 | .appendTo($('#searchbox')); 208 | } 209 | }, 210 | 211 | /** 212 | * init the domain index toggle buttons 213 | */ 214 | initIndexTable : function() { 215 | var togglers = $('img.toggler').click(function() { 216 | var src = $(this).attr('src'); 217 | var idnum = $(this).attr('id').substr(7); 218 | $('tr.cg-' + idnum).toggle(); 219 | if (src.substr(-9) == 'minus.png') 220 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 221 | else 222 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 223 | }).css('display', ''); 224 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 225 | togglers.click(); 226 | } 227 | }, 228 | 229 | /** 230 | * helper function to hide the search marks again 231 | */ 232 | hideSearchWords : function() { 233 | $('#searchbox .highlight-link').fadeOut(300); 234 | $('span.highlighted').removeClass('highlighted'); 235 | }, 236 | 237 | /** 238 | * make the url absolute 239 | */ 240 | makeURL : function(relativeURL) { 241 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 242 | }, 243 | 244 | /** 245 | * get the current relative url 246 | */ 247 | getCurrentURL : function() { 248 | var path = document.location.pathname; 249 | var parts = path.split(/\//); 250 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 251 | if (this == '..') 252 | parts.pop(); 253 | }); 254 | var url = parts.join('/'); 255 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 256 | }, 257 | 258 | initOnKeyListeners: function() { 259 | $(document).keyup(function(event) { 260 | var activeElementType = document.activeElement.tagName; 261 | // don't navigate when in search box or textarea 262 | if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') { 263 | switch (event.keyCode) { 264 | case 37: // left 265 | var prevHref = $('link[rel="prev"]').prop('href'); 266 | if (prevHref) { 267 | window.location.href = prevHref; 268 | return false; 269 | } 270 | case 39: // right 271 | var nextHref = $('link[rel="next"]').prop('href'); 272 | if (nextHref) { 273 | window.location.href = nextHref; 274 | return false; 275 | } 276 | } 277 | } 278 | }); 279 | } 280 | }; 281 | 282 | // quick alias for translations 283 | _ = Documentation.gettext; 284 | 285 | $(document).ready(function() { 286 | Documentation.init(); 287 | }); --------------------------------------------------------------------------------