├── .venv ├── .envrc ├── data ├── raw │ └── .gitkeep ├── external │ └── .gitkeep ├── interim │ └── .gitkeep ├── processed │ └── .gitkeep └── README.md ├── tests ├── __init__.py └── test_pylint.py ├── notebooks ├── __init__.py ├── flake8.py ├── loguru.py ├── black.py ├── autoflake.py ├── pylint.py ├── pyupgrade.py ├── isort.py ├── bandit.py ├── 2023-03-07 - Talk Timer.ipynb ├── 2020-05-05 - Python Productivity Power-Ups (PyData UK).ipynb ├── 2023-03-07 - Python Productivity Power-Ups (GeoPython 2023).ipynb ├── 2022-09-24 - Python Productivity Power-Ups (PyCon PT).ipynb └── 2023-02-23 - Python Productivity Power-Ups (PyCon NA).ipynb ├── .python-version ├── .changelog ├── .gitignore └── template.md ├── .env.template ├── .github ├── workflows │ ├── constraints.txt │ ├── codeql-analysis.yml │ └── main.yaml └── dependabot.yml ├── images ├── about-me.jpeg ├── geopython-header.jpg ├── pycon-na-header.png ├── pycon-pt-header.jpg └── pydatauk-may-logo.jpeg ├── .vscode ├── extensions.json └── settings.json ├── CHANGELOG.md ├── docs ├── project_specific_setup.md ├── updating_requirements.md ├── using_towncrier.md ├── quickstart.md ├── using_poetry.md ├── detect_secrets.md └── getting_started.md ├── setup.cfg ├── .gitignore ├── README.md ├── pyproject.toml ├── .secrets.baseline ├── .pre-commit-config.yaml ├── LICENSE └── requirements.txt /.venv: -------------------------------------------------------------------------------- 1 | ppp 2 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | dotenv 2 | -------------------------------------------------------------------------------- /data/raw/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/external/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/interim/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/processed/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.10.6 2 | -------------------------------------------------------------------------------- /.changelog/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | export SECRET='value' 2 | export PYTHONBREAKPOINT=ipdb.set_trace 3 | -------------------------------------------------------------------------------- /.github/workflows/constraints.txt: -------------------------------------------------------------------------------- 1 | pip==23.0.1 2 | poetry==1.3.2 3 | virtualenv==20.19.0 4 | -------------------------------------------------------------------------------- /images/about-me.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/john-sandall/python-productivity-powerups/HEAD/images/about-me.jpeg -------------------------------------------------------------------------------- /notebooks/flake8.py: -------------------------------------------------------------------------------- 1 | """flake8 ./notebooks/flake8.py""" 2 | 3 | def CAPITALISED_FUNCTION(x): 4 | return x+1 5 | -------------------------------------------------------------------------------- /images/geopython-header.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/john-sandall/python-productivity-powerups/HEAD/images/geopython-header.jpg -------------------------------------------------------------------------------- /images/pycon-na-header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/john-sandall/python-productivity-powerups/HEAD/images/pycon-na-header.png -------------------------------------------------------------------------------- /images/pycon-pt-header.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/john-sandall/python-productivity-powerups/HEAD/images/pycon-pt-header.jpg -------------------------------------------------------------------------------- /images/pydatauk-may-logo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/john-sandall/python-productivity-powerups/HEAD/images/pydatauk-may-logo.jpeg -------------------------------------------------------------------------------- /notebooks/loguru.py: -------------------------------------------------------------------------------- 1 | from loguru import logger 2 | 3 | logger.debug("That's it, beautiful and simple logging!") 4 | logger.info(r"This message is purely informational. ¯\_(ツ)_/¯") 5 | logger.error("😱😱😱😱😱") 6 | -------------------------------------------------------------------------------- /notebooks/black.py: -------------------------------------------------------------------------------- 1 | j = [ 2 | 1, 3 | 2, 4 | 3 5 | ] 6 | 7 | import os 8 | 9 | 10 | def very_important_function(template: str,*variables,file: os.PathLike,debug:bool=False,): 11 | """Does something very important.""" 12 | pass 13 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-python.python", 4 | "ms-python.vscode-pylance", 5 | "esbenp.prettier-vscode", 6 | "wayou.vscode-todo-highlight", 7 | "bungcip.better-toml" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 7 | 8 | 9 | -------------------------------------------------------------------------------- /notebooks/autoflake.py: -------------------------------------------------------------------------------- 1 | """Initial state. 2 | 3 | autoflake --in-place --remove-unused-variables notebooks/autoflake.py 4 | 5 | import numpy as np 6 | import pandas as pd 7 | 8 | print(pd.read_html("https://na.pycon.org/programme/")) 9 | """ 10 | 11 | import numpy as np 12 | import pandas as pd 13 | 14 | print(pd.read_html("https://na.pycon.org/programme/")) 15 | -------------------------------------------------------------------------------- /notebooks/pylint.py: -------------------------------------------------------------------------------- 1 | """Pylint examples. 2 | 3 | https://pylint.readthedocs.io/en/latest/user_guide/messages/error/return-outside-function.html 4 | https://pylint.readthedocs.io/en/latest/user_guide/messages/refactor/no-else-return.html 5 | """ 6 | 7 | 8 | def is_john(name): 9 | """Check if name is 'john' or 'John'.""" 10 | if name.lower() == "john": 11 | return True 12 | else: 13 | return False 14 | -------------------------------------------------------------------------------- /tests/test_pylint.py: -------------------------------------------------------------------------------- 1 | """Example test.""" 2 | import pytest 3 | 4 | from notebooks.pylint import is_john 5 | 6 | 7 | def test_is_john(): 8 | """Test the is_john() function.""" 9 | assert is_john("John") is True 10 | assert is_john("JOHN") is True 11 | assert is_john("john") is True 12 | assert is_john("Coefficient") is False 13 | 14 | with pytest.raises(AttributeError): 15 | is_john(123) 16 | -------------------------------------------------------------------------------- /notebooks/pyupgrade.py: -------------------------------------------------------------------------------- 1 | """pyupgrade --py36-plus ./notebooks/pyupgrade.py""" 2 | 3 | # In python3.8+, comparison to literals becomes a SyntaxWarning as the success of those comparisons 4 | # is implementation specific (due to common object caching). 5 | NUMBER = 1 6 | # print(NUMBER is 5) 7 | print(NUMBER is 5) 8 | 9 | 10 | # f-strings 11 | # print("The number is {}".format(NUMBER)) 12 | print("The number is {}".format(NUMBER)) 13 | -------------------------------------------------------------------------------- /notebooks/isort.py: -------------------------------------------------------------------------------- 1 | from my_lib import Object 2 | 3 | import os 4 | 5 | from my_lib import Object3 6 | 7 | from my_lib import Object2 8 | 9 | import sys 10 | 11 | from third_party import lib15, lib1, lib2, lib3, lib4, lib5, lib6, lib7, lib8, lib9, lib10, lib11, lib12, lib13, lib14 12 | 13 | import sys 14 | 15 | from __future__ import absolute_import 16 | 17 | from third_party import lib3 18 | 19 | print("Hey") 20 | print("yo") 21 | -------------------------------------------------------------------------------- /docs/project_specific_setup.md: -------------------------------------------------------------------------------- 1 | # Project-specific setup 2 | 3 | This document contains setup instructions specifically for this project only. This design enables 4 | us to keep other docs easily aligned with future upstream changes to 5 | [coefficient-cookiecutter](https://github.com/CoefficientSystems/coefficient-cookiecutter/). 6 | 7 | 8 | ## Jupyter kernel 9 | 10 | ```sh 11 | python -m ipykernel install --user --name ppp --display-name "Python (ppp)" 12 | ``` 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | disable-noqa = True 3 | max-line-length = 100 4 | extend-exclude = .git,__pycache__ 5 | extend-ignore = 6 | E203, # whitespace before : is not PEP8 compliant & conflicts with black 7 | T100, # line contains FIXME 8 | T101, # line contains TODO 9 | D100, # Missing docstring in public module 10 | D104, # Missing docstring in public package 11 | T201, # print found 12 | # per-file-ignores = 13 | # src/path/file.py" 14 | # E123 15 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Basic dependabot.yml file with minimum configuration 2 | 3 | version: 2 4 | updates: 5 | - package-ecosystem: github-actions 6 | directory: "/" 7 | schedule: 8 | interval: daily 9 | - package-ecosystem: pip 10 | directory: "/.github/workflows" 11 | schedule: 12 | interval: daily 13 | - package-ecosystem: pip 14 | directory: "/" 15 | schedule: 16 | interval: daily 17 | versioning-strategy: lockfile-only 18 | allow: 19 | - dependency-type: "all" 20 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | # `data` folder overview 2 | 3 | Any data that needs to be stored locally should be saved in this location. This folder, 4 | and its sub-folders, are not version-controlled. 5 | 6 | The sub-folders should be used as follows: 7 | 8 | - `external`: any data that will not be processed at all, such as reference data; 9 | - `raw`: any raw data before any processing; 10 | - `interim`: any raw data that has been partially processed and, for whatever reason, 11 | needs to be stored before further processing is completed; and 12 | - `processed`: any raw or interim data that has been fully processed into its final 13 | state. 14 | -------------------------------------------------------------------------------- /notebooks/bandit.py: -------------------------------------------------------------------------------- 1 | """Bandit examples. 2 | 3 | bandit notebooks/bandit.py 4 | """ 5 | 6 | import pickle 7 | 8 | import pandas as pd 9 | import requests 10 | import sqlalchemy 11 | 12 | # Bad: hard coded passwords 13 | EMAIL_PASSWORD = "secret" 14 | 15 | # Bad: potential SQL injection 16 | identifier = "123" 17 | query = "SELECT * FROM foo WHERE id = '%s'" % identifier 18 | 19 | # Bad: No timeout when using requests 20 | requests.get("https://gmail.com") 21 | 22 | # Bad: Pickle can be insecure 23 | # https://github.com/PyCQA/bandit/pull/710 24 | df = pd.DataFrame({"col_A": [1, 2]}) 25 | pick = pickle.dumps(df) 26 | print(pd.read_pickle(pick)) 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Caches 2 | **/.ipynb_checkpoints/* 3 | **/__pycache__/* 4 | *.pyc 5 | notebooks/cache 6 | 7 | 8 | # Data 9 | ./data/ 10 | !**/data/**/README.md 11 | !**/data/**/*.gitkeep 12 | 13 | # Secrets 14 | .env 15 | 16 | # Personal TODO files 17 | TODO.txt 18 | 19 | # VisualStudioCode 20 | .vscode/* 21 | !.vscode/settings.json 22 | !.vscode/tasks.json 23 | !.vscode/launch.json 24 | !.vscode/extensions.json 25 | .RData 26 | 27 | # Environment 28 | venv/ 29 | 30 | # coverage 31 | .coverage 32 | .coverage.* 33 | htmlcov 34 | 35 | # IDE config 36 | .idea/ 37 | ipython_config.py 38 | profile_default/ 39 | 40 | # Other 41 | .DS_Store 42 | ignore/ 43 | *.log 44 | *.pem 45 | 46 | # Editor 47 | pyrightconfig.json 48 | 49 | # https://www.gitignore.io/ 50 | -------------------------------------------------------------------------------- /.changelog/template.md: -------------------------------------------------------------------------------- 1 | {% for section, _ in sections.items() %} 2 | {% if section %}## {{section}}{% endif %} 3 | 4 | {% if sections[section] %} 5 | {% for category, val in definitions.items() if category in sections[section]%} 6 | ### {{ definitions[category]['name'] }} 7 | 8 | {% if definitions[category]['showcontent'] %} 9 | {% for text, values in sections[section][category].items() %} 10 | - {{ text }} ({{ values|join(', ') }}) 11 | {% endfor %} 12 | 13 | {% else %} 14 | - {{ sections[section][category]['']|join(', ') }} 15 | 16 | {% endif %} 17 | {% if sections[section][category]|length == 0 %} 18 | No significant changes. 19 | 20 | {% else %} 21 | {% endif %} 22 | 23 | {% endfor %} 24 | {% else %} 25 | No significant changes. 26 | 27 | {% endif %} 28 | {% endfor %} 29 | -------------------------------------------------------------------------------- /docs/updating_requirements.md: -------------------------------------------------------------------------------- 1 | # Updating requirements 2 | 3 | ```sh 4 | # Create feature branch 5 | poetry-sync 6 | poetryup --latest 7 | # If any issues, likely due to two "latest" packages conflicting. See example below. 8 | poetry-regenerate 9 | poetry-sync 10 | pre-commit run --all-files --hook-stage=manual 11 | poetry-sync 12 | pytest 13 | # Commit & push 14 | ``` 15 | 16 | If the latest version of `green` requires `blue (>=1.2, <1.3)` and the latest version of `blue` is 17 | `1.4` then you will encounter a `SolverProblemError`, for example: 18 | 19 | ```sh 20 | SolverProblemError 21 | 22 | Because green (0.8) depends on blue (>=1.2,<1.3) 23 | and no versions of green match (>=0.8,<1.0) requires blue (>=1.2,<1.3). 24 | So, because src depends on both blue (^1.4) and green (^0.8), version solving failed. 25 | ``` 26 | 27 | In this situation, do the following: 28 | - Comment out `blue` 29 | - Re-run `poetryup --latest` 30 | - Handle any other new package conflicts the same way until poetryup resolves 31 | - Uncomment out `blue` with package version that works with `green`, e.g. `blue = "^1.2"` 32 | - Run `poetry-regenerate` onwards 33 | -------------------------------------------------------------------------------- /notebooks/2023-03-07 - Talk Timer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# ⚡ Lightning Talk Timer ⏳️" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import time\n", 17 | "from tqdm import tqdm\n", 18 | "\n", 19 | "for i in tqdm(range(5*60)):\n", 20 | " time.sleep(1)" 21 | ] 22 | } 23 | ], 24 | "metadata": { 25 | "kernelspec": { 26 | "display_name": "Python 3 (ipykernel)", 27 | "language": "python", 28 | "name": "python3" 29 | }, 30 | "language_info": { 31 | "codemirror_mode": { 32 | "name": "ipython", 33 | "version": 3 34 | }, 35 | "file_extension": ".py", 36 | "mimetype": "text/x-python", 37 | "name": "python", 38 | "nbconvert_exporter": "python", 39 | "pygments_lexer": "ipython3", 40 | "version": "3.10.6" 41 | }, 42 | "toc": { 43 | "base_numbering": 1, 44 | "nav_menu": {}, 45 | "number_sections": true, 46 | "sideBar": true, 47 | "skip_h1_title": true, 48 | "title_cell": "Table of Contents", 49 | "title_sidebar": "Contents", 50 | "toc_cell": true, 51 | "toc_position": {}, 52 | "toc_section_display": true, 53 | "toc_window_display": true 54 | }, 55 | "vscode": { 56 | "interpreter": { 57 | "hash": "be34d95dbc58180ac3633c8e41e8772c710783434d94474c3fd4437fff7f4790" 58 | } 59 | } 60 | }, 61 | "nbformat": 4, 62 | "nbformat_minor": 4 63 | } 64 | -------------------------------------------------------------------------------- /docs/using_towncrier.md: -------------------------------------------------------------------------------- 1 | # Changelog & releases 2 | 3 | The format of `CHANGELOG.md` is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 4 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 5 | 6 | 7 | ## Using towncrier to create news entries 8 | 9 | We use [towncrier](https://github.com/twisted/towncrier) to reduce merge conflicts by generating 10 | `CHANGELOG.md` from news fragments, rather than maintaining it directly. Create a news fragment for 11 | each MR if you would like to ensure your changes are communicated to other project contributors. 12 | 13 | ```bash 14 | # To create a news entry for an added feature relating to MR !123 15 | # Adding --edit is optional and will open in your default shell's $EDITOR 16 | towncrier create 123.added --edit 17 | ``` 18 | 19 | Top tips: 20 | - You may wish to add `export EDITOR="code -w"` to your `.zshrc` file to open this directly in VS Code. 21 | - News fragments should be written in markdown. 22 | - The generated news fragments live in `.changelog/` and can be easily rewritten as an MR evolves. 23 | 24 | We use the following custom types (adapted from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)): 25 | - `.added` for new features 26 | - `.changed` for changes in existing functionality 27 | - `.deprecated` for soon-to-be removed features 28 | - `.removed` for now removed features 29 | - `.fixed` for any bug fixes 30 | - `.security` in case of vulnerabilities 31 | - `.analysis` for data analyses 32 | - `.docs` for documentation improvements 33 | - `.maintenance` for maintenance tasks & upgrades 34 | 35 | 36 | ## Releasing a new version & updating `CHANGELOG.md` 37 | 38 | Release versions are tied to Gitlab milestones and sprints. Release checklist: 39 | 40 | 1. Review MRs assigned to the release milestone in Gitlab & reallocate to the next release. 41 | 2. Run `towncrier build --version=VERSION` (preview with `--draft`) 42 | 3. Add a git tag for the release. 43 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | // editor & autoformatter settings 3 | "editor.formatOnSave": true, 4 | "editor.trimAutoWhitespace": true, 5 | "files.trimTrailingWhitespace": true, 6 | "prettier.enable": true, 7 | 8 | // python - black 9 | "python.formatting.provider": "black", 10 | "python.formatting.blackArgs": [], 11 | 12 | // python - other 13 | "python.languageServer": "Pylance", 14 | "python.pythonPath": "${env:VIRTUAL_ENV}", 15 | 16 | // python - linting & static analysis 17 | "python.analysis.extraPaths": ["notebooks"], 18 | "python.analysis.typeCheckingMode": "basic", 19 | "python.linting.enabled": true, 20 | "python.linting.flake8Enabled": true, 21 | "python.linting.flake8Args": [], 22 | "python.linting.pylintEnabled": true, 23 | "python.linting.pylintArgs": [], 24 | "python.linting.pylintUseMinimalCheckers": false, 25 | 26 | // Python files only 27 | "[python]": { 28 | // isort on save 29 | "editor.codeActionsOnSave": {"source.organizeImports": true}, 30 | // Stop the 'Black does not support "Format Selection"' message every 31 | // time you paste something (https://stackoverflow.com/a/63612401/3279076) 32 | "editor.formatOnPaste": false 33 | }, 34 | 35 | // python - pytest (https://code.visualstudio.com/docs/python/testing) 36 | "python.testing.unittestEnabled": false, 37 | "python.testing.pytestEnabled": true, 38 | "python.testing.pytestArgs": ["tests"], 39 | 40 | // git 41 | "git.enabled": true, 42 | 43 | // file associations 44 | "workbench.editorAssociations": {"*.ipynb": "jupyter.notebook.ipynb"}, 45 | "files.associations": { 46 | "**/requirements{/**,*}.{txt,in}": "pip-requirements", 47 | }, 48 | 49 | // Disable VS Code experiments to ensure consistent experience across the team 50 | // (see https://github.com/microsoft/vscode-python/wiki/AB-Experiments) 51 | "python.experiments.enabled": false, 52 | } 53 | -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | This document contains instructions _only_ to get a fully working development environment for 4 | running this repo. For pre-requisites (e.g. `pyenv` install instructions) plus details on what's 5 | being installed and why, please see [docs/getting_started.md](docs/getting_started.md). 6 | 7 | We assume the following are installed and configured: 8 | - [pyenv](https://github.com/pyenv/pyenv) 9 | - [pyenv-virtualenvwrapper](https://github.com/pyenv/pyenv-virtualenvwrapper) 10 | - [Poetry](https://python-poetry.org/docs/) 11 | - [zsh-autoswitch-virtualenv](https://github.com/MichaelAquilina/zsh-autoswitch-virtualenv) 12 | - [direnv](https://direnv.net/) 13 | 14 | 15 | ## Part 1: Generic Python setup 16 | 17 | ```sh 18 | # Get the repo 19 | git clone ${REPO_GIT_URL} 20 | 21 | # Install Python 22 | pyenv install $(cat .python-version) 23 | pyenv shell $(cat .python-version) 24 | python -m pip install --upgrade pip 25 | python -m pip install virtualenvwrapper 26 | pyenv virtualenvwrapper 27 | 28 | # Setup the virtualenv 29 | mkvirtualenv -p python$(cat .python-version) $(cat .venv) 30 | python -V 31 | python -m pip install --upgrade pip 32 | 33 | # Install dependencies with Poetry 34 | poetry self update 35 | poetry install --no-root --sync 36 | 37 | # Create templated .env for storing secrets 38 | cp .env.template .env 39 | direnv allow 40 | 41 | # Create and audit secrets baseline 42 | # N.B. Adjust the exclusions here depending on your needs (check .pre-commit-config.yaml) 43 | detect-secrets --verbose scan \ 44 | --exclude-files 'poetry\.lock' \ 45 | --exclude-files '\.secrets\.baseline' \ 46 | --exclude-files '\.env\.template' \ 47 | --exclude-files '.*\\.ipynb$' \ 48 | --exclude-secrets 'password|ENTER_PASSWORD_HERE|INSERT_API_KEY_HERE' \ 49 | --exclude-lines 'integrity=*sha' \ 50 | > .secrets.baseline 51 | 52 | detect-secrets audit .secrets.baseline 53 | ``` 54 | 55 | 56 | ## Part 2: Project-specific setup 57 | 58 | Please check [docs/project_specific_setup.md](docs/project_specific_setup.md) for further instructions. 59 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [main] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [main] 20 | schedule: 21 | - cron: "33 18 * * 2" 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: ["python"] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v2 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v2 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python Productivity Power-Ups 2 | 3 | [![CI](https://github.com/john-sandall/python-productivity-powerups/actions/workflows/main.yaml/badge.svg)](https://github.com/john-sandall/python-productivity-powerups/actions/workflows/main.yaml) 4 | 5 | This was created for the PyDataUK May Talks which you can watch in YouTube here: [https://www.youtube.com/watch?v=C1hqHk1SfrA](https://www.youtube.com/watch?v=C1hqHk1SfrA). If you would like to skip to this talk [please use this link](https://youtu.be/C1hqHk1SfrA?t=1851). 6 | 7 | 8 | ## Code 9 | You can find the notebook from the presentation here: [Python Productivity Power-Ups](./notebooks/Python%20Productivity%20Power-Ups.ipynb) 10 | 11 | The tools featured were: 12 | 1. tqdm: [https://pypi.org/project/tqdm/] 13 | 2. joblib: [https://joblib.readthedocs.io/en/latest/] 14 | 3. black: [https://pypi.org/project/black/] 15 | 4. jupyter-black: [https://pypi.org/project/jupyter-black/] 16 | 5. isort: [https://pycqa.github.io/isort/] 17 | 6. fzf: [https://github.com/junegunn/fzf] 18 | 7. pre-commit: [https://pre-commit.com/] 19 | 8. coefficient-cookiecutter: [https://github.com/CoefficientSystems/coefficient-cookiecutter] 20 | 21 | ## Project cheatsheet 22 | 23 | - **pre-commit:** `pre-commit run --all-files` 24 | - **pytest:** `pytest` or `pytest -s` 25 | - **coverage:** `coverage run -m pytest` or `coverage html` 26 | - **poetry sync:** `poetry install --no-root --remove-untracked` 27 | - **updating requirements:** see [docs/updating_requirements.md](docs/updating_requirements.md) 28 | - **create towncrier entry:** `towncrier create 123.added --edit` 29 | 30 | 31 | ## Initial project setup 32 | 33 | 1. See [docs/getting_started.md](docs/getting_started.md) or [docs/quickstart.md](docs/quickstart.md) 34 | for how to get up & running. 35 | 2. Check [docs/project_specific_setup.md](docs/project_specific_setup.md) for project specific setup. 36 | 3. See [docs/using_poetry.md](docs/using_poetry.md) for how to update Python requirements using 37 | [Poetry](https://python-poetry.org/). 38 | 4. See [docs/detect_secrets.md](docs/detect_secrets.md) for more on creating a `.secrets.baseline` 39 | file using [detect-secrets](https://github.com/Yelp/detect-secrets). 40 | 5. See [docs/using_towncrier.md](docs/using_towncrier.md) for how to update the `CHANGELOG.md` 41 | using [towncrier](https://github.com/twisted/towncrier). 42 | 43 | --- 44 | 45 | ## License 46 | 47 | Creative Commons License
This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. 48 | -------------------------------------------------------------------------------- /docs/using_poetry.md: -------------------------------------------------------------------------------- 1 | # Updating Python requirements using Poetry 2 | 3 | This project uses Poetry to manage dependencies: 4 | [https://python-poetry.org/docs/](https://python-poetry.org/docs/) 5 | 6 | The Poetry virtualenv should activate automatically if you are using 7 | [zsh-autoswitch-virtualenv](https://github.com/MichaelAquilina/zsh-autoswitch-virtualenv). You can 8 | also activate it manually by running `workon ppp` (preferred) or `poetry shell` (Poetry 9 | created/managed virtualenv). 10 | 11 | 12 | ## Poetry tl;dr 13 | 14 | ```bash 15 | # Sync your environment with poetry.lock 16 | poetry install --no-root --remove-untracked 17 | 18 | # Add and install a new package into your environment (or update an existing one) 19 | # 1. Add a package to pyproject.toml 20 | # 2. Run this 21 | poetry add package@version 22 | 23 | # N.B. We lock & export to requirements when you run `pre-commit run --all-files` so you 24 | # shouldn't need to run the commands below yourself. 25 | 26 | # Compile all poetry.lock file 27 | poetry update 28 | 29 | # Export to requirements.txt (this is for compatibility with team members who may not 30 | # have Poetry; Poetry itself uses poetry.lock when you run `poetry install`) 31 | poetry export -f requirements.txt --output requirements.txt 32 | ``` 33 | 34 | 35 | ## Poetry FAQ 36 | 37 | **How do I sync my environment with the latest `poetry.lock`?** 38 | To install dependencies, simply `cd backend-security` and run `poetry install --no-root --remove-untracked`. 39 | This will resolve & install all deps in `pyproject.toml` via `poetry.lock`. Options: 40 | - `--no-root` skips installing the repo itself as a Python package 41 | - `--remove-untracked` removes old dependencies no longer present in the lock file 42 | 43 | You may wish to set an alias e.g. `alias poetry-sync='poetry install --no-root --remove-untracked'` 44 | 45 | **How do I add/change packages?** 46 | In order to install a new package or remove an old one, please edit `pyproject.toml` 47 | or use either `poetry add package` or `pipx run poetry add package@version` as desired. 48 | 49 | **How do I update `poetry.lock` to match new changes in `pyproject.toml`?** 50 | You can run `poetry update`, although this will also get run when you run pre-commit. This fetches 51 | the latest matching versions (according to `pyproject.toml`) and updates `poetry.lock`, and is 52 | equivalent to deleting `poetry.lock` and running `poetry install --no-root` again from scratch. 53 | 54 | **What do `~`, `^` and `*` mean?** 55 | Check out the Poetry docs page for [specifying version constraints](https://python-poetry.org/docs/dependency-specification/#version-constraints). 56 | Environment markers are great, e.g. this means "for Python 2.7.* OR Windows, install pathlib2 >=2.2, <3.0" 57 | `pathlib2 = { version = "^2.2", markers = "python_version ~= '2.7' or sys_platform == 'win32'" }` 58 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | concurrency: 10 | group: ${{ github.head_ref || github.run_id }} 11 | cancel-in-progress: true 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | timeout-minutes: 30 17 | strategy: 18 | matrix: 19 | python-version: ["3.10"] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | 29 | - name: Create virtualenv 30 | run: | 31 | which python 32 | python -m venv venv 33 | source venv/bin/activate 34 | python -m pip install --constraint=.github/workflows/constraints.txt --upgrade pip 35 | which python 36 | 37 | - name: Set up Poetry cache 38 | uses: actions/cache@v3.2.6 39 | with: 40 | path: venv 41 | key: venv-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} 42 | 43 | - name: Install Poetry and Python dependencies 44 | run: | 45 | curl -sSL https://install.python-poetry.org | python3 - 46 | poetry --version 47 | poetry config virtualenvs.in-project true 48 | poetry config virtualenvs.create false 49 | poetry config virtualenvs.path venv 50 | source venv/bin/activate 51 | which python 52 | poetry install --no-root 53 | 54 | - name: Compute pre-commit cache key 55 | id: pre-commit-cache 56 | shell: python 57 | run: | 58 | import hashlib 59 | import sys 60 | python = "py{}.{}".format(*sys.version_info[:2]) 61 | payload = sys.version.encode() + sys.executable.encode() 62 | digest = hashlib.sha256(payload).hexdigest() 63 | result = "${{ runner.os }}-{}-{}-pre-commit".format(python, digest[:8]) 64 | print("::set-output name=result::{}".format(result)) 65 | 66 | - name: Restore pre-commit cache 67 | uses: actions/cache@v3.2.6 68 | with: 69 | path: ~/.cache/pre-commit 70 | key: ${{ steps.pre-commit-cache.outputs.result }}-${{ hashFiles('.pre-commit-config.yaml') }} 71 | restore-keys: | 72 | ${{ steps.pre-commit-cache.outputs.result }}- 73 | 74 | - name: pre-commit 75 | run: | 76 | source venv/bin/activate 77 | pre-commit run --hook-stage=manual --show-diff-on-failure --all-files 78 | 79 | # - name: mypy 80 | # run: mypy . 81 | 82 | # - name: pytest 83 | # # run: pytest --junitxml=junit.xml --cov --cov-report=term-missing:skip-covered --cov-report=xml 84 | # run: | 85 | # source venv/bin/activate 86 | # pytest 87 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | ########### 2 | # 📜 Poetry 3 | ########### 4 | [tool.poetry] 5 | name = "python_productivity_powerups" 6 | version = "0.1.0" 7 | description = "Packages To Power-Up Your Python Productivity" 8 | authors = ["John Sandall <2884159+john-sandall@users.noreply.github.com>"] 9 | license = "UNLICENSED" 10 | classifiers = ["Private :: Do Not Upload"] 11 | packages = [{ include = "notebooks", from = "notebooks" }] 12 | 13 | [tool.poetry.dependencies] 14 | python = "^3.10.6" 15 | # Everything below here is alphabetically sorted 16 | black = "^22.12.0" 17 | ipykernel = "^6.21.2" 18 | isort = "^5.12.0" 19 | joblib = "^1.2.0" 20 | jupyter-black = "^0.3.3" 21 | jupyterlab = "^3.6.1" 22 | loguru = "^0.6.0" 23 | pandarallel = "^1.6.4" 24 | pandas = "^1.5.3" 25 | seaborn = "^0.12.2" 26 | tqdm = "^4.64.1" 27 | treon = "^0.1.4" 28 | bandit = "^1.7.4" 29 | 30 | [tool.poetry.dev-dependencies] 31 | # Everything below here is alphabetically sorted 32 | autoflake = "<2.0.1" 33 | detect-secrets = "1.2.0" 34 | flake8 = ">=5,<6" 35 | flake8-docstrings = "^1.7.0" 36 | flake8-eradicate = "^1.4.0" 37 | flake8-fixme = "^1.1.1" 38 | flake8-implicit-str-concat = "^0.3.0" 39 | flake8-no-pep420 = "^2.3.0" 40 | flake8-print = "^5.0.0" 41 | flake8-return = "^1.2.0" 42 | ipdb = "^0.13.11" 43 | pip-audit = "^2.4.14" 44 | pre-commit = "^2.21.0" 45 | pylint = "^2.16.2" 46 | pytest = "^7.2.1" 47 | pyupgrade = "^3.3.1" 48 | towncrier = "^21.9.0" 49 | 50 | [build-system] 51 | requires = ["poetry-core>=1.0.0"] 52 | build-backend = "poetry.core.masonry.api" 53 | 54 | ############ 55 | # ✅ Linters 56 | ############ 57 | [tool.black] 58 | line-length = 100 59 | target-version = ["py310"] 60 | 61 | [tool.isort] 62 | profile = "black" 63 | line_length = 100 64 | default_section = "THIRDPARTY" 65 | known_first_party = ["notebooks"] 66 | sections = "FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" 67 | # add_imports = "from __future__ import annotations" 68 | 69 | [tool.pylint.MASTER] 70 | ignore-paths = [".git", "notebooks/isort_example.py"] 71 | load-plugins = [] 72 | 73 | [tool.pylint."MESSAGES CONTROL"] 74 | enable = "all" 75 | max-module-lines = 2000 76 | max-line-length = 100 77 | max-locals = 50 78 | min-similarity-lines = 150 79 | max-statements = 89 80 | max-args = 22 81 | max-branches = 17 82 | # good-names = [] 83 | disable = ["missing-module-docstring"] 84 | logging-format-style = "new" 85 | 86 | ############## 87 | # 📣 Towncrier 88 | ############## 89 | [tool.towncrier] 90 | package = "notebooks" 91 | package_dir = "." 92 | filename = "CHANGELOG.md" 93 | directory = ".changelog/" 94 | template = ".changelog/template.md" 95 | title_format = "## [{version}] - {project_date}" 96 | issue_format = "[#{issue}](https://github.com/john-sandall/python-productivity-powerups/issues/{issue})" 97 | start_string = "\n" 98 | underlines = ["", ""] 99 | 100 | # .added for new features 101 | # .changed for changes in existing functionality 102 | # .deprecated for soon-to-be removed features 103 | # .removed for now removed features 104 | # .fixed for any bug fixes 105 | # .security in case of vulnerabilities 106 | # .analysis for data analyses 107 | # .docs for documentation improvements 108 | # .maintenance for maintenance tasks & upgrades 109 | [[tool.towncrier.type]] 110 | directory = "added" 111 | name = "Added" 112 | showcontent = true 113 | 114 | [[tool.towncrier.type]] 115 | directory = "changed" 116 | name = "Changed" 117 | showcontent = true 118 | 119 | [[tool.towncrier.type]] 120 | directory = "deprecated" 121 | name = "Deprecated" 122 | showcontent = true 123 | 124 | [[tool.towncrier.type]] 125 | directory = "removed" 126 | name = "Removed" 127 | showcontent = true 128 | 129 | [[tool.towncrier.type]] 130 | directory = "fixed" 131 | name = "Bug Fixes" 132 | showcontent = true 133 | 134 | [[tool.towncrier.type]] 135 | directory = "security" 136 | name = "Security" 137 | showcontent = true 138 | 139 | [[tool.towncrier.type]] 140 | directory = "analysis" 141 | name = "Analyses" 142 | showcontent = true 143 | 144 | [[tool.towncrier.type]] 145 | directory = "docs" 146 | name = "Improved Documentation" 147 | showcontent = true 148 | 149 | [[tool.towncrier.type]] 150 | directory = "maintenance" 151 | name = "Maintenance Changes" 152 | showcontent = true 153 | -------------------------------------------------------------------------------- /docs/detect_secrets.md: -------------------------------------------------------------------------------- 1 | # Using the `detect-secrets` pre-commit hook 2 | 3 | [We use `detect-secrets` to check that no secrets are accidentally committed](detect-secrets). The 4 | `detect-secrets` package does its best to prevent accidental committing of secrets, but it may miss 5 | things. Instead, focus on good software development practices! See the [definition of a secret for 6 | further information](#definition-of-a-secret-according-to-detect-secrets). 7 | 8 | 9 | ## Initial setup 10 | 11 | This pre-commit hook requires you to generate a baseline file if one is not already present within 12 | the root directory. To create the baseline file, run the following at the root of the repository: 13 | 14 | ```shell 15 | detect-secrets scan > .secrets.baseline 16 | ``` 17 | 18 | Next, audit the baseline that has been generated by running: 19 | 20 | ```shell 21 | detect-secrets audit .secrets.baseline 22 | ``` 23 | 24 | When you run this command, you'll enter an interactive console. This will present you with a list 25 | of high-entropy string and/or anything which could be a secret. It will then ask you to verify 26 | whether this is the case. This allows the hook to remember false positives in the future, and alert 27 | you to new secrets. 28 | 29 | 30 | ## Definition of a "secret" according to `detect-secrets` 31 | 32 | The `detect-secrets` documentation, as of January 2021, says it works: 33 | 34 | > ...by running periodic diff outputs against heuristically crafted \[regular 35 | > expression\] statements, to identify whether any new secret has been committed. 36 | 37 | This means it uses regular expression patterns to scan your code changes for anything that looks 38 | like a secret according to the patterns. By definition, there are only a limited number of 39 | patterns, so the `detect-secrets` package cannot detect every conceivable type of secret. 40 | 41 | To understand what types of secrets will be detected, read the `detect-secrets` documentation on 42 | caveats, and the list of supported plugins. Also, you should use secret variable names with words 43 | that will trip the KeywordDetector plugin; see the 44 | [`DENYLIST` variable for the full list of words][detect-secrets-keyword-detector]. 45 | 46 | 47 | ## If `pre-commit` detects secrets during commit 48 | 49 | If `pre-commit` detects any secrets when you try to create a commit, it will detail what it found 50 | and where to go to check the secret. 51 | 52 | If the detected secret is a false positive, there are two options to resolve this, and prevent your 53 | commit from being blocked: 54 | 55 | - [inline allowlisting of false positives (recommended)](#inline-allowlisting-recommended); or 56 | - [updating the `.secrets.baseline` to include the false positives](#updating-secretsbaseline). 57 | 58 | In either case, if an actual secret is detected (or a combination of actual secrets and false 59 | positives), first remove the actual secret. Then following either of these processes. 60 | 61 | ### Inline allowlisting (recommended) 62 | 63 | To exclude a false positive, add a `pragma` comment such as: 64 | 65 | ```python 66 | secret = "Password123" # pragma: allowlist secret 67 | ``` 68 | 69 | or 70 | 71 | ```python 72 | # pragma: allowlist nextline secret 73 | secret = "Password123" 74 | ``` 75 | 76 | If the detected secret is actually a secret (or other sensitive information), remove the secret and 77 | re-commit; there is no need to add any `pragma` comments. 78 | 79 | If your commit contains a mixture of false positives and actual secrets, remove the actual secrets 80 | first before adding `pragma` comments to the false positives. 81 | 82 | 83 | ### Updating `.secrets.baseline` 84 | 85 | To exclude a false positive, you can also [update the `.secrets.baseline` by repeating the same two 86 | commands as in the initial setup](#using-the-detect-secrets-pre-commit-hook). 87 | 88 | During auditing, if the detected secret is actually a secret (or other sensitive information), 89 | remove the secret and re-commit. There is no need to update the `.secrets.baseline` file in this 90 | case. 91 | 92 | If your commit contains a mixture of false positives and actual secrets, remove the actual secrets 93 | first before updating and auditing the `.secrets.baseline` file. 94 | 95 | 96 | [detect-secrets]: https://github.com/Yelp/detect-secrets 97 | [detect-secrets-plugins]: https://github.com/Yelp/detect-secrets#currently-supported-plugins 98 | 99 | --- 100 | 101 | Credit: This document has been adapted from the 102 | [govukcookiecutter](https://github.com/best-practice-and-impact/govcookiecutter/). 103 | -------------------------------------------------------------------------------- /.secrets.baseline: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.3.0", 3 | "plugins_used": [ 4 | { 5 | "name": "ArtifactoryDetector" 6 | }, 7 | { 8 | "name": "AWSKeyDetector" 9 | }, 10 | { 11 | "name": "AzureStorageKeyDetector" 12 | }, 13 | { 14 | "name": "Base64HighEntropyString", 15 | "limit": 4.5 16 | }, 17 | { 18 | "name": "BasicAuthDetector" 19 | }, 20 | { 21 | "name": "CloudantDetector" 22 | }, 23 | { 24 | "name": "GitHubTokenDetector" 25 | }, 26 | { 27 | "name": "HexHighEntropyString", 28 | "limit": 3.0 29 | }, 30 | { 31 | "name": "IbmCloudIamDetector" 32 | }, 33 | { 34 | "name": "IbmCosHmacDetector" 35 | }, 36 | { 37 | "name": "JwtTokenDetector" 38 | }, 39 | { 40 | "name": "KeywordDetector", 41 | "keyword_exclude": "" 42 | }, 43 | { 44 | "name": "MailchimpDetector" 45 | }, 46 | { 47 | "name": "NpmDetector" 48 | }, 49 | { 50 | "name": "PrivateKeyDetector" 51 | }, 52 | { 53 | "name": "SendGridDetector" 54 | }, 55 | { 56 | "name": "SlackDetector" 57 | }, 58 | { 59 | "name": "SoftlayerDetector" 60 | }, 61 | { 62 | "name": "SquareOAuthDetector" 63 | }, 64 | { 65 | "name": "StripeDetector" 66 | }, 67 | { 68 | "name": "TwilioKeyDetector" 69 | } 70 | ], 71 | "filters_used": [ 72 | { 73 | "path": "detect_secrets.filters.allowlist.is_line_allowlisted" 74 | }, 75 | { 76 | "path": "detect_secrets.filters.common.is_baseline_file", 77 | "filename": ".secrets.baseline" 78 | }, 79 | { 80 | "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", 81 | "min_level": 2 82 | }, 83 | { 84 | "path": "detect_secrets.filters.heuristic.is_indirect_reference" 85 | }, 86 | { 87 | "path": "detect_secrets.filters.heuristic.is_likely_id_string" 88 | }, 89 | { 90 | "path": "detect_secrets.filters.heuristic.is_lock_file" 91 | }, 92 | { 93 | "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" 94 | }, 95 | { 96 | "path": "detect_secrets.filters.heuristic.is_potential_uuid" 97 | }, 98 | { 99 | "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" 100 | }, 101 | { 102 | "path": "detect_secrets.filters.heuristic.is_sequential_string" 103 | }, 104 | { 105 | "path": "detect_secrets.filters.heuristic.is_swagger_file" 106 | }, 107 | { 108 | "path": "detect_secrets.filters.heuristic.is_templated_secret" 109 | }, 110 | { 111 | "path": "detect_secrets.filters.regex.should_exclude_file", 112 | "pattern": [ 113 | "poetry\\.lock", 114 | "\\.secrets\\.baseline", 115 | "\\.env\\.template", 116 | ".*\\\\.ipynb$" 117 | ] 118 | }, 119 | { 120 | "path": "detect_secrets.filters.regex.should_exclude_line", 121 | "pattern": [ 122 | "integrity=*sha" 123 | ] 124 | }, 125 | { 126 | "path": "detect_secrets.filters.regex.should_exclude_secret", 127 | "pattern": [ 128 | "password|ENTER_PASSWORD_HERE|INSERT_API_KEY_HERE" 129 | ] 130 | } 131 | ], 132 | "results": { 133 | "notebooks/2020-05-05 - Python Productivity Power-Ups (PyData UK).ipynb": [ 134 | { 135 | "type": "Hex High Entropy String", 136 | "filename": "notebooks/2020-05-05 - Python Productivity Power-Ups (PyData UK).ipynb", 137 | "hashed_secret": "8031711dbab838661298cc0282eaa670a3d3febe", 138 | "is_verified": false, 139 | "line_number": 531, 140 | "is_secret": false 141 | } 142 | ], 143 | "notebooks/2022-09-24 - Python Productivity Power-Ups (PyCon PT).ipynb": [ 144 | { 145 | "type": "Hex High Entropy String", 146 | "filename": "notebooks/2022-09-24 - Python Productivity Power-Ups (PyCon PT).ipynb", 147 | "hashed_secret": "8031711dbab838661298cc0282eaa670a3d3febe", 148 | "is_verified": false, 149 | "line_number": 846, 150 | "is_secret": false 151 | } 152 | ], 153 | "notebooks/2022-09-24 - Talk Timer.ipynb": [ 154 | { 155 | "type": "Hex High Entropy String", 156 | "filename": "notebooks/2022-09-24 - Talk Timer.ipynb", 157 | "hashed_secret": "8031711dbab838661298cc0282eaa670a3d3febe", 158 | "is_verified": false, 159 | "line_number": 57, 160 | "is_secret": false 161 | } 162 | ] 163 | }, 164 | "generated_at": "2022-09-24T15:52:41Z" 165 | } 166 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Update all versions in this file by running: 2 | # $ pre-commit autoupdate 3 | minimum_pre_commit_version: 2.20.0 4 | default_language_version: 5 | python: python3.10 6 | # exclude: isort.py 7 | repos: 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v4.4.0 10 | hooks: 11 | - id: end-of-file-fixer 12 | name: Check for a blank line at the end of scripts (auto-fixes) 13 | - id: trailing-whitespace 14 | - id: check-builtin-literals 15 | - id: check-byte-order-marker 16 | - id: check-case-conflict 17 | - id: check-merge-conflict 18 | - id: check-symlinks 19 | - id: check-toml 20 | - id: check-vcs-permalinks 21 | - id: check-xml 22 | - id: debug-statements 23 | - id: detect-private-key 24 | - id: mixed-line-ending 25 | - id: fix-encoding-pragma 26 | args: ["--remove"] 27 | - id: check-yaml 28 | - id: check-added-large-files 29 | name: Check for files larger than 5 MB 30 | args: ["--maxkb=5120"] 31 | - id: check-ast 32 | - id: check-docstring-first 33 | - repo: https://github.com/PyCQA/autoflake 34 | rev: v2.0.1 35 | hooks: 36 | - id: autoflake 37 | args: &autoflake 38 | - --in-place 39 | - --remove-all-unused-imports 40 | - --expand-star-imports 41 | - --remove-duplicate-keys 42 | - --remove-unused-variables 43 | - repo: https://github.com/PyCQA/flake8 44 | rev: 5.0.4 45 | hooks: 46 | - &flake8 47 | id: flake8 48 | additional_dependencies: 49 | - flake8-docstrings==1.6.0 50 | - flake8-eradicate==1.4.0 51 | - flake8-fixme==1.1.1 52 | - flake8-implicit-str-concat==0.3.0 53 | - flake8-no-pep420==2.3.0 54 | - flake8-print==5.0.0 55 | - flake8-return==1.1.3 56 | args: ["--config=setup.cfg"] 57 | - repo: https://github.com/psf/black 58 | rev: 23.1.0 59 | hooks: 60 | - id: black 61 | - repo: https://github.com/asottile/add-trailing-comma 62 | rev: v2.4.0 63 | hooks: 64 | - id: add-trailing-comma 65 | args: [--py36-plus] 66 | - repo: https://github.com/timothycrosley/isort 67 | rev: 5.12.0 68 | hooks: 69 | - id: isort 70 | types: [python] 71 | - repo: https://github.com/asottile/pyupgrade 72 | rev: v3.3.1 73 | hooks: 74 | - id: pyupgrade 75 | args: 76 | - "--py310-plus" 77 | # - repo: https://github.com/Yelp/detect-secrets 78 | # rev: v1.3.0 79 | # hooks: 80 | # - id: detect-secrets 81 | # name: detect-secrets - Detect secrets in staged code 82 | # args: [ 83 | # "--baseline", 84 | # ".secrets.baseline", 85 | # # https://regex101.com/ 86 | # "--exclude-files 'poetry\\.lock'", 87 | # "--exclude-files '\\.secrets\\.baseline'", 88 | # "--exclude-files '\\.env\\.template'", 89 | # "--exclude-files '.*\\.ipynb$'", 90 | # # "--exclude-files '.*build/'", 91 | # "--exclude-secrets 'password|ENTER_PASSWORD_HERE|INSERT_API_KEY_HERE'", 92 | # "--exclude-lines 'integrity=*sha'", 93 | # ] 94 | # # https://pre-commit.com/#regular-expressions 95 | # exclude: | 96 | # (?x)^( 97 | # poetry\.lock| 98 | # \.secrets\.baseline| 99 | # \.env\.template 100 | # )$ 101 | # Poetry is not finding the correct repo 102 | # - repo: https://github.com/python-poetry/poetry 103 | # rev: 1.2.1 104 | # hooks: 105 | # - id: poetry-check 106 | # - id: poetry-lock 107 | # args: ["--no-update"] 108 | # - id: poetry-export 109 | # args: 110 | # [ 111 | # "-f", 112 | # "requirements.txt", 113 | # "-o", 114 | # "requirements.txt", 115 | # "--without-hashes", 116 | # ] 117 | - repo: https://github.com/pre-commit/mirrors-prettier 118 | rev: v3.0.0-alpha.4 119 | hooks: 120 | - id: prettier 121 | types_or: [yaml] 122 | additional_dependencies: 123 | - "prettier@2.7.1" 124 | - repo: local 125 | hooks: 126 | - id: pylint 127 | name: pylint 128 | entry: pylint notebooks 129 | # entry: bash -c 'pylint ./path/package1/; pylint ./path/package2/' 130 | language: system 131 | types: [python] 132 | always_run: true 133 | pass_filenames: false 134 | stages: [manual] 135 | - id: pip-audit 136 | name: pip-audit 137 | entry: pip-audit 138 | language: system 139 | always_run: true 140 | pass_filenames: false 141 | stages: [manual] 142 | -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | This document contains instructions to get a fully working development environment for running this repo. 4 | 5 | 6 | ## 1. pyenv 7 | 8 | Install here: [https://github.com/pyenv/pyenv#homebrew-on-macos] 9 | 10 | Configure by adding the following to your `~/.zshrc` or equivalent: 11 | 12 | ```sh 13 | # Pyenv environment variables 14 | export PYENV_ROOT="$HOME/.pyenv" 15 | export PATH="$PYENV_ROOT/bin:$PATH" 16 | 17 | # Pyenv initialization 18 | eval "$(pyenv init --path)" 19 | eval "$(pyenv init -)" 20 | ``` 21 | 22 | Basic usage: 23 | 24 | ```sh 25 | # Check Python versions 26 | pyenv install --list 27 | 28 | # Install the Python version defined in this repo 29 | pyenv install $(cat .python-version) 30 | 31 | # See installed Python versions 32 | pyenv versions 33 | ``` 34 | 35 | 36 | ## 2. [pyenv-virtualenvwrapper](https://github.com/pyenv/pyenv-virtualenvwrapper) 37 | 38 | ```sh 39 | # Install with homebrew (recommended if you installed pyenv with homebrew) 40 | brew install pyenv-virtualenvwrapper 41 | ``` 42 | 43 | Configure by adding the following to your `~/.zshrc` or equivalent: 44 | 45 | ```sh 46 | # pyenv-virtualenvwrapper 47 | export PYENV_VIRTUALENVWRAPPER_PREFER_PYVENV="true" 48 | export WORKON_HOME=$HOME/.virtualenvs 49 | export PROJECT_HOME=$HOME/code # <- change this to wherever you store your repos 50 | export VIRTUALENVWRAPPER_PYTHON=$HOME/.pyenv/shims/python 51 | pyenv virtualenvwrapper_lazy 52 | ``` 53 | 54 | Test everything is working by opening a new shell (e.g. new Terminal window): 55 | 56 | ```sh 57 | # Change to the Python version you just installed 58 | pyenv shell $(cat .python-version) 59 | # This only needs to be run once after installing a new Python version through pyenv 60 | # in order to initialise virtualenvwrapper for this Python version 61 | python -m pip install --upgrade pip 62 | python -m pip install virtualenvwrapper 63 | pyenv virtualenvwrapper_lazy 64 | 65 | # Create test virtualenv (if this doesn't work, try sourcing ~/.zshrc or opening new shell) 66 | mkvirtualenv venv_test 67 | which python 68 | python -V 69 | 70 | # Deactivate & remove test virtualenv 71 | deactivate 72 | rmvirtualenv venv_test 73 | ``` 74 | 75 | 76 | ## 3. Get the repo & initialise the repo environment 77 | 78 | ⚠️ N.B. You should replace `REPO_GIT_URL` here with your actual URL to your GitHub repo. 79 | 80 | ```sh 81 | git clone ${REPO_GIT_URL} 82 | pyenv shell $(cat .python-version) 83 | 84 | # Make a new virtual environment using the Python version & environment name specified in the repo 85 | mkvirtualenv -p python$(cat .python-version) $(cat .venv) 86 | python -V # check this is the correct version of Python 87 | python -m pip install --upgrade pip 88 | ``` 89 | 90 | 91 | ## 4. Install Python requirements into the virtual environment using [Poetry](https://python-poetry.org/docs/) 92 | 93 | Install Poetry onto your system by following the instructions here: [https://python-poetry.org/docs/] 94 | 95 | Note that Poetry "lives" outside of project/environment, and if you follow the recommended install 96 | process it will be installed isolated from the rest of your system. 97 | 98 | ```sh 99 | # Update Poetry regularly as you would any other system-level tool. Poetry is environment agnostic, 100 | # it doesn't matter if you run this command inside/outside the virtualenv. 101 | poetry self update 102 | 103 | # This command should be run inside the virtualenv. 104 | poetry install --no-root --remove-untracked 105 | ``` 106 | 107 | 108 | ## 5. [zsh-autoswitch-virtualenv](https://github.com/MichaelAquilina/zsh-autoswitch-virtualenv) 109 | 110 | Download with `git clone "https://github.com/MichaelAquilina/zsh-autoswitch-virtualenv.git" "$ZSH_CUSTOM/plugins/autoswitch_virtualenv"` 111 | 112 | Configure by adding the following to your `~/.zshrc` or equivalent: 113 | 114 | ```sh 115 | # Find line containing plugins=(git) and replace with below 116 | plugins=(git autoswitch_virtualenv) 117 | ``` 118 | 119 | Check it's working by cd-ing into & out of the repo. The environment should load & unload respectively. 120 | 121 | 122 | ## 6. Add secrets into .env 123 | 124 | - Run `cp .env.template .env` and update the secrets. 125 | - [Install direnv](https://direnv.net/) to autoload environment variables specified in `.env` 126 | - Run `direnv allow` to authorise direnv to load the secrets from `.env` into the environment 127 | (these will unload when you `cd` out of the repo; note that you will need to re-run this 128 | command whenever you change `.env`) 129 | 130 | 131 | ## 7. Initialise the `detect-secrets` pre-commit hook 132 | 133 | We use [`detect-secrets`](https://github.com/Yelp/detect-secrets) to check that no secrets are 134 | accidentally committed. Please read [docs/detect_secrets.md](docs/detect_secrets.md) for more information. 135 | 136 | 137 | ```shell 138 | # Generate a baseline 139 | detect-secrets scan > .secrets.baseline 140 | 141 | # You may want to check/amend the exclusions in `.pre-commit-config.yaml` e.g. 142 | detect-secrets --verbose scan \ 143 | --exclude-files 'poetry\.lock' \ 144 | --exclude-files '\.secrets\.baseline' \ 145 | --exclude-files '\.env\.template' \ 146 | --exclude-files '.*\\.ipynb$' \ 147 | --exclude-secrets 'password|ENTER_PASSWORD_HERE|INSERT_API_KEY_HERE' \ 148 | --exclude-lines 'integrity=*sha' \ 149 | > .secrets.baseline 150 | 151 | # Audit the generated baseline 152 | detect-secrets audit .secrets.baseline 153 | ``` 154 | 155 | When you run this command, you'll enter an interactive console. This will present you with a list 156 | of high-entropy string and/or anything which could be a secret. It will then ask you to verify 157 | whether this is the case. This allows the hook to remember false positives in the future, and alert 158 | you to new secrets. 159 | 160 | 161 | ## 8. Project-specific setup 162 | 163 | Please check [docs/project_specific_setup.md](docs/project_specific_setup.md) for further instructions. 164 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiofiles==22.1.0 ; python_full_version >= "3.10.6" and python_version < "4.0" 2 | aiosqlite==0.18.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 3 | anyio==3.6.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 4 | appnope==0.1.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" and platform_system == "Darwin" or python_full_version >= "3.10.6" and python_full_version < "4.0.0" and sys_platform == "darwin" 5 | argon2-cffi-bindings==21.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 6 | argon2-cffi==21.3.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 7 | arrow==1.2.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 8 | asttokens==2.2.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 9 | attrs==22.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 10 | babel==2.11.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 11 | backcall==0.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 12 | bandit==1.7.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 13 | beautifulsoup4==4.11.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 14 | black==22.12.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 15 | bleach==6.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 16 | certifi==2022.12.7 ; python_full_version >= "3.10.6" and python_version < "4" 17 | cffi==1.15.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 18 | charset-normalizer==3.0.1 ; python_full_version >= "3.10.6" and python_version < "4" 19 | click==8.1.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 20 | colorama==0.4.6 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" and sys_platform == "win32" or python_full_version >= "3.10.6" and python_full_version < "4.0.0" and platform_system == "Windows" 21 | comm==0.1.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 22 | contourpy==1.0.7 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 23 | cycler==0.11.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 24 | debugpy==1.6.6 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 25 | decorator==5.1.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 26 | defusedxml==0.7.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 27 | dill==0.3.6 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 28 | docopt==0.6.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 29 | executing==1.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 30 | fastjsonschema==2.16.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 31 | fonttools==4.38.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 32 | fqdn==1.5.1 ; python_full_version >= "3.10.6" and python_version < "4" 33 | gitdb==4.0.10 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 34 | gitpython==3.1.31 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 35 | idna==3.4 ; python_full_version >= "3.10.6" and python_version < "4" 36 | ipykernel==6.21.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 37 | ipython-genutils==0.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 38 | ipython==8.10.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 39 | ipywidgets==8.0.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 40 | isoduration==20.11.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 41 | isort==5.12.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 42 | jedi==0.18.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 43 | jinja2==3.1.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 44 | joblib==1.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 45 | json5==0.9.11 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 46 | jsonpointer==2.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 47 | jsonschema==4.17.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 48 | jsonschema[format-nongpl]==4.17.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 49 | jupyter-black==0.3.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 50 | jupyter-client==8.0.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 51 | jupyter-console==6.6.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 52 | jupyter-core==5.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 53 | jupyter-events==0.6.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 54 | jupyter-server-fileid==0.7.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 55 | jupyter-server-terminals==0.4.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 56 | jupyter-server-ydoc==0.6.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 57 | jupyter-server==2.3.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 58 | jupyter-ydoc==0.2.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 59 | jupyter==1.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 60 | jupyterlab-pygments==0.2.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 61 | jupyterlab-server==2.19.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 62 | jupyterlab-widgets==3.0.5 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 63 | jupyterlab==3.6.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 64 | kiwisolver==1.4.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 65 | loguru==0.6.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 66 | markupsafe==2.1.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 67 | matplotlib-inline==0.1.6 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 68 | matplotlib==3.7.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 69 | mistune==2.0.5 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 70 | mypy-extensions==1.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 71 | nbclassic==0.5.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 72 | nbclient==0.7.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 73 | nbconvert==7.2.9 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 74 | nbformat==5.7.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 75 | nest-asyncio==1.5.6 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 76 | notebook-shim==0.2.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 77 | notebook==6.5.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 78 | numpy==1.24.2 ; python_full_version < "4.0.0" and python_full_version >= "3.10.6" 79 | packaging==23.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 80 | pandarallel==1.6.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 81 | pandas==1.5.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 82 | pandocfilters==1.5.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 83 | parso==0.8.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 84 | pathspec==0.11.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 85 | pbr==5.11.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 86 | pexpect==4.8.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" and sys_platform != "win32" 87 | pickleshare==0.7.5 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 88 | pillow==9.4.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 89 | platformdirs==3.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 90 | prometheus-client==0.16.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 91 | prompt-toolkit==3.0.37 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 92 | psutil==5.9.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 93 | ptyprocess==0.7.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" and sys_platform != "win32" or python_full_version >= "3.10.6" and python_full_version < "4.0.0" and os_name != "nt" 94 | pure-eval==0.2.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 95 | pycparser==2.21 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 96 | pygments==2.14.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 97 | pyparsing==3.0.9 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 98 | pyrsistent==0.19.3 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 99 | python-dateutil==2.8.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 100 | python-json-logger==2.0.7 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 101 | pytz==2022.7.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 102 | pywin32==305 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_full_version >= "3.10.6" and python_full_version < "4.0.0" 103 | pywinpty==2.0.10 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" and os_name == "nt" 104 | pyyaml==6.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 105 | pyzmq==25.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 106 | qtconsole==5.4.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 107 | qtpy==2.3.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 108 | requests==2.28.2 ; python_full_version >= "3.10.6" and python_version < "4" 109 | rfc3339-validator==0.1.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 110 | rfc3986-validator==0.1.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 111 | seaborn==0.12.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 112 | send2trash==1.8.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 113 | six==1.16.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 114 | smmap==5.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 115 | sniffio==1.3.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 116 | soupsieve==2.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 117 | stack-data==0.6.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 118 | stevedore==5.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 119 | terminado==0.17.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 120 | tinycss2==1.2.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 121 | tokenize-rt==5.0.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 122 | tomli==2.0.1 ; python_full_version >= "3.10.6" and python_full_version < "3.11.0a7" 123 | tornado==6.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 124 | tqdm==4.64.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 125 | traitlets==5.9.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 126 | treon==0.1.4 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 127 | uri-template==1.2.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 128 | urllib3==1.26.14 ; python_full_version >= "3.10.6" and python_version < "4" 129 | wcwidth==0.2.6 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 130 | webcolors==1.12 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 131 | webencodings==0.5.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 132 | websocket-client==1.5.1 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 133 | widgetsnbextension==4.0.5 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 134 | win32-setctime==1.1.0 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" and sys_platform == "win32" 135 | y-py==0.5.9 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 136 | ypy-websocket==0.8.2 ; python_full_version >= "3.10.6" and python_full_version < "4.0.0" 137 | -------------------------------------------------------------------------------- /notebooks/2020-05-05 - Python Productivity Power-Ups (PyData UK).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[![pydatauk](../images/pydatauk-may-logo.jpeg)](https://www.meetup.com/PyData-Manchester/events/270272244/)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# Python Productivity Power-Ups\n", 15 | "1. tqdm: https://pypi.org/project/tqdm/\n", 16 | "2. black: https://pypi.org/project/black/\n", 17 | "3. nb-black: https://pypi.org/project/nb-black/\n", 18 | "4. isort: https://github.com/timothycrosley/isort\n", 19 | "5. logaru: https://github.com/Delgan/loguru\n", 20 | "6. joblib: https://joblib.readthedocs.io/en/latest/" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "import time\n", 30 | "\n", 31 | "import numpy as np\n", 32 | "import pandas as pd" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "## tqdm" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": null, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "from tqdm import tqdm" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "for i in tqdm(range(500)):\n", 58 | " time.sleep(0.01)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "### tqdm and pandas" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "# Run this once in your script/notebook\n", 96 | "tqdm.pandas()" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "df = pd.DataFrame({'n': np.random.randint(1, 100, 100000000)})\n", 106 | "df.head()" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "# Then just use .progress_apply() wherever you'd usually use .apply()\n", 116 | "df['n_squared'] = df.progress_apply(lambda x: np.cos(x**2))" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "### Talk timer!" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "import time\n", 161 | "from tqdm import tqdm\n", 162 | "\n", 163 | "for i in tqdm(range(4*60)):\n", 164 | " time.sleep(1)" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "metadata": {}, 198 | "source": [ 199 | "## black" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "# This is where the magic happens. ✨\n", 209 | "%load_ext jupyter_black" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [ 218 | "j = [\n", 219 | " 1 ,\n", 220 | " 2,\n", 221 | " 3\n", 222 | "]" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "import os\n", 232 | "\n", 233 | "def very_important_function(template: str, *variables, file: os.PathLike, engine: str, header: bool = True, debug: bool = False):\n", 234 | " \"\"\"Does something very important.\"\"\"\n", 235 | " pass" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "metadata": {}, 242 | "outputs": [], 243 | "source": [] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "metadata": {}, 256 | "outputs": [], 257 | "source": [] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [] 265 | }, 266 | { 267 | "cell_type": "markdown", 268 | "metadata": {}, 269 | "source": [ 270 | "## isort" 271 | ] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "metadata": {}, 276 | "source": [ 277 | "![](https://camo.githubusercontent.com/841a7830ea6e8d363a53214917144b259685cc15/68747470733a2f2f7261772e6769746875622e636f6d2f74696d6f74687963726f736c65792f69736f72742f646576656c6f702f6578616d706c652e676966)" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": null, 283 | "metadata": {}, 284 | "outputs": [], 285 | "source": [] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": null, 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": null, 297 | "metadata": {}, 298 | "outputs": [], 299 | "source": [] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": null, 304 | "metadata": {}, 305 | "outputs": [], 306 | "source": [] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "metadata": {}, 311 | "source": [ 312 | "## logaru" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "metadata": {}, 319 | "outputs": [], 320 | "source": [ 321 | "from loguru import logger" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": null, 327 | "metadata": {}, 328 | "outputs": [], 329 | "source": [ 330 | "logger.debug(\"That's it, beautiful and simple logging!\")" 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": null, 336 | "metadata": {}, 337 | "outputs": [], 338 | "source": [ 339 | "logger.error(\"😱😱😱😱😱\")" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": null, 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": null, 352 | "metadata": {}, 353 | "outputs": [], 354 | "source": [] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": null, 366 | "metadata": {}, 367 | "outputs": [], 368 | "source": [] 369 | }, 370 | { 371 | "cell_type": "markdown", 372 | "metadata": {}, 373 | "source": [ 374 | "## joblib" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": null, 380 | "metadata": {}, 381 | "outputs": [], 382 | "source": [ 383 | "from joblib import Memory\n", 384 | "from pathlib import Path\n", 385 | "\n", 386 | "# Config\n", 387 | "CACHE_DIR = Path(\".\") / \"cache\"\n", 388 | "memory = Memory(CACHE_DIR, verbose=0)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": null, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "@memory.cache\n", 398 | "def calculate(x):\n", 399 | " time.sleep(3)\n", 400 | " return x * 10" 401 | ] 402 | }, 403 | { 404 | "cell_type": "code", 405 | "execution_count": null, 406 | "metadata": {}, 407 | "outputs": [], 408 | "source": [ 409 | "calculate(666) # slow 😴" 410 | ] 411 | }, 412 | { 413 | "cell_type": "code", 414 | "execution_count": null, 415 | "metadata": {}, 416 | "outputs": [], 417 | "source": [ 418 | "calculate(666) # fast! 🏃‍♂️" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": null, 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "%%time\n", 428 | "calculate(42)" 429 | ] 430 | }, 431 | { 432 | "cell_type": "code", 433 | "execution_count": null, 434 | "metadata": {}, 435 | "outputs": [], 436 | "source": [ 437 | "%%time\n", 438 | "calculate(42)" 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "metadata": {}, 445 | "outputs": [], 446 | "source": [] 447 | }, 448 | { 449 | "cell_type": "code", 450 | "execution_count": null, 451 | "metadata": {}, 452 | "outputs": [], 453 | "source": [] 454 | }, 455 | { 456 | "cell_type": "code", 457 | "execution_count": null, 458 | "metadata": {}, 459 | "outputs": [], 460 | "source": [] 461 | }, 462 | { 463 | "cell_type": "code", 464 | "execution_count": null, 465 | "metadata": {}, 466 | "outputs": [], 467 | "source": [] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "---\n", 474 | "# `https://github.com/john-sandall/python-productivity-powerups`" 475 | ] 476 | }, 477 | { 478 | "cell_type": "markdown", 479 | "metadata": {}, 480 | "source": [ 481 | "---\n", 482 | "\n", 483 | "
\n", 484 | " About\n", 485 | "

\n", 486 | " This notebook has been made by @John_Sandall. I run training workshops in Python, data science and data engineering.\n", 487 | "


\n", 488 | "

\n", 489 | " You can follow my free First Steps with Python and First Steps with pandas workshops for free as part of PyData Bristol's Zero To Hero 2020 monthly free workshop series. PyData Bristol will be running more free virtual workshops over the coming months so sign up via Meetup.com or follow us @PyDataBristol on Twitter.\n", 490 | "


\n", 491 | "

\n", 492 | " I am the Founder of data science consultancy Coefficient. If you would like to work with us, our team can help you with your data science, software engineering and machine learning projects as an on-demand resource. We can also create bespoke training workshops adapted to your industry, virtual or in-person, with training clients currently including BNP Paribas, EY, the Met Police and the BBC.\n", 493 | "

\n", 494 | "
" 495 | ] 496 | } 497 | ], 498 | "metadata": { 499 | "kernelspec": { 500 | "display_name": "Python 3.10.6 ('ppp')", 501 | "language": "python", 502 | "name": "python3" 503 | }, 504 | "language_info": { 505 | "codemirror_mode": { 506 | "name": "ipython", 507 | "version": 3 508 | }, 509 | "file_extension": ".py", 510 | "mimetype": "text/x-python", 511 | "name": "python", 512 | "nbconvert_exporter": "python", 513 | "pygments_lexer": "ipython3", 514 | "version": "3.10.6" 515 | }, 516 | "toc": { 517 | "base_numbering": 1, 518 | "nav_menu": {}, 519 | "number_sections": true, 520 | "sideBar": true, 521 | "skip_h1_title": true, 522 | "title_cell": "Table of Contents", 523 | "title_sidebar": "Contents", 524 | "toc_cell": true, 525 | "toc_position": {}, 526 | "toc_section_display": true, 527 | "toc_window_display": true 528 | }, 529 | "vscode": { 530 | "interpreter": { 531 | "hash": "be34d95dbc58180ac3633c8e41e8772c710783434d94474c3fd4437fff7f4790" 532 | } 533 | } 534 | }, 535 | "nbformat": 4, 536 | "nbformat_minor": 4 537 | } 538 | -------------------------------------------------------------------------------- /notebooks/2023-03-07 - Python Productivity Power-Ups (GeoPython 2023).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[![geopython](../images/geopython-header.jpg)](https://2023.geopython.net/)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# 🐍 Python Productivity Power-Ups 🚀\n", 15 | "## 👤 John Sandall" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "---\n", 23 | "
\n", 24 | "
\n", 25 | "
\n", 26 | "
\n", 27 | "
\n", 28 | "
\n", 29 | "
\n", 30 | "
\n", 31 | "
\n", 32 | "
\n", 33 | "
\n", 34 | "
\n", 35 | "
\n", 36 | "
\n", 37 | "
\n", 38 | "
\n", 39 | "
\n", 40 | "
\n", 41 | "\n", 42 | "---" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "[![about](../images/about-me.jpeg)](https://coefficient.ai)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "# Python Productivity Power-Ups\n", 57 | "1. tqdm: https://pypi.org/project/tqdm/\n", 58 | "2. joblib: https://joblib.readthedocs.io/en/latest/\n", 59 | "3. black: https://pypi.org/project/black/\n", 60 | "4. jupyter-black: https://pypi.org/project/jupyter-black/\n", 61 | "5. isort: https://github.com/timothycrosley/isort\n", 62 | "6. fzf: https://github.com/junegunn/fzf\n", 63 | "7. pre-commit: https://pre-commit.com/\n", 64 | "8. coefficient-cookiecutter: https://github.com/CoefficientSystems/coefficient-cookiecutter" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "import time\n", 74 | "\n", 75 | "import numpy as np\n", 76 | "import pandas as pd" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "## tqdm" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "from tqdm import tqdm" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "for i in range(500):\n", 102 | " time.sleep(0.01)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "⬇\n", 110 | "
\n", 111 | "⬇" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": {}, 118 | "outputs": [], 119 | "source": [ 120 | "for i in tqdm(range(500)):\n", 121 | " time.sleep(0.01)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "metadata": {}, 127 | "source": [ 128 | "---\n", 129 | "
\n", 130 | "
\n", 131 | "
\n", 132 | "
\n", 133 | "
\n", 134 | "
\n", 135 | "
\n", 136 | "
\n", 137 | "
\n", 138 | "
\n", 139 | "
\n", 140 | "
\n", 141 | "
\n", 142 | "
\n", 143 | "
\n", 144 | "
\n", 145 | "
\n", 146 | "
\n", 147 | "\n", 148 | "---" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "### tqdm and pandas" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "total_rows = 1_000_0000\n", 165 | "df = pd.DataFrame({'n': np.random.randint(1, 100, total_rows)})\n", 166 | "df" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "%%time\n", 176 | "\n", 177 | "# Let's first do this using .apply()\n", 178 | "df['n_squared'] = df['n'].apply(lambda x: np.cos(x**2))" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "⬇\n", 186 | "
\n", 187 | "⬇" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "# Run this once in your script/notebook\n", 197 | "tqdm.pandas()" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": null, 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "%%time\n", 207 | "\n", 208 | "# Then just use .progress_apply() wherever you'd usually use .apply()\n", 209 | "df['n_squared'] = df['n'].progress_apply(lambda x: np.cos(x**2))" 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "⬇\n", 217 | "
\n", 218 | "⬇" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [ 227 | "from pandarallel import pandarallel\n", 228 | "\n", 229 | "pandarallel.initialize(progress_bar=True)" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "metadata": {}, 236 | "outputs": [], 237 | "source": [ 238 | "%%time\n", 239 | "\n", 240 | "df['n_squared'] = df['n'].parallel_apply(lambda x: np.cos(x**2))" 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": {}, 246 | "source": [ 247 | "---\n", 248 | "
\n", 249 | "
\n", 250 | "
\n", 251 | "
\n", 252 | "
\n", 253 | "
\n", 254 | "
\n", 255 | "
\n", 256 | "
\n", 257 | "
\n", 258 | "
\n", 259 | "
\n", 260 | "
\n", 261 | "
\n", 262 | "
\n", 263 | "
\n", 264 | "
\n", 265 | "
\n", 266 | "\n", 267 | "---" 268 | ] 269 | }, 270 | { 271 | "cell_type": "markdown", 272 | "metadata": {}, 273 | "source": [ 274 | "## joblib" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [ 283 | "def calculate(x):\n", 284 | " time.sleep(3)\n", 285 | " return x * 10" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": null, 291 | "metadata": {}, 292 | "outputs": [], 293 | "source": [ 294 | "calculate(666) # slow 😴" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "⬇\n", 302 | "
\n", 303 | "⬇" 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": null, 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [ 312 | "from joblib import Memory\n", 313 | "\n", 314 | "memory = Memory(\"./cache\", verbose=0)" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": null, 320 | "metadata": {}, 321 | "outputs": [], 322 | "source": [ 323 | "@memory.cache\n", 324 | "def calculate(x):\n", 325 | " time.sleep(3)\n", 326 | " return x * 10" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "calculate(666) # slow 😴" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [ 344 | "calculate(666) # fast! 🏃‍♂️" 345 | ] 346 | }, 347 | { 348 | "cell_type": "markdown", 349 | "metadata": {}, 350 | "source": [ 351 | "⬇\n", 352 | "
\n", 353 | "⬇" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [ 362 | "%%time\n", 363 | "calculate(42)" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": null, 369 | "metadata": {}, 370 | "outputs": [], 371 | "source": [ 372 | "%%time\n", 373 | "calculate(42)" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": {}, 379 | "source": [ 380 | "⬇\n", 381 | "
\n", 382 | "⬇" 383 | ] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": null, 388 | "metadata": {}, 389 | "outputs": [], 390 | "source": [ 391 | "# Clear the cache so I can give this talk again!\n", 392 | "memory.clear()" 393 | ] 394 | }, 395 | { 396 | "cell_type": "markdown", 397 | "metadata": {}, 398 | "source": [ 399 | "---\n", 400 | "
\n", 401 | "
\n", 402 | "
\n", 403 | "
\n", 404 | "
\n", 405 | "
\n", 406 | "
\n", 407 | "
\n", 408 | "
\n", 409 | "
\n", 410 | "
\n", 411 | "
\n", 412 | "
\n", 413 | "
\n", 414 | "
\n", 415 | "
\n", 416 | "
\n", 417 | "
\n", 418 | "\n", 419 | "---" 420 | ] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": {}, 425 | "source": [ 426 | "## black" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": null, 432 | "metadata": {}, 433 | "outputs": [], 434 | "source": [ 435 | "# This is where the magic happens. ✨\n", 436 | "%load_ext jupyter_black" 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "execution_count": null, 442 | "metadata": {}, 443 | "outputs": [], 444 | "source": [ 445 | "j = [\n", 446 | " 1 , \n", 447 | " 2, 3\n", 448 | "]" 449 | ] 450 | }, 451 | { 452 | "cell_type": "code", 453 | "execution_count": null, 454 | "metadata": {}, 455 | "outputs": [], 456 | "source": [ 457 | "import os\n", 458 | "\n", 459 | "def very_important_function(template: str,*variables,file: os.PathLike,debug:bool=False,):\n", 460 | " \"\"\"Does something very important.\"\"\"\n", 461 | " pass" 462 | ] 463 | }, 464 | { 465 | "cell_type": "markdown", 466 | "metadata": {}, 467 | "source": [ 468 | "---\n", 469 | "
\n", 470 | "
\n", 471 | "
\n", 472 | "
\n", 473 | "
\n", 474 | "
\n", 475 | "
\n", 476 | "
\n", 477 | "
\n", 478 | "
\n", 479 | "
\n", 480 | "
\n", 481 | "
\n", 482 | "
\n", 483 | "
\n", 484 | "
\n", 485 | "
\n", 486 | "
\n", 487 | "\n", 488 | "---" 489 | ] 490 | }, 491 | { 492 | "cell_type": "markdown", 493 | "metadata": {}, 494 | "source": [ 495 | "## isort" 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "metadata": {}, 501 | "source": [ 502 | "![](https://camo.githubusercontent.com/841a7830ea6e8d363a53214917144b259685cc15/68747470733a2f2f7261772e6769746875622e636f6d2f74696d6f74687963726f736c65792f69736f72742f646576656c6f702f6578616d706c652e676966)" 503 | ] 504 | }, 505 | { 506 | "cell_type": "markdown", 507 | "metadata": {}, 508 | "source": [ 509 | "---" 510 | ] 511 | }, 512 | { 513 | "cell_type": "markdown", 514 | "metadata": {}, 515 | "source": [ 516 | "## pre-commit" 517 | ] 518 | }, 519 | { 520 | "cell_type": "markdown", 521 | "metadata": {}, 522 | "source": [ 523 | "## fzf" 524 | ] 525 | }, 526 | { 527 | "cell_type": "markdown", 528 | "metadata": {}, 529 | "source": [ 530 | "## coefficient-cookiecutter" 531 | ] 532 | }, 533 | { 534 | "cell_type": "markdown", 535 | "metadata": {}, 536 | "source": [ 537 | "---\n", 538 | "# Follow me at @john_sandall for links to all of the above" 539 | ] 540 | }, 541 | { 542 | "cell_type": "markdown", 543 | "metadata": {}, 544 | "source": [ 545 | "---\n", 546 | "# `https://github.com/john-sandall/python-productivity-powerups`" 547 | ] 548 | }, 549 | { 550 | "cell_type": "markdown", 551 | "metadata": {}, 552 | "source": [ 553 | "---\n", 554 | "\n", 555 | "
\n", 556 | " About\n", 557 | "

\n", 558 | " This notebook has been made by @John_Sandall. I run training workshops in Python, data science and data engineering.\n", 559 | "


\n", 560 | "

\n", 561 | " You can follow my free First Steps with Python and First Steps with pandas workshops for free as part of PyData Bristol's Zero To Hero 2020 monthly free workshop series. PyData Bristol will be running more free virtual workshops over the coming months so sign up via Meetup.com or follow us @PyDataBristol on Twitter.\n", 562 | "


\n", 563 | "

\n", 564 | " I am the Founder of data science consultancy Coefficient. If you would like to work with us, our team can help you with your data science, software engineering and machine learning projects as an on-demand resource. We can also create bespoke training workshops adapted to your industry, virtual or in-person, with training clients currently including BNP Paribas, EY, the Met Police and the BBC.\n", 565 | "

\n", 566 | "
" 567 | ] 568 | } 569 | ], 570 | "metadata": { 571 | "kernelspec": { 572 | "display_name": "Python (ppp)", 573 | "language": "python", 574 | "name": "ppp" 575 | }, 576 | "language_info": { 577 | "codemirror_mode": { 578 | "name": "ipython", 579 | "version": 3 580 | }, 581 | "file_extension": ".py", 582 | "mimetype": "text/x-python", 583 | "name": "python", 584 | "nbconvert_exporter": "python", 585 | "pygments_lexer": "ipython3", 586 | "version": "3.10.6" 587 | }, 588 | "toc": { 589 | "base_numbering": 1, 590 | "nav_menu": {}, 591 | "number_sections": true, 592 | "sideBar": true, 593 | "skip_h1_title": true, 594 | "title_cell": "Table of Contents", 595 | "title_sidebar": "Contents", 596 | "toc_cell": true, 597 | "toc_position": {}, 598 | "toc_section_display": true, 599 | "toc_window_display": true 600 | }, 601 | "vscode": { 602 | "interpreter": { 603 | "hash": "be34d95dbc58180ac3633c8e41e8772c710783434d94474c3fd4437fff7f4790" 604 | } 605 | } 606 | }, 607 | "nbformat": 4, 608 | "nbformat_minor": 4 609 | } 610 | -------------------------------------------------------------------------------- /notebooks/2022-09-24 - Python Productivity Power-Ups (PyCon PT).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[![pycon-pt](../images/pycon-pt-header.jpg)](https://2022.pycon.pt/)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# 🐍 Python Productivity Power-Ups 🚀\n", 15 | "## 👤 John Sandall" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "---\n", 23 | "
\n", 24 | "
\n", 25 | "
\n", 26 | "
\n", 27 | "
\n", 28 | "
\n", 29 | "
\n", 30 | "
\n", 31 | "
\n", 32 | "
\n", 33 | "
\n", 34 | "
\n", 35 | "
\n", 36 | "
\n", 37 | "
\n", 38 | "
\n", 39 | "
\n", 40 | "
\n", 41 | "\n", 42 | "---" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "[![about](../images/about-me.jpeg)](https://coefficient.ai)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "# Python Productivity Power-Ups\n", 57 | "1. tqdm: https://pypi.org/project/tqdm/\n", 58 | "2. joblib: https://joblib.readthedocs.io/en/latest/\n", 59 | "3. black: https://pypi.org/project/black/\n", 60 | "4. jupyter-black: https://pypi.org/project/jupyter-black/\n", 61 | "5. isort: https://github.com/timothycrosley/isort\n", 62 | "6. fzf: https://github.com/junegunn/fzf\n", 63 | "7. pre-commit: https://pre-commit.com/\n", 64 | "8. coefficient-cookiecutter: https://github.com/CoefficientSystems/coefficient-cookiecutter" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 1, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "import time\n", 74 | "\n", 75 | "import numpy as np\n", 76 | "import pandas as pd" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "## tqdm" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 2, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "from tqdm import tqdm" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 3, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "ename": "KeyboardInterrupt", 102 | "evalue": "", 103 | "output_type": "error", 104 | "traceback": [ 105 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 106 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 107 | "\u001b[0;32m/var/folders/30/2pzsy6kx1x5dx_zn6n8h0w8r0000gn/T/ipykernel_50708/1820794216.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.01\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 108 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "for i in range(500):\n", 114 | " time.sleep(0.01)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "⬇\n", 122 | "
\n", 123 | "⬇" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 4, 129 | "metadata": {}, 130 | "outputs": [ 131 | { 132 | "name": "stderr", 133 | "output_type": "stream", 134 | "text": [ 135 | "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [00:06<00:00, 81.07it/s]\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "for i in tqdm(range(500)):\n", 141 | " time.sleep(0.01)" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "---\n", 149 | "
\n", 150 | "
\n", 151 | "
\n", 152 | "
\n", 153 | "
\n", 154 | "
\n", 155 | "
\n", 156 | "
\n", 157 | "
\n", 158 | "
\n", 159 | "
\n", 160 | "
\n", 161 | "
\n", 162 | "
\n", 163 | "
\n", 164 | "
\n", 165 | "
\n", 166 | "
\n", 167 | "\n", 168 | "---" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "### tqdm and pandas" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 5, 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "data": { 185 | "text/html": [ 186 | "
\n", 187 | "\n", 200 | "\n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | "
n
061
123
238
340
471
......
999999567
999999661
999999783
999999882
999999951
\n", 254 | "

10000000 rows × 1 columns

\n", 255 | "
" 256 | ], 257 | "text/plain": [ 258 | " n\n", 259 | "0 61\n", 260 | "1 23\n", 261 | "2 38\n", 262 | "3 40\n", 263 | "4 71\n", 264 | "... ..\n", 265 | "9999995 67\n", 266 | "9999996 61\n", 267 | "9999997 83\n", 268 | "9999998 82\n", 269 | "9999999 51\n", 270 | "\n", 271 | "[10000000 rows x 1 columns]" 272 | ] 273 | }, 274 | "execution_count": 5, 275 | "metadata": {}, 276 | "output_type": "execute_result" 277 | } 278 | ], 279 | "source": [ 280 | "total_rows = 1_000_0000\n", 281 | "df = pd.DataFrame({'n': np.random.randint(1, 100, total_rows)})\n", 282 | "df" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "# Then just use .progress_apply() wherever you'd usually use .apply()\n", 292 | "df['n_squared'] = df['n'].apply(lambda x: np.cos(x**2))" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "⬇\n", 300 | "
\n", 301 | "⬇" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 7, 307 | "metadata": {}, 308 | "outputs": [], 309 | "source": [ 310 | "# Run this once in your script/notebook\n", 311 | "tqdm.pandas()" 312 | ] 313 | }, 314 | { 315 | "cell_type": "code", 316 | "execution_count": null, 317 | "metadata": {}, 318 | "outputs": [], 319 | "source": [ 320 | "# Then just use .progress_apply() wherever you'd usually use .apply()\n", 321 | "df['n_squared'] = df['n'].progress_apply(lambda x: np.cos(x**2))" 322 | ] 323 | }, 324 | { 325 | "cell_type": "markdown", 326 | "metadata": {}, 327 | "source": [ 328 | "⬇\n", 329 | "
\n", 330 | "⬇" 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": 9, 336 | "metadata": {}, 337 | "outputs": [ 338 | { 339 | "name": "stdout", 340 | "output_type": "stream", 341 | "text": [ 342 | "INFO: Pandarallel will run on 10 workers.\n", 343 | "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n" 344 | ] 345 | } 346 | ], 347 | "source": [ 348 | "from pandarallel import pandarallel\n", 349 | "\n", 350 | "pandarallel.initialize(progress_bar=True)" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 10, 356 | "metadata": {}, 357 | "outputs": [ 358 | { 359 | "data": { 360 | "application/vnd.jupyter.widget-view+json": { 361 | "model_id": "ba276e3a5752498096e5c406b092b377", 362 | "version_major": 2, 363 | "version_minor": 0 364 | }, 365 | "text/plain": [ 366 | "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=1000000), Label(value='0 / 1000000…" 367 | ] 368 | }, 369 | "metadata": {}, 370 | "output_type": "display_data" 371 | } 372 | ], 373 | "source": [ 374 | "df['n_squared'] = df['n'].parallel_apply(lambda x: np.cos(x**2))" 375 | ] 376 | }, 377 | { 378 | "cell_type": "markdown", 379 | "metadata": {}, 380 | "source": [ 381 | "---\n", 382 | "
\n", 383 | "
\n", 384 | "
\n", 385 | "
\n", 386 | "
\n", 387 | "
\n", 388 | "
\n", 389 | "
\n", 390 | "
\n", 391 | "
\n", 392 | "
\n", 393 | "
\n", 394 | "
\n", 395 | "
\n", 396 | "
\n", 397 | "
\n", 398 | "
\n", 399 | "
\n", 400 | "\n", 401 | "---" 402 | ] 403 | }, 404 | { 405 | "cell_type": "markdown", 406 | "metadata": {}, 407 | "source": [ 408 | "## joblib" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": 11, 414 | "metadata": {}, 415 | "outputs": [], 416 | "source": [ 417 | "def calculate(x):\n", 418 | " time.sleep(3)\n", 419 | " return x * 10" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": 12, 425 | "metadata": {}, 426 | "outputs": [ 427 | { 428 | "data": { 429 | "text/plain": [ 430 | "6660" 431 | ] 432 | }, 433 | "execution_count": 12, 434 | "metadata": {}, 435 | "output_type": "execute_result" 436 | } 437 | ], 438 | "source": [ 439 | "calculate(666) # slow 😴" 440 | ] 441 | }, 442 | { 443 | "cell_type": "markdown", 444 | "metadata": {}, 445 | "source": [ 446 | "⬇\n", 447 | "
\n", 448 | "⬇" 449 | ] 450 | }, 451 | { 452 | "cell_type": "code", 453 | "execution_count": 13, 454 | "metadata": {}, 455 | "outputs": [], 456 | "source": [ 457 | "from joblib import Memory\n", 458 | "\n", 459 | "memory = Memory(\"./cache\", verbose=0)" 460 | ] 461 | }, 462 | { 463 | "cell_type": "code", 464 | "execution_count": 14, 465 | "metadata": {}, 466 | "outputs": [], 467 | "source": [ 468 | "@memory.cache\n", 469 | "def calculate(x):\n", 470 | " time.sleep(3)\n", 471 | " return x * 10" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 15, 477 | "metadata": {}, 478 | "outputs": [ 479 | { 480 | "data": { 481 | "text/plain": [ 482 | "6660" 483 | ] 484 | }, 485 | "execution_count": 15, 486 | "metadata": {}, 487 | "output_type": "execute_result" 488 | } 489 | ], 490 | "source": [ 491 | "calculate(666) # slow 😴" 492 | ] 493 | }, 494 | { 495 | "cell_type": "code", 496 | "execution_count": 16, 497 | "metadata": {}, 498 | "outputs": [ 499 | { 500 | "data": { 501 | "text/plain": [ 502 | "6660" 503 | ] 504 | }, 505 | "execution_count": 16, 506 | "metadata": {}, 507 | "output_type": "execute_result" 508 | } 509 | ], 510 | "source": [ 511 | "calculate(666) # fast! 🏃‍♂️" 512 | ] 513 | }, 514 | { 515 | "cell_type": "markdown", 516 | "metadata": {}, 517 | "source": [ 518 | "⬇\n", 519 | "
\n", 520 | "⬇" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": 17, 526 | "metadata": {}, 527 | "outputs": [ 528 | { 529 | "name": "stdout", 530 | "output_type": "stream", 531 | "text": [ 532 | "CPU times: user 4.15 ms, sys: 4.71 ms, total: 8.86 ms\n", 533 | "Wall time: 3.01 s\n" 534 | ] 535 | }, 536 | { 537 | "data": { 538 | "text/plain": [ 539 | "420" 540 | ] 541 | }, 542 | "execution_count": 17, 543 | "metadata": {}, 544 | "output_type": "execute_result" 545 | } 546 | ], 547 | "source": [ 548 | "%%time\n", 549 | "calculate(42)" 550 | ] 551 | }, 552 | { 553 | "cell_type": "code", 554 | "execution_count": 18, 555 | "metadata": {}, 556 | "outputs": [ 557 | { 558 | "name": "stdout", 559 | "output_type": "stream", 560 | "text": [ 561 | "CPU times: user 996 µs, sys: 450 µs, total: 1.45 ms\n", 562 | "Wall time: 1.4 ms\n" 563 | ] 564 | }, 565 | { 566 | "data": { 567 | "text/plain": [ 568 | "420" 569 | ] 570 | }, 571 | "execution_count": 18, 572 | "metadata": {}, 573 | "output_type": "execute_result" 574 | } 575 | ], 576 | "source": [ 577 | "%%time\n", 578 | "calculate(42)" 579 | ] 580 | }, 581 | { 582 | "cell_type": "markdown", 583 | "metadata": {}, 584 | "source": [ 585 | "⬇\n", 586 | "
\n", 587 | "⬇" 588 | ] 589 | }, 590 | { 591 | "cell_type": "code", 592 | "execution_count": 19, 593 | "metadata": {}, 594 | "outputs": [ 595 | { 596 | "name": "stderr", 597 | "output_type": "stream", 598 | "text": [ 599 | "WARNING:root:[Memory(location=./cache/joblib)]: Flushing completely the cache\n" 600 | ] 601 | } 602 | ], 603 | "source": [ 604 | "# Clear the cache so I can give this talk again!\n", 605 | "memory.clear()" 606 | ] 607 | }, 608 | { 609 | "cell_type": "markdown", 610 | "metadata": {}, 611 | "source": [ 612 | "---\n", 613 | "
\n", 614 | "
\n", 615 | "
\n", 616 | "
\n", 617 | "
\n", 618 | "
\n", 619 | "
\n", 620 | "
\n", 621 | "
\n", 622 | "
\n", 623 | "
\n", 624 | "
\n", 625 | "
\n", 626 | "
\n", 627 | "
\n", 628 | "
\n", 629 | "
\n", 630 | "
\n", 631 | "\n", 632 | "---" 633 | ] 634 | }, 635 | { 636 | "cell_type": "markdown", 637 | "metadata": {}, 638 | "source": [ 639 | "## black" 640 | ] 641 | }, 642 | { 643 | "cell_type": "code", 644 | "execution_count": 20, 645 | "metadata": {}, 646 | "outputs": [ 647 | { 648 | "data": { 649 | "text/html": [ 650 | "\n", 651 | " \n", 664 | " " 665 | ], 666 | "text/plain": [ 667 | "" 668 | ] 669 | }, 670 | "metadata": {}, 671 | "output_type": "display_data" 672 | } 673 | ], 674 | "source": [ 675 | "# This is where the magic happens. ✨\n", 676 | "%load_ext jupyter_black" 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "execution_count": 21, 682 | "metadata": {}, 683 | "outputs": [], 684 | "source": [ 685 | "j = [1, 2, 3]" 686 | ] 687 | }, 688 | { 689 | "cell_type": "code", 690 | "execution_count": 22, 691 | "metadata": {}, 692 | "outputs": [], 693 | "source": [ 694 | "import os\n", 695 | "\n", 696 | "\n", 697 | "def very_important_function(\n", 698 | " template: str,\n", 699 | " *variables,\n", 700 | " file: os.PathLike,\n", 701 | " engine: str,\n", 702 | " header: bool = True,\n", 703 | " debug: bool = False,\n", 704 | "):\n", 705 | " \"\"\"Does something very important.\"\"\"\n", 706 | " pass" 707 | ] 708 | }, 709 | { 710 | "cell_type": "markdown", 711 | "metadata": {}, 712 | "source": [ 713 | "---\n", 714 | "
\n", 715 | "
\n", 716 | "
\n", 717 | "
\n", 718 | "
\n", 719 | "
\n", 720 | "
\n", 721 | "
\n", 722 | "
\n", 723 | "
\n", 724 | "
\n", 725 | "
\n", 726 | "
\n", 727 | "
\n", 728 | "
\n", 729 | "
\n", 730 | "
\n", 731 | "
\n", 732 | "\n", 733 | "---" 734 | ] 735 | }, 736 | { 737 | "cell_type": "markdown", 738 | "metadata": {}, 739 | "source": [ 740 | "## isort" 741 | ] 742 | }, 743 | { 744 | "cell_type": "markdown", 745 | "metadata": {}, 746 | "source": [ 747 | "![](https://camo.githubusercontent.com/841a7830ea6e8d363a53214917144b259685cc15/68747470733a2f2f7261772e6769746875622e636f6d2f74696d6f74687963726f736c65792f69736f72742f646576656c6f702f6578616d706c652e676966)" 748 | ] 749 | }, 750 | { 751 | "cell_type": "markdown", 752 | "metadata": {}, 753 | "source": [ 754 | "---" 755 | ] 756 | }, 757 | { 758 | "cell_type": "markdown", 759 | "metadata": {}, 760 | "source": [ 761 | "## pre-commit" 762 | ] 763 | }, 764 | { 765 | "cell_type": "markdown", 766 | "metadata": {}, 767 | "source": [ 768 | "## fzf" 769 | ] 770 | }, 771 | { 772 | "cell_type": "markdown", 773 | "metadata": {}, 774 | "source": [ 775 | "## coefficient-cookiecutter" 776 | ] 777 | }, 778 | { 779 | "cell_type": "markdown", 780 | "metadata": {}, 781 | "source": [ 782 | "---\n", 783 | "# Follow me at @john_sandall for links to all of the above" 784 | ] 785 | }, 786 | { 787 | "cell_type": "markdown", 788 | "metadata": {}, 789 | "source": [ 790 | "---\n", 791 | "# `https://github.com/john-sandall/python-productivity-powerups`" 792 | ] 793 | }, 794 | { 795 | "cell_type": "markdown", 796 | "metadata": {}, 797 | "source": [ 798 | "---\n", 799 | "\n", 800 | "
\n", 801 | " About\n", 802 | "

\n", 803 | " This notebook has been made by @John_Sandall. I run training workshops in Python, data science and data engineering.\n", 804 | "


\n", 805 | "

\n", 806 | " You can follow my free First Steps with Python and First Steps with pandas workshops for free as part of PyData Bristol's Zero To Hero 2020 monthly free workshop series. PyData Bristol will be running more free virtual workshops over the coming months so sign up via Meetup.com or follow us @PyDataBristol on Twitter.\n", 807 | "


\n", 808 | "

\n", 809 | " I am the Founder of data science consultancy Coefficient. If you would like to work with us, our team can help you with your data science, software engineering and machine learning projects as an on-demand resource. We can also create bespoke training workshops adapted to your industry, virtual or in-person, with training clients currently including BNP Paribas, EY, the Met Police and the BBC.\n", 810 | "

\n", 811 | "
" 812 | ] 813 | } 814 | ], 815 | "metadata": { 816 | "kernelspec": { 817 | "display_name": "Python (ppp)", 818 | "language": "python", 819 | "name": "ppp" 820 | }, 821 | "language_info": { 822 | "codemirror_mode": { 823 | "name": "ipython", 824 | "version": 3 825 | }, 826 | "file_extension": ".py", 827 | "mimetype": "text/x-python", 828 | "name": "python", 829 | "nbconvert_exporter": "python", 830 | "pygments_lexer": "ipython3", 831 | "version": "3.10.6" 832 | }, 833 | "toc": { 834 | "base_numbering": 1, 835 | "nav_menu": {}, 836 | "number_sections": true, 837 | "sideBar": true, 838 | "skip_h1_title": true, 839 | "title_cell": "Table of Contents", 840 | "title_sidebar": "Contents", 841 | "toc_cell": true, 842 | "toc_position": {}, 843 | "toc_section_display": true, 844 | "toc_window_display": true 845 | }, 846 | "vscode": { 847 | "interpreter": { 848 | "hash": "be34d95dbc58180ac3633c8e41e8772c710783434d94474c3fd4437fff7f4790" 849 | } 850 | } 851 | }, 852 | "nbformat": 4, 853 | "nbformat_minor": 4 854 | } 855 | -------------------------------------------------------------------------------- /notebooks/2023-02-23 - Python Productivity Power-Ups (PyCon NA).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[![pycon-na](../images/pycon-na-header.png)](https://na.pycon.org/)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# 🐍 Python Productivity Power-Ups 🚀\n", 15 | "## 👤 John Sandall" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "---\n", 23 | "
\n", 24 | "
\n", 25 | "
\n", 26 | "
\n", 27 | "
\n", 28 | "
\n", 29 | "
\n", 30 | "
\n", 31 | "
\n", 32 | "
\n", 33 | "
\n", 34 | "
\n", 35 | "
\n", 36 | "
\n", 37 | "
\n", 38 | "
\n", 39 | "
\n", 40 | "
\n", 41 | "\n", 42 | "---" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "# Python Productivity Power-Ups\n", 50 | "1. tqdm: https://pypi.org/project/tqdm/\n", 51 | "2. joblib: https://joblib.readthedocs.io/en/latest/\n", 52 | "3. black: https://pypi.org/project/black/\n", 53 | "4. jupyter-black: https://pypi.org/project/jupyter-black/\n", 54 | "5. isort: https://github.com/timothycrosley/isort\n", 55 | "6. fzf: https://github.com/junegunn/fzf\n", 56 | "7. pre-commit: https://pre-commit.com/\n", 57 | "8. coefficient-cookiecutter: https://github.com/CoefficientSystems/coefficient-cookiecutter" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "## black" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 1, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "data": { 74 | "text/html": [ 75 | "\n", 76 | " \n", 89 | " " 90 | ], 91 | "text/plain": [ 92 | "" 93 | ] 94 | }, 95 | "metadata": {}, 96 | "output_type": "display_data" 97 | } 98 | ], 99 | "source": [ 100 | "# This is where the magic happens. ✨\n", 101 | "%load_ext jupyter_black" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 2, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "j = [\n", 111 | " 1, \n", 112 | " 2, \n", 113 | " 3\n", 114 | "]" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 3, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "import os\n", 124 | "\n", 125 | "\n", 126 | "def very_important_function(template: str,*variables,file: os.PathLike,debug:bool=False,):\n", 127 | " \"\"\"Does something very important.\"\"\"\n", 128 | " pass" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "---\n", 136 | "
\n", 137 | "
\n", 138 | "
\n", 139 | "
\n", 140 | "
\n", 141 | "
\n", 142 | "
\n", 143 | "
\n", 144 | "
\n", 145 | "
\n", 146 | "
\n", 147 | "
\n", 148 | "
\n", 149 | "
\n", 150 | "
\n", 151 | "
\n", 152 | "
\n", 153 | "
\n", 154 | "\n", 155 | "---" 156 | ] 157 | }, 158 | { 159 | "cell_type": "markdown", 160 | "metadata": {}, 161 | "source": [ 162 | "## direnv" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": 6, 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [ 171 | "import os" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 7, 177 | "metadata": { 178 | "tags": [] 179 | }, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/plain": [ 184 | "'value'" 185 | ] 186 | }, 187 | "execution_count": 7, 188 | "metadata": {}, 189 | "output_type": "execute_result" 190 | } 191 | ], 192 | "source": [ 193 | "os.environ[\"SECRET\"]" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "---\n", 201 | "
\n", 202 | "
\n", 203 | "
\n", 204 | "
\n", 205 | "
\n", 206 | "
\n", 207 | "
\n", 208 | "
\n", 209 | "
\n", 210 | "
\n", 211 | "
\n", 212 | "
\n", 213 | "
\n", 214 | "
\n", 215 | "
\n", 216 | "
\n", 217 | "
\n", 218 | "
\n", 219 | "\n", 220 | "---" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": null, 254 | "metadata": {}, 255 | "outputs": [], 256 | "source": [] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 1, 275 | "metadata": {}, 276 | "outputs": [], 277 | "source": [ 278 | "import time\n", 279 | "\n", 280 | "import numpy as np\n", 281 | "import pandas as pd" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": {}, 287 | "source": [ 288 | "## tqdm" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 2, 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "from tqdm import tqdm" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 3, 303 | "metadata": {}, 304 | "outputs": [ 305 | { 306 | "ename": "KeyboardInterrupt", 307 | "evalue": "", 308 | "output_type": "error", 309 | "traceback": [ 310 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 311 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 312 | "\u001b[0;32m/var/folders/30/2pzsy6kx1x5dx_zn6n8h0w8r0000gn/T/ipykernel_50708/1820794216.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.01\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 313 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 314 | ] 315 | } 316 | ], 317 | "source": [ 318 | "for i in range(500):\n", 319 | " time.sleep(0.01)" 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "⬇\n", 327 | "
\n", 328 | "⬇" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": 4, 334 | "metadata": {}, 335 | "outputs": [ 336 | { 337 | "name": "stderr", 338 | "output_type": "stream", 339 | "text": [ 340 | "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [00:06<00:00, 81.07it/s]\n" 341 | ] 342 | } 343 | ], 344 | "source": [ 345 | "for i in tqdm(range(500)):\n", 346 | " time.sleep(0.01)" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "---\n", 354 | "
\n", 355 | "
\n", 356 | "
\n", 357 | "
\n", 358 | "
\n", 359 | "
\n", 360 | "
\n", 361 | "
\n", 362 | "
\n", 363 | "
\n", 364 | "
\n", 365 | "
\n", 366 | "
\n", 367 | "
\n", 368 | "
\n", 369 | "
\n", 370 | "
\n", 371 | "
\n", 372 | "\n", 373 | "---" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": {}, 379 | "source": [ 380 | "### tqdm and pandas" 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 5, 386 | "metadata": {}, 387 | "outputs": [ 388 | { 389 | "data": { 390 | "text/html": [ 391 | "
\n", 392 | "\n", 405 | "\n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | "
n
061
123
238
340
471
......
999999567
999999661
999999783
999999882
999999951
\n", 459 | "

10000000 rows × 1 columns

\n", 460 | "
" 461 | ], 462 | "text/plain": [ 463 | " n\n", 464 | "0 61\n", 465 | "1 23\n", 466 | "2 38\n", 467 | "3 40\n", 468 | "4 71\n", 469 | "... ..\n", 470 | "9999995 67\n", 471 | "9999996 61\n", 472 | "9999997 83\n", 473 | "9999998 82\n", 474 | "9999999 51\n", 475 | "\n", 476 | "[10000000 rows x 1 columns]" 477 | ] 478 | }, 479 | "execution_count": 5, 480 | "metadata": {}, 481 | "output_type": "execute_result" 482 | } 483 | ], 484 | "source": [ 485 | "total_rows = 1_000_0000\n", 486 | "df = pd.DataFrame({'n': np.random.randint(1, 100, total_rows)})\n", 487 | "df" 488 | ] 489 | }, 490 | { 491 | "cell_type": "code", 492 | "execution_count": null, 493 | "metadata": {}, 494 | "outputs": [], 495 | "source": [ 496 | "# Then just use .progress_apply() wherever you'd usually use .apply()\n", 497 | "df['n_squared'] = df['n'].apply(lambda x: np.cos(x**2))" 498 | ] 499 | }, 500 | { 501 | "cell_type": "markdown", 502 | "metadata": {}, 503 | "source": [ 504 | "⬇\n", 505 | "
\n", 506 | "⬇" 507 | ] 508 | }, 509 | { 510 | "cell_type": "code", 511 | "execution_count": 7, 512 | "metadata": {}, 513 | "outputs": [], 514 | "source": [ 515 | "# Run this once in your script/notebook\n", 516 | "tqdm.pandas()" 517 | ] 518 | }, 519 | { 520 | "cell_type": "code", 521 | "execution_count": null, 522 | "metadata": {}, 523 | "outputs": [], 524 | "source": [ 525 | "# Then just use .progress_apply() wherever you'd usually use .apply()\n", 526 | "df['n_squared'] = df['n'].progress_apply(lambda x: np.cos(x**2))" 527 | ] 528 | }, 529 | { 530 | "cell_type": "markdown", 531 | "metadata": {}, 532 | "source": [ 533 | "⬇\n", 534 | "
\n", 535 | "⬇" 536 | ] 537 | }, 538 | { 539 | "cell_type": "code", 540 | "execution_count": 9, 541 | "metadata": {}, 542 | "outputs": [ 543 | { 544 | "name": "stdout", 545 | "output_type": "stream", 546 | "text": [ 547 | "INFO: Pandarallel will run on 10 workers.\n", 548 | "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n" 549 | ] 550 | } 551 | ], 552 | "source": [ 553 | "from pandarallel import pandarallel\n", 554 | "\n", 555 | "pandarallel.initialize(progress_bar=True)" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 10, 561 | "metadata": {}, 562 | "outputs": [ 563 | { 564 | "data": { 565 | "application/vnd.jupyter.widget-view+json": { 566 | "model_id": "ba276e3a5752498096e5c406b092b377", 567 | "version_major": 2, 568 | "version_minor": 0 569 | }, 570 | "text/plain": [ 571 | "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=1000000), Label(value='0 / 1000000…" 572 | ] 573 | }, 574 | "metadata": {}, 575 | "output_type": "display_data" 576 | } 577 | ], 578 | "source": [ 579 | "df['n_squared'] = df['n'].parallel_apply(lambda x: np.cos(x**2))" 580 | ] 581 | }, 582 | { 583 | "cell_type": "markdown", 584 | "metadata": {}, 585 | "source": [ 586 | "---\n", 587 | "
\n", 588 | "
\n", 589 | "
\n", 590 | "
\n", 591 | "
\n", 592 | "
\n", 593 | "
\n", 594 | "
\n", 595 | "
\n", 596 | "
\n", 597 | "
\n", 598 | "
\n", 599 | "
\n", 600 | "
\n", 601 | "
\n", 602 | "
\n", 603 | "
\n", 604 | "
\n", 605 | "\n", 606 | "---" 607 | ] 608 | }, 609 | { 610 | "cell_type": "markdown", 611 | "metadata": {}, 612 | "source": [ 613 | "## joblib" 614 | ] 615 | }, 616 | { 617 | "cell_type": "code", 618 | "execution_count": 11, 619 | "metadata": {}, 620 | "outputs": [], 621 | "source": [ 622 | "def calculate(x):\n", 623 | " time.sleep(3)\n", 624 | " return x * 10" 625 | ] 626 | }, 627 | { 628 | "cell_type": "code", 629 | "execution_count": 12, 630 | "metadata": {}, 631 | "outputs": [ 632 | { 633 | "data": { 634 | "text/plain": [ 635 | "6660" 636 | ] 637 | }, 638 | "execution_count": 12, 639 | "metadata": {}, 640 | "output_type": "execute_result" 641 | } 642 | ], 643 | "source": [ 644 | "calculate(666) # slow 😴" 645 | ] 646 | }, 647 | { 648 | "cell_type": "markdown", 649 | "metadata": {}, 650 | "source": [ 651 | "⬇\n", 652 | "
\n", 653 | "⬇" 654 | ] 655 | }, 656 | { 657 | "cell_type": "code", 658 | "execution_count": 13, 659 | "metadata": {}, 660 | "outputs": [], 661 | "source": [ 662 | "from joblib import Memory\n", 663 | "\n", 664 | "memory = Memory(\"./cache\", verbose=0)" 665 | ] 666 | }, 667 | { 668 | "cell_type": "code", 669 | "execution_count": 14, 670 | "metadata": {}, 671 | "outputs": [], 672 | "source": [ 673 | "@memory.cache\n", 674 | "def calculate(x):\n", 675 | " time.sleep(3)\n", 676 | " return x * 10" 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "execution_count": 15, 682 | "metadata": {}, 683 | "outputs": [ 684 | { 685 | "data": { 686 | "text/plain": [ 687 | "6660" 688 | ] 689 | }, 690 | "execution_count": 15, 691 | "metadata": {}, 692 | "output_type": "execute_result" 693 | } 694 | ], 695 | "source": [ 696 | "calculate(666) # slow 😴" 697 | ] 698 | }, 699 | { 700 | "cell_type": "code", 701 | "execution_count": 16, 702 | "metadata": {}, 703 | "outputs": [ 704 | { 705 | "data": { 706 | "text/plain": [ 707 | "6660" 708 | ] 709 | }, 710 | "execution_count": 16, 711 | "metadata": {}, 712 | "output_type": "execute_result" 713 | } 714 | ], 715 | "source": [ 716 | "calculate(666) # fast! 🏃‍♂️" 717 | ] 718 | }, 719 | { 720 | "cell_type": "markdown", 721 | "metadata": {}, 722 | "source": [ 723 | "⬇\n", 724 | "
\n", 725 | "⬇" 726 | ] 727 | }, 728 | { 729 | "cell_type": "code", 730 | "execution_count": 17, 731 | "metadata": {}, 732 | "outputs": [ 733 | { 734 | "name": "stdout", 735 | "output_type": "stream", 736 | "text": [ 737 | "CPU times: user 4.15 ms, sys: 4.71 ms, total: 8.86 ms\n", 738 | "Wall time: 3.01 s\n" 739 | ] 740 | }, 741 | { 742 | "data": { 743 | "text/plain": [ 744 | "420" 745 | ] 746 | }, 747 | "execution_count": 17, 748 | "metadata": {}, 749 | "output_type": "execute_result" 750 | } 751 | ], 752 | "source": [ 753 | "%%time\n", 754 | "calculate(42)" 755 | ] 756 | }, 757 | { 758 | "cell_type": "code", 759 | "execution_count": 18, 760 | "metadata": {}, 761 | "outputs": [ 762 | { 763 | "name": "stdout", 764 | "output_type": "stream", 765 | "text": [ 766 | "CPU times: user 996 µs, sys: 450 µs, total: 1.45 ms\n", 767 | "Wall time: 1.4 ms\n" 768 | ] 769 | }, 770 | { 771 | "data": { 772 | "text/plain": [ 773 | "420" 774 | ] 775 | }, 776 | "execution_count": 18, 777 | "metadata": {}, 778 | "output_type": "execute_result" 779 | } 780 | ], 781 | "source": [ 782 | "%%time\n", 783 | "calculate(42)" 784 | ] 785 | }, 786 | { 787 | "cell_type": "markdown", 788 | "metadata": {}, 789 | "source": [ 790 | "⬇\n", 791 | "
\n", 792 | "⬇" 793 | ] 794 | }, 795 | { 796 | "cell_type": "code", 797 | "execution_count": 19, 798 | "metadata": {}, 799 | "outputs": [ 800 | { 801 | "name": "stderr", 802 | "output_type": "stream", 803 | "text": [ 804 | "WARNING:root:[Memory(location=./cache/joblib)]: Flushing completely the cache\n" 805 | ] 806 | } 807 | ], 808 | "source": [ 809 | "# Clear the cache so I can give this talk again!\n", 810 | "memory.clear()" 811 | ] 812 | }, 813 | { 814 | "cell_type": "markdown", 815 | "metadata": {}, 816 | "source": [ 817 | "---" 818 | ] 819 | }, 820 | { 821 | "cell_type": "markdown", 822 | "metadata": {}, 823 | "source": [ 824 | "## pre-commit" 825 | ] 826 | }, 827 | { 828 | "cell_type": "markdown", 829 | "metadata": {}, 830 | "source": [ 831 | "## fzf" 832 | ] 833 | }, 834 | { 835 | "cell_type": "markdown", 836 | "metadata": {}, 837 | "source": [ 838 | "## coefficient-cookiecutter" 839 | ] 840 | }, 841 | { 842 | "cell_type": "markdown", 843 | "metadata": {}, 844 | "source": [ 845 | "---\n", 846 | "# Follow me at @john_sandall for links to all of the above" 847 | ] 848 | }, 849 | { 850 | "cell_type": "markdown", 851 | "metadata": {}, 852 | "source": [ 853 | "---\n", 854 | "# `https://github.com/john-sandall/python-productivity-powerups`" 855 | ] 856 | }, 857 | { 858 | "cell_type": "markdown", 859 | "metadata": {}, 860 | "source": [ 861 | "[![about](../images/about-me.jpeg)](https://coefficient.ai)" 862 | ] 863 | }, 864 | { 865 | "cell_type": "markdown", 866 | "metadata": {}, 867 | "source": [ 868 | "---\n", 869 | "\n", 870 | "
\n", 871 | " About\n", 872 | "

\n", 873 | " This notebook has been made by @John_Sandall. I run training workshops in Python, data science and data engineering.\n", 874 | "


\n", 875 | "

\n", 876 | " You can follow my free First Steps with Python and First Steps with pandas workshops for free as part of PyData Bristol's Zero To Hero 2020 monthly free workshop series. PyData Bristol will be running more free virtual workshops over the coming months so sign up via Meetup.com or follow us @PyDataBristol on Twitter.\n", 877 | "


\n", 878 | "

\n", 879 | " I am the Founder of data science consultancy Coefficient. If you would like to work with us, our team can help you with your data science, software engineering and machine learning projects as an on-demand resource. We can also create bespoke training workshops adapted to your industry, virtual or in-person, with training clients currently including BNP Paribas, EY, the Met Police and the BBC.\n", 880 | "

\n", 881 | "
" 882 | ] 883 | } 884 | ], 885 | "metadata": { 886 | "kernelspec": { 887 | "display_name": "Python (ppp)", 888 | "language": "python", 889 | "name": "ppp" 890 | }, 891 | "language_info": { 892 | "codemirror_mode": { 893 | "name": "ipython", 894 | "version": 3 895 | }, 896 | "file_extension": ".py", 897 | "mimetype": "text/x-python", 898 | "name": "python", 899 | "nbconvert_exporter": "python", 900 | "pygments_lexer": "ipython3", 901 | "version": "3.10.6" 902 | }, 903 | "toc": { 904 | "base_numbering": 1, 905 | "nav_menu": {}, 906 | "number_sections": true, 907 | "sideBar": true, 908 | "skip_h1_title": true, 909 | "title_cell": "Table of Contents", 910 | "title_sidebar": "Contents", 911 | "toc_cell": true, 912 | "toc_position": {}, 913 | "toc_section_display": true, 914 | "toc_window_display": true 915 | }, 916 | "vscode": { 917 | "interpreter": { 918 | "hash": "be34d95dbc58180ac3633c8e41e8772c710783434d94474c3fd4437fff7f4790" 919 | } 920 | } 921 | }, 922 | "nbformat": 4, 923 | "nbformat_minor": 4 924 | } 925 | --------------------------------------------------------------------------------