├── .github └── workflows │ ├── docs.yml │ ├── python-lint.yml │ ├── python-package.yml │ └── python-publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CITATION.cff ├── LICENSE.txt ├── README.md ├── cbsodata ├── __init__.py ├── __main__.py └── cbsodata3.py ├── docs ├── Makefile ├── conf.py ├── index.rst ├── make.bat ├── readme_link.rst ├── reference.rst └── requirements.txt ├── pyproject.toml └── tests ├── __init__.py └── test_cbsodata.py /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: "Docs Check" 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | docs: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v1 9 | - uses: ammaraskar/sphinx-action@master 10 | with: 11 | docs-folder: "docs/" 12 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 13 | -------------------------------------------------------------------------------- /.github/workflows/python-lint.yml: -------------------------------------------------------------------------------- 1 | name: Linter 2 | on: [ push, pull_request ] 3 | jobs: 4 | lint-ruff: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - uses: chartboost/ruff-action@v1 9 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | name: Python package and test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | 7 | runs-on: ubuntu-latest 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | python-version: ["3.8", "3.12"] 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | - name: Install package and dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | python -m pip install .[test] 23 | python -m pip install pytest-xdist 24 | - name: Test with pytest 25 | run: | 26 | pytest -n 4 27 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Set up Python 18 | uses: actions/setup-python@v4 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install build 25 | - name: Build package 26 | run: python -m build 27 | - name: Publish package 28 | uses: pypa/gh-action-pypi-publish@release/v1 29 | with: 30 | user: __token__ 31 | password: ${{ secrets.pypi_password }} 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | test_env/ 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # dotenv 82 | .env 83 | 84 | # virtualenv 85 | .venv/ 86 | venv/ 87 | ENV/ 88 | 89 | # Spyder project settings 90 | .spyderproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autoupdate_schedule: 'monthly' 3 | 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v5.0.0 7 | hooks: 8 | - id: trailing-whitespace 9 | - id: end-of-file-fixer 10 | exclude: "^.*drawio|.*.svg$" 11 | - id: check-yaml 12 | - id: check-added-large-files 13 | - repo: https://github.com/astral-sh/ruff-pre-commit 14 | rev: v0.8.3 15 | hooks: 16 | - id: ruff 17 | - repo: https://github.com/psf/black-pre-commit-mirror 18 | rev: 24.10.0 19 | hooks: 20 | - id: black 21 | - repo: https://github.com/asottile/pyupgrade 22 | rev: v3.19.0 23 | hooks: 24 | - id: pyupgrade 25 | args: [--py38-plus] 26 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | title: >- 3 | cbsodata - Statistics Netherlands opendata API client for 4 | Python 5 | message: 'We appreciate, but do not require, attribution.' 6 | type: software 7 | authors: 8 | - family-names: De Bruin 9 | given-names: Jonathan 10 | orcid: 'https://orcid.org/0000-0002-4297-0502' 11 | repository-code: 'https://github.com/J535D165/cbsodata' 12 | url: 'https://github.com/J535D165/cbsodata' 13 | repository-artifact: 'https://pypi.org/project/cbsodata/' 14 | keywords: 15 | - census-data 16 | - national-statistics 17 | - census 18 | - dataset 19 | - open-data 20 | - netherlands 21 | license: MIT 22 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Jonathan de Bruin 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Statistics Netherlands opendata API client for Python 2 | 3 | [![pypi](https://badge.fury.io/py/cbsodata.svg)](https://badge.fury.io/py/cbsodata) 4 | [![tests](https://github.com/J535D165/cbsodata/workflows/tests/badge.svg)](https://github.com/J535D165/cbsodata/actions) 5 | 6 | Retrieve data from the [open data interface of Statistics 7 | Netherlands](http://www.cbs.nl/nl-NL/menu/cijfers/statline/open-data/default.htm) 8 | (Centraal Bureau voor de Statistiek) with *Python*. The data is 9 | identical in content to the tables which can be retrieved and downloaded 10 | from [StatLine](http://statline.cbs.nl/). CBS datasets are accessed via 11 | the [CBS open data 12 | portal](https://opendata.cbs.nl/statline/portal.html). 13 | 14 | The documentation of this package is found at this page and on 15 | [readthedocs.io](http://cbsodata.readthedocs.io/). 16 | 17 | R user? Use 18 | [cbsodataR](https://cran.r-project.org/web/packages/cbsodataR/index.html). 19 | 20 | ## Installation 21 | 22 | From PyPi 23 | 24 | ``` sh 25 | pip install cbsodata 26 | ``` 27 | 28 | ## Usage 29 | 30 | Load the package with 31 | 32 | ``` python 33 | >>> import cbsodata 34 | ``` 35 | 36 | ### Tables 37 | 38 | Statistics Netherlands (CBS) has a large amount of public available data 39 | tables (more than 4000 at the moment of writing). Each table is 40 | identified by a unique identifier (`Identifier`). 41 | 42 | ``` python 43 | >>> tables = cbsodata.get_table_list() 44 | >>> print(tables[0]) 45 | {'Catalog': 'CBS', 46 | 'ColumnCount': 18, 47 | 'DefaultPresentation': '_la=nl&_si=&_gu=&_ed=LandVanUiteindelijkeZeggenschapUCI&_td=Perioden&graphType=line', 48 | 'DefaultSelection': "$filter=((LandVanUiteindelijkeZeggenschapUCI eq '11111') or (LandVanUiteindelijkeZeggenschapUCI eq '22222')) and (Bedrijfsgrootte eq '10000') and (substringof('JJ',Perioden))&$select=LandVanUiteindelijkeZeggenschapUCI, Bedrijfsgrootte, Perioden, FiscaalJaarloonPerBaan_15", 49 | 'ExplanatoryText': '', 50 | 'Frequency': 'Perjaar', 51 | 'GraphTypes': 'Table,Bar,Line', 52 | 'ID': 0, 53 | 'Identifier': '82010NED', 54 | 'Language': 'nl', 55 | 'MetaDataModified': '2014-02-04T02:00:00', 56 | 'Modified': '2014-02-04T02:00:00', 57 | 'OutputStatus': 'Regulier', 58 | 'Period': '2008 t/m 2011', 59 | 'ReasonDelivery': 'Actualisering', 60 | 'RecordCount': 32, 61 | 'SearchPriority': '2', 62 | 'ShortDescription': '\nDeze tabel bevat informatie over banen en lonen bij bedrijven in Nederland, uitgesplitst naar het land van uiteindelijke zeggenschap van die bedrijven. Hierbij wordt onderscheid gemaakt tussen bedrijven onder Nederlandse zeggenschap en bedrijven onder buitenlandse zeggenschap. In de tabel zijn alleen de bedrijven met werknemers in loondienst meegenomen. De cijfers hebben betrekking op het totale aantal banen bij deze bedrijven en de samenstelling van die banen naar kenmerken van de werknemers (baanstatus, geslacht, leeftijd, herkomst en hoogte van het loon). Ook het gemiddelde fiscale jaarloon per baan is in de tabel te vinden. \n\nGegevens beschikbaar vanaf: 2008 \n\nStatus van de cijfers: \nDe cijfers in deze tabel zijn definitief.\n\nWijzigingen per 4 februari 2014\nDe cijfers van 2011 zijn toegevoegd.\n\nWanneer komen er nieuwe cijfers?\nDe cijfers over 2012 verschijnen in de eerste helft van 2015.\n', 63 | 'ShortTitle': 'Zeggenschap bedrijven; banen, grootte', 64 | 'Source': 'CBS.', 65 | 'Summary': 'Banen en lonen van werknemers bij bedrijven in Nederland\nnaar land van uiteindelijke zeggenschap en bedrijfsgrootte', 66 | 'SummaryAndLinks': 'Banen en lonen van werknemers bij bedrijven in Nederland
naar land van uiteindelijke zeggenschap en bedrijfsgrootte
http://opendata.cbs.nl/ODataApi/OData/82010NED
http://opendata.cbs.nl/ODataFeed/OData/82010NED', 67 | 'Title': 'Zeggenschap bedrijven in Nederland; banen en lonen, bedrijfsgrootte', 68 | 'Updated': '2014-02-04T02:00:00'} 69 | ``` 70 | 71 | ### Info 72 | 73 | Get information about a table with the `get_info` function. 74 | 75 | ``` python 76 | >>> info = cbsodata.get_info('82070ENG') # Returns a dict with info 77 | >>> info['Title'] 78 | 'Caribbean Netherlands; employed labour force characteristics 2012' 79 | >>> info['Modified'] 80 | '2013-11-28T15:00:00' 81 | ``` 82 | 83 | ### Data 84 | 85 | The function you are looking for!! The function `get_data` returns a 86 | list of dicts with the table data. 87 | 88 | ``` python 89 | >>> data = cbsodata.get_data('82070ENG') 90 | [{'CaribbeanNetherlands': 'Bonaire', 91 | 'EmployedLabourForceInternatDef_1': 8837, 92 | 'EmployedLabourForceNationalDef_2': 8559, 93 | 'Gender': 'Total male and female', 94 | 'ID': 0, 95 | 'Periods': '2012', 96 | 'PersonalCharacteristics': 'Total personal characteristics'}, 97 | {'CaribbeanNetherlands': 'St. Eustatius', 98 | 'EmployedLabourForceInternatDef_1': 2099, 99 | 'EmployedLabourForceNationalDef_2': 1940, 100 | 'Gender': 'Total male and female', 101 | 'ID': 1, 102 | 'Periods': '2012', 103 | 'PersonalCharacteristics': 'Total personal characteristics'}, 104 | {'CaribbeanNetherlands': 'Saba', 105 | 'EmployedLabourForceInternatDef_1': 1045, 106 | 'EmployedLabourForceNationalDef_2': 971, 107 | 'Gender': 'Total male and female', 108 | 'ID': 2, 109 | 'Periods': '2012', 110 | 'PersonalCharacteristics': 'Total personal characteristics'}, 111 | # ... 112 | ] 113 | ``` 114 | 115 | The keyword argument `dir` can be used to download the data directly to 116 | your file system. 117 | 118 | ``` python 119 | >>> data = cbsodata.get_data('82070ENG', dir="dir_to_save_data") 120 | ``` 121 | 122 | ### Catalogs (dataderden) 123 | 124 | There are multiple ways to retrieve data from catalogs other than 125 | 'opendata.cbs.nl'. The code below shows 3 different ways to retrieve 126 | data from the catalog 'dataderden.cbs.nl' (known from Iv3). 127 | 128 | On module level. 129 | 130 | ``` python 131 | cbsodata.options.catalog_url = 'dataderden.cbs.nl' 132 | # list tables 133 | cbsodata.get_table_list() 134 | # get dataset 47003NED 135 | cbsodata.get_data('47003NED') 136 | ``` 137 | 138 | With context managers. 139 | 140 | ``` python 141 | with cbsodata.catalog('dataderden.cbs.nl'): 142 | # list tables 143 | cbsodata.get_table_list() 144 | # get dataset 47003NED 145 | cbsodata.get_data('47003NED') 146 | ``` 147 | 148 | As a function argument. 149 | 150 | ``` python 151 | # list tables 152 | cbsodata.get_table_list(catalog_url='dataderden.cbs.nl') 153 | # get dataset 47003NED 154 | cbsodata.get_data('47003NED', catalog_url='dataderden.cbs.nl') 155 | ``` 156 | 157 | ### Pandas users 158 | 159 | The package works well with Pandas. Convert the result easily into a 160 | pandas DataFrame with the code below. 161 | 162 | ``` python 163 | >>> data = pandas.DataFrame(cbsodata.get_data('82070ENG')) 164 | >>> data.head() 165 | ``` 166 | 167 | The list of tables can be turned into a pandas DataFrame as well. 168 | 169 | ``` python 170 | >>> tables = pandas.DataFrame(cbsodata.get_table_list()) 171 | >>> tables.head() 172 | ``` 173 | 174 | ## Command Line Interface 175 | 176 | This library ships with a Command Line Interface (CLI). 177 | 178 | ``` bash 179 | > cbsodata -h 180 | usage: cbsodata [-h] [--version] [subcommand] 181 | 182 | CBS Open Data: Command Line Interface 183 | 184 | positional arguments: 185 | subcommand the subcommand (one of 'data', 'info', 'list') 186 | 187 | optional arguments: 188 | -h, --help show this help message and exit 189 | --version show the package version 190 | ``` 191 | 192 | Download data: 193 | 194 | ``` bash 195 | > cbsodata data 82010NED 196 | ``` 197 | 198 | Retrieve table information: 199 | 200 | ``` bash 201 | > cbsodata info 82010NED 202 | ``` 203 | 204 | Retrieve a list with all tables: 205 | 206 | ``` bash 207 | > cbsodata list 208 | ``` 209 | 210 | ### Export data 211 | 212 | Use the flag `-o` to load data to a file (JSON lines). 213 | 214 | ``` bash 215 | > cbsodata data 82010NED -o table_82010NED.jl 216 | ``` 217 | -------------------------------------------------------------------------------- /cbsodata/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Jonathan de Bruin 2 | 3 | # Permission is hereby granted, free of charge, to any person 4 | # obtaining a copy of this software and associated documentation 5 | # files (the "Software"), to deal in the Software without 6 | # restriction, including without limitation the rights to use, 7 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the 9 | # Software is furnished to do so, subject to the following 10 | # conditions: 11 | 12 | # The above copyright notice and this permission notice shall be 13 | # included in all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | # OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | """Statistics Netherlands opendata API client for Python""" 25 | 26 | try: 27 | from cbsodata._version import __version__ 28 | from cbsodata._version import __version_tuple__ 29 | except ImportError: 30 | __version__ = "0.0.0" 31 | __version_tuple__ = (0, 0, 0) 32 | 33 | from cbsodata.cbsodata3 import catalog 34 | from cbsodata.cbsodata3 import download_data 35 | from cbsodata.cbsodata3 import get_data 36 | from cbsodata.cbsodata3 import get_info 37 | from cbsodata.cbsodata3 import get_meta 38 | from cbsodata.cbsodata3 import get_table_list 39 | from cbsodata.cbsodata3 import options 40 | 41 | __all__ = [ 42 | "download_data", 43 | "get_data", 44 | "get_info", 45 | "get_meta", 46 | "get_table_list", 47 | "options", 48 | "catalog", 49 | ] 50 | -------------------------------------------------------------------------------- /cbsodata/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Jonathan de Bruin 2 | 3 | # Permission is hereby granted, free of charge, to any person 4 | # obtaining a copy of this software and associated documentation 5 | # files (the "Software"), to deal in the Software without 6 | # restriction, including without limitation the rights to use, 7 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the 9 | # Software is furnished to do so, subject to the following 10 | # conditions: 11 | 12 | # The above copyright notice and this permission notice shall be 13 | # included in all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | # OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | """Statistics Netherlands opendata CLI client""" 25 | 26 | import argparse 27 | import json 28 | import sys 29 | 30 | import cbsodata 31 | 32 | AVAILABLE_CMDS = ["data", "info", "list"] 33 | 34 | 35 | def json_outputter(data_obj, max_rows=None): 36 | """Print data in JSON format.""" 37 | 38 | # cut of the at max_rows 39 | if isinstance(max_rows, (int, float)): 40 | data_obj = data_obj[0:max_rows] 41 | 42 | for line in data_obj: 43 | print(json.dumps(line)) 44 | 45 | 46 | def text_outputter(data_obj, max_rows=None): 47 | """Print data in text format.""" 48 | 49 | # cut of the at max_rows 50 | if isinstance(max_rows, (int, float)): 51 | data_obj = data_obj[0:max_rows] 52 | 53 | # collect the maximum length in each column 54 | value_max_len = {} 55 | for d in data_obj: 56 | for key, value in d.items(): 57 | try: 58 | if len(str(value)) > value_max_len[key]: 59 | value_max_len[key] = len(str(value)) 60 | except KeyError: 61 | value_max_len[key] = len(str(value)) 62 | 63 | # get a list of columns 64 | columns = value_max_len.keys() 65 | 66 | # check if the column name is larger than the largest value 67 | for col in columns: 68 | if len(str(col)) > value_max_len[col]: 69 | value_max_len[col] = len(str(col)) 70 | 71 | # print the column line 72 | col_line = "" 73 | for i, col in enumerate(columns): 74 | if i != (len(columns) - 1): 75 | col_line = col_line + str(col).upper().ljust(value_max_len[col] + 2) 76 | else: 77 | col_line = col_line + str(col).upper() 78 | print(col_line) 79 | 80 | # print the data 81 | for d in data_obj: 82 | line = "" 83 | for i, col in enumerate(columns): 84 | if i != (len(columns) - 1): 85 | try: 86 | line = line + str(d[col]).ljust(value_max_len[col] + 2) 87 | except KeyError: 88 | line = " " * (value_max_len[col] + 2) 89 | else: 90 | try: 91 | line = line + str(d[col]) 92 | except KeyError: 93 | pass 94 | print(line) 95 | 96 | 97 | def parse_argument_table_id(parser): 98 | parser.add_argument("table_id", type=str, help="table identifier") 99 | 100 | 101 | def parse_argument_catalog(parser): 102 | parser.add_argument( 103 | "--catalog_url", 104 | default=None, 105 | type=str, 106 | help="the catalog to download the data from", 107 | ) 108 | 109 | 110 | def parse_argument_output_format(parser): 111 | parser.add_argument( 112 | "--output_format", 113 | "-f", 114 | default="json", 115 | type=str, 116 | help="format to show table ('json', 'text')", 117 | ) 118 | 119 | 120 | def parse_argument_output(parser): 121 | parser.add_argument( 122 | "--output_file", 123 | "-o", 124 | default=None, 125 | type=str, 126 | help="file to store the output (only json support)", 127 | ) 128 | 129 | 130 | def parse_argument_max_rows(parser): 131 | parser.add_argument( 132 | "--max_rows", 133 | "-n", 134 | default=100, 135 | type=int, 136 | help="maximum number of rows to output", 137 | ) 138 | 139 | 140 | def save_list_to_json(data_obj, fp): 141 | """Write list with dicts to json""" 142 | 143 | with open(fp, "w+") as f: 144 | for line in data_obj: 145 | f.write(json.dumps(line) + "\n") 146 | 147 | 148 | def main(): 149 | if len(sys.argv) > 1 and sys.argv[1] == "data": 150 | parser = argparse.ArgumentParser( 151 | prog="cbsodata", 152 | description=""" 153 | CBS Open Data: Command Line Interface 154 | 155 | Get data by table identifier. 156 | """, 157 | ) 158 | parse_argument_table_id(parser) 159 | parse_argument_catalog(parser) 160 | parse_argument_output_format(parser) 161 | parse_argument_max_rows(parser) 162 | parse_argument_output(parser) 163 | args = parser.parse_args(sys.argv[2:]) 164 | 165 | result = cbsodata.get_data(args.table_id, catalog_url=args.catalog_url) 166 | 167 | if args.output_file: 168 | save_list_to_json(result, args.output_file) 169 | 170 | if args.output_format == "text": 171 | text_outputter(result, max_rows=args.max_rows) 172 | else: 173 | json_outputter(result, max_rows=args.max_rows) 174 | 175 | elif len(sys.argv) > 1 and sys.argv[1] == "info": 176 | parser = argparse.ArgumentParser( 177 | prog="cbsodata", 178 | description=""" 179 | CBS Open Data: Command Line Interface 180 | 181 | Get data infomation by table identifier. 182 | """, 183 | ) 184 | parse_argument_table_id(parser) 185 | parse_argument_catalog(parser) 186 | parse_argument_output_format(parser) 187 | parse_argument_output(parser) 188 | args = parser.parse_args(sys.argv[2:]) 189 | 190 | result = cbsodata.get_info(args.table_id, catalog_url=args.catalog_url) 191 | 192 | if args.output_file: 193 | with open(args.output_file, "w") as f: 194 | json.dump(result, f, indent=4) 195 | 196 | if args.output_format == "text": 197 | text_outputter([{"Label": k, "Value": v} for k, v in result.items()]) 198 | else: 199 | print(json.dumps(result, indent=4)) 200 | 201 | elif len(sys.argv) > 1 and sys.argv[1] == "list": 202 | parser = argparse.ArgumentParser( 203 | prog="cbsodata", 204 | description=""" 205 | CBS Open Data: Command Line Interface 206 | 207 | Get list of available tables. 208 | """, 209 | ) 210 | parse_argument_catalog(parser) 211 | parse_argument_output_format(parser) 212 | parse_argument_max_rows(parser) 213 | parse_argument_output(parser) 214 | args = parser.parse_args(sys.argv[2:]) 215 | 216 | result = cbsodata.get_table_list(catalog_url=args.catalog_url) 217 | 218 | if args.output_file: 219 | save_list_to_json(result, args.output_file) 220 | 221 | if args.output_format == "text": 222 | text_outputter(result, max_rows=args.max_rows) 223 | else: 224 | json_outputter(result, max_rows=args.max_rows) 225 | 226 | # no valid sub command 227 | else: 228 | parser = argparse.ArgumentParser( 229 | prog="cbsodata", 230 | description=""" 231 | CBS Open Data: Command Line Interface 232 | """, 233 | ) 234 | parser.add_argument( 235 | "subcommand", 236 | nargs="?", 237 | type=lambda x: isinstance(x, str) and x in AVAILABLE_CMDS, 238 | help="the subcommand (one of '{}')".format("', '".join(AVAILABLE_CMDS)), 239 | ) 240 | parser.add_argument( 241 | "--version", action="store_true", help="show the package version" 242 | ) 243 | 244 | args = parser.parse_args() 245 | 246 | if args.subcommand is None: 247 | parser.print_help() 248 | 249 | 250 | if __name__ == "__main__": 251 | main() 252 | -------------------------------------------------------------------------------- /cbsodata/cbsodata3.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Jonathan de Bruin 2 | 3 | # Permission is hereby granted, free of charge, to any person 4 | # obtaining a copy of this software and associated documentation 5 | # files (the "Software"), to deal in the Software without 6 | # restriction, including without limitation the rights to use, 7 | # copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the 9 | # Software is furnished to do so, subject to the following 10 | # conditions: 11 | 12 | # The above copyright notice and this permission notice shall be 13 | # included in all copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | # OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | """Statistics Netherlands opendata API client for Python""" 25 | 26 | __all__ = [ 27 | "download_data", 28 | "get_data", 29 | "get_info", 30 | "get_meta", 31 | "get_table_list", 32 | "options", 33 | "catalog", 34 | ] 35 | 36 | import copy 37 | import json 38 | import logging 39 | import os 40 | import warnings 41 | from contextlib import contextmanager 42 | 43 | import requests 44 | from requests import Request 45 | from requests import Session 46 | 47 | CBSOPENDATA = "opendata.cbs.nl" # deprecate in next version 48 | API = "ODataApi/odata" 49 | BULK = "ODataFeed/odata" 50 | 51 | CATALOG = "ODataCatalog" 52 | FORMAT = "json" 53 | 54 | 55 | class OptionsManager: 56 | """Class for option management""" 57 | 58 | def __init__(self): 59 | self.use_https = True 60 | self.api_version = "3" 61 | # Get default proxy settings from environment variables 62 | proxies = { 63 | "http": os.environ.get("http_proxy", None), 64 | "https": os.environ.get("https_proxy", None), 65 | } 66 | self.requests = {"proxies": proxies} # proxies, cert, verify 67 | 68 | # Enable in next version 69 | # self.catalog_url = "opendata.cbs.nl" 70 | 71 | def __repr__(self): 72 | return self.__str__() 73 | 74 | def __str__(self): 75 | return f"catalog_url = {self.catalog_url}, use_https = {self.use_https}" 76 | 77 | def __getitem__(self, arg): 78 | return getattr(self, arg) 79 | 80 | def __setitem__(self, arg, value): 81 | setattr(self, arg, value) 82 | 83 | def _log_setting_change(self, setting_name, old_value, new_value): 84 | logging.info( 85 | f"Setting '{setting_name}' changed from '{old_value}' to '{new_value}'." 86 | ) 87 | 88 | def __setattr__(self, arg, value): 89 | try: 90 | old_value = copy.copy(getattr(self, arg)) 91 | except Exception: 92 | old_value = "undefined" 93 | 94 | self._log_setting_change(arg, old_value, value) 95 | super().__setattr__(arg, value) 96 | 97 | @property 98 | def catalog_url(self): 99 | return CBSOPENDATA 100 | 101 | @catalog_url.setter 102 | def catalog_url(self, url): 103 | global CBSOPENDATA 104 | CBSOPENDATA = url # noqa 105 | 106 | @property 107 | def proxies(self): 108 | return self.requests.get("proxies", None) 109 | 110 | @proxies.setter 111 | def proxies(self, proxies): 112 | warnings.warn( 113 | "Deprecated, use options.requests['proxies'] instead", 114 | DeprecationWarning, 115 | stacklevel=2, 116 | ) 117 | self.requests["proxies"] = proxies 118 | 119 | 120 | # User options 121 | options = OptionsManager() 122 | 123 | 124 | def _get_catalog_url(url): 125 | return options.catalog_url if url is None else url 126 | 127 | 128 | def _get_table_url(table_id, catalog_url=None): 129 | """Create a table url for the given table identifier.""" 130 | 131 | if catalog_url is None: 132 | _catalog_url = options.catalog_url 133 | else: 134 | _catalog_url = catalog_url 135 | 136 | components = { 137 | "http": "https://" if options.use_https else "http://", 138 | "baseurl": _catalog_url, 139 | "bulk": BULK, 140 | "table_id": table_id, 141 | } 142 | 143 | # http://opendata.cbs.nl/ODataApi/OData/37506wwm 144 | return "{http}{baseurl}/{bulk}/{table_id}/".format(**components) 145 | 146 | 147 | def _download_metadata( 148 | table_id, metadata_name, select=None, filters=None, catalog_url=None, **kwargs 149 | ): 150 | """Download metadata.""" 151 | 152 | # http://opendata.cbs.nl/ODataApi/OData/37506wwm/UntypedDataSet?$format=json 153 | url = _get_table_url(table_id, catalog_url=catalog_url) + metadata_name 154 | 155 | params = {} 156 | params["$format"] = FORMAT 157 | 158 | if select: 159 | params["$select"] = _select(select) 160 | if filters: 161 | params["$filter"] = _filters(filters) 162 | 163 | # additional parameters to requests 164 | request_kwargs = options.requests.copy() 165 | request_kwargs.update(kwargs) 166 | 167 | try: 168 | data = [] 169 | 170 | while url is not None: 171 | s = Session() 172 | p = Request("GET", url, params=params).prepare() 173 | 174 | logging.info("Download " + p.url) 175 | 176 | r = s.send(p, **request_kwargs) 177 | r.raise_for_status() 178 | r.encoding = "utf-8" 179 | 180 | res = json.loads(r.text) 181 | data.extend(res["value"]) 182 | 183 | try: 184 | url = res["odata.nextLink"] 185 | params = {} 186 | except KeyError: 187 | url = None 188 | 189 | return data 190 | 191 | except requests.HTTPError as http_err: 192 | raise requests.HTTPError( 193 | f"Downloading table '{table_id}' failed. {str(http_err)}" 194 | ) from http_err 195 | 196 | 197 | def _save_data(data, dir, metadata_name): 198 | """Save the data.""" 199 | 200 | if not os.path.exists(dir): 201 | os.makedirs(dir) 202 | 203 | fp = os.path.join(dir, metadata_name + ".json") 204 | 205 | with open(fp, "w") as output_file: 206 | json.dump(data, output_file, indent=2) 207 | 208 | 209 | def _filters(query): 210 | """Filter rows with a CBS-style query. 211 | 212 | Parameters 213 | ---------- 214 | query : str 215 | The rows to return. 216 | 217 | Returns 218 | ------- 219 | str 220 | Filter parameter for URL 221 | """ 222 | 223 | return query 224 | 225 | 226 | def _select(select): 227 | """Select columns. 228 | 229 | Parameters 230 | ---------- 231 | select : str 232 | The columns to return. 233 | 234 | Returns 235 | ------- 236 | str 237 | Select parameter for URL 238 | """ 239 | 240 | if isinstance(select, list): 241 | select = ",".join(select) 242 | 243 | return select 244 | 245 | 246 | def download_data( 247 | table_id, 248 | dir=None, 249 | typed=False, 250 | select=None, 251 | filters=None, 252 | catalog_url=None, 253 | **kwargs, 254 | ): 255 | """Download the CBS data and metadata. 256 | 257 | Parameters 258 | ---------- 259 | table_id : str 260 | The identifier of the table. 261 | dir : str 262 | Folder to save data to. If not given, data is not stored on 263 | disk. 264 | typed : bool 265 | Return a typed data table. Default False. 266 | select : str, list 267 | Column label or list of column labels to return. 268 | filters : str 269 | Return only rows that agree on the filter. 270 | catalog_url : str 271 | The url of the catalog. Default "opendata.cbs.nl". 272 | **kwargs : 273 | Optional arguments that ``requests.get()`` takes. For example, 274 | `proxies`, `cert` and `verify`. 275 | 276 | Returns 277 | ------- 278 | list 279 | A dictionary with the (meta)data of the table 280 | """ 281 | 282 | _catalog_url = _get_catalog_url(catalog_url) 283 | 284 | # http://opendata.cbs.nl/ODataApi/OData/37506wwm?$format=json 285 | metadata_tables = _download_metadata( 286 | table_id, "", catalog_url=_catalog_url, **kwargs 287 | ) 288 | 289 | # The names of the tables with metadata 290 | metadata_table_names = [table["name"] for table in metadata_tables] 291 | 292 | # Download only the typed or untyped data 293 | typed_or_not_str = "TypedDataSet" if typed else "UntypedDataSet" 294 | metadata_table_names.remove(typed_or_not_str) 295 | 296 | data = {} 297 | 298 | for table_name in metadata_table_names: 299 | # download table 300 | if table_name in ["TypedDataSet", "UntypedDataSet"]: 301 | metadata = _download_metadata( 302 | table_id, 303 | table_name, 304 | select=select, 305 | filters=filters, 306 | catalog_url=_catalog_url, 307 | **kwargs, 308 | ) 309 | else: 310 | metadata = _download_metadata( 311 | table_id, table_name, catalog_url=_catalog_url, **kwargs 312 | ) 313 | 314 | data[table_name] = metadata 315 | 316 | # save the data 317 | if dir: 318 | _save_data(metadata, dir, table_name) 319 | 320 | return data 321 | 322 | 323 | def get_table_list(select=None, filters=None, catalog_url=None, **kwargs): 324 | """Get a list with the available tables. 325 | 326 | Parameters 327 | ---------- 328 | select : list, str 329 | Column label or list of column labels to return. 330 | filters : str 331 | Return only rows that agree on the filter. 332 | catalog_url : str 333 | The url of the catalog. Default "opendata.cbs.nl". 334 | **kwargs : 335 | Optional arguments that ``requests.get()`` takes. For example, 336 | `proxies`, `cert` and `verify`. 337 | 338 | Returns 339 | ------- 340 | list 341 | A list with the description of each table in the catalog. 342 | """ 343 | 344 | # http://opendata.cbs.nl/ODataCatalog/Tables?$format=json&$filter=ShortTit 345 | # le%20eq%20%27Zeggenschap%20bedrijven;%20banen,%20grootte%27 346 | 347 | # http://opendata.cbs.nl/ODataCatalog/Tables?$format=json 348 | 349 | _catalog_url = _get_catalog_url(catalog_url) 350 | 351 | components = { 352 | "http": "https://" if options.use_https else "http://", 353 | "baseurl": _catalog_url, 354 | "catalog": CATALOG, 355 | } 356 | 357 | url = "{http}{baseurl}/{catalog}/Tables?$format=json".format(**components) 358 | 359 | params = {} 360 | if select: 361 | params["$select"] = _select(select) 362 | if filters: 363 | params["$filter"] = _filters(filters) 364 | 365 | # additional parameters to requests 366 | request_kwargs = options.requests.copy() 367 | request_kwargs.update(kwargs) 368 | 369 | try: 370 | s = Session() 371 | p = Request("GET", url, params=params).prepare() 372 | 373 | logging.info("Download " + p.url) 374 | 375 | r = s.send(p, **request_kwargs) 376 | r.raise_for_status() 377 | res = r.json() 378 | 379 | return res["value"] 380 | 381 | except requests.HTTPError as http_err: 382 | raise requests.HTTPError( 383 | f"Downloading table list failed. {str(http_err)}" 384 | ) from http_err 385 | 386 | 387 | def get_info(table_id, catalog_url=None, **kwargs): 388 | """Get information about a table. 389 | 390 | Parameters 391 | ---------- 392 | table_id : str 393 | The identifier of the table. 394 | catalog_url : str 395 | The url of the catalog. Default "opendata.cbs.nl". 396 | **kwargs : 397 | Optional arguments that ``requests.get()`` takes. For example, 398 | `proxies`, `cert` and `verify`. 399 | 400 | Returns 401 | ------- 402 | dict 403 | Table information 404 | """ 405 | 406 | info_list = _download_metadata( 407 | table_id, "TableInfos", catalog_url=_get_catalog_url(catalog_url), **kwargs 408 | ) 409 | 410 | if len(info_list) > 0: 411 | return info_list[0] 412 | else: 413 | return None 414 | 415 | 416 | def get_meta(table_id, name, catalog_url=None, **kwargs): 417 | """Get the metadata of a table. 418 | 419 | Parameters 420 | ---------- 421 | table_id : str 422 | The identifier of the table. 423 | name : str 424 | The name of the metadata (for example DataProperties). 425 | catalog_url : str 426 | The url of the catalog. Default "opendata.cbs.nl". 427 | **kwargs : 428 | Optional arguments that ``requests.get()`` takes. For example, 429 | `proxies`, `cert` and `verify`. 430 | 431 | Returns 432 | ------- 433 | list 434 | A list with metadata (dict type) 435 | """ 436 | 437 | return _download_metadata( 438 | table_id, name, catalog_url=_get_catalog_url(catalog_url), **kwargs 439 | ) 440 | 441 | 442 | def get_data( 443 | table_id, 444 | dir=None, 445 | typed=False, 446 | select=None, 447 | filters=None, 448 | catalog_url=None, 449 | **kwargs, 450 | ): 451 | """Get the CBS data table. 452 | 453 | Parameters 454 | ---------- 455 | table_id : str 456 | The identifier of the table. 457 | dir : str 458 | Folder to save data to. If not given, data is not stored 459 | on disk. 460 | typed : bool 461 | Return a typed data table. Default False. 462 | select : list 463 | Column label or list of column labels to return. 464 | filters : str 465 | Return only rows that agree on the filter. 466 | catalog_url : str 467 | The url of the catalog. Default "opendata.cbs.nl". 468 | **kwargs : 469 | Optional arguments that ``requests.get()`` takes. For example, 470 | `proxies`, `cert` and `verify`. 471 | 472 | Returns 473 | ------- 474 | list 475 | The requested data. 476 | """ 477 | 478 | _catalog_url = _get_catalog_url(catalog_url) 479 | 480 | metadata = download_data( 481 | table_id, 482 | dir=dir, 483 | typed=typed, 484 | select=select, 485 | filters=filters, 486 | catalog_url=_catalog_url, 487 | **kwargs, 488 | ) 489 | 490 | if "TypedDataSet" in metadata.keys(): 491 | data = metadata["TypedDataSet"] 492 | else: 493 | data = metadata["UntypedDataSet"] 494 | 495 | exclude = [ 496 | "TableInfos", 497 | "TypedDataSet", 498 | "UntypedDataSet", 499 | "DataProperties", 500 | "CategoryGroups", 501 | ] 502 | 503 | norm_cols = list(set(metadata.keys()) - set(exclude)) 504 | 505 | for norm_col in norm_cols: 506 | metadata[norm_col] = {r["Key"]: r for r in metadata[norm_col]} 507 | 508 | for i in range(0, len(data)): 509 | for norm_col in norm_cols: 510 | try: 511 | v = data[i][norm_col] 512 | data[i][norm_col] = metadata[norm_col][v]["Title"] 513 | except KeyError: 514 | pass 515 | 516 | return data 517 | 518 | 519 | @contextmanager 520 | def catalog(catalog_url, use_https=True): 521 | """Context manager for catalogs. 522 | 523 | Parameters 524 | ---------- 525 | catalog_url : str 526 | Url for the catalog. For example: 527 | dataderden.cbs.nl. 528 | use_https : bool 529 | Use https. Default True. 530 | 531 | """ 532 | 533 | old_url = copy.copy(options.catalog_url) 534 | options.catalog_url = catalog_url 535 | 536 | yield 537 | 538 | options.catalog_url = old_url 539 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @echo " html to make standalone HTML files" 21 | @echo " dirhtml to make HTML files named index.html in directories" 22 | @echo " singlehtml to make a single large HTML file" 23 | @echo " pickle to make pickle files" 24 | @echo " json to make JSON files" 25 | @echo " htmlhelp to make HTML files and a HTML help project" 26 | @echo " qthelp to make HTML files and a qthelp project" 27 | @echo " applehelp to make an Apple Help Book" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " epub3 to make an epub3" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " xml to make Docutils-native XML files" 41 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | @echo " coverage to run coverage check of the documentation (if enabled)" 45 | @echo " dummy to check syntax errors of document sources" 46 | 47 | .PHONY: clean 48 | clean: 49 | rm -rf $(BUILDDIR)/* 50 | 51 | .PHONY: html 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | .PHONY: dirhtml 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | .PHONY: singlehtml 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | .PHONY: pickle 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | .PHONY: json 76 | json: 77 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 78 | @echo 79 | @echo "Build finished; now you can process the JSON files." 80 | 81 | .PHONY: htmlhelp 82 | htmlhelp: 83 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 84 | @echo 85 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 86 | ".hhp project file in $(BUILDDIR)/htmlhelp." 87 | 88 | .PHONY: qthelp 89 | qthelp: 90 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 91 | @echo 92 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 93 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 94 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyCBS.qhcp" 95 | @echo "To view the help file:" 96 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyCBS.qhc" 97 | 98 | .PHONY: applehelp 99 | applehelp: 100 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 101 | @echo 102 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 103 | @echo "N.B. You won't be able to view it unless you put it in" \ 104 | "~/Library/Documentation/Help or install it in your application" \ 105 | "bundle." 106 | 107 | .PHONY: devhelp 108 | devhelp: 109 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 110 | @echo 111 | @echo "Build finished." 112 | @echo "To view the help file:" 113 | @echo "# mkdir -p $$HOME/.local/share/devhelp/PyCBS" 114 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyCBS" 115 | @echo "# devhelp" 116 | 117 | .PHONY: epub 118 | epub: 119 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 120 | @echo 121 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 122 | 123 | .PHONY: epub3 124 | epub3: 125 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 126 | @echo 127 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 128 | 129 | .PHONY: latex 130 | latex: 131 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 132 | @echo 133 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 134 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 135 | "(use \`make latexpdf' here to do that automatically)." 136 | 137 | .PHONY: latexpdf 138 | latexpdf: 139 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 140 | @echo "Running LaTeX files through pdflatex..." 141 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 142 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 143 | 144 | .PHONY: latexpdfja 145 | latexpdfja: 146 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 147 | @echo "Running LaTeX files through platex and dvipdfmx..." 148 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 149 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 150 | 151 | .PHONY: text 152 | text: 153 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 154 | @echo 155 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 156 | 157 | .PHONY: man 158 | man: 159 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 160 | @echo 161 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 162 | 163 | .PHONY: texinfo 164 | texinfo: 165 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 166 | @echo 167 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 168 | @echo "Run \`make' in that directory to run these through makeinfo" \ 169 | "(use \`make info' here to do that automatically)." 170 | 171 | .PHONY: info 172 | info: 173 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 174 | @echo "Running Texinfo files through makeinfo..." 175 | make -C $(BUILDDIR)/texinfo info 176 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 177 | 178 | .PHONY: gettext 179 | gettext: 180 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 181 | @echo 182 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 183 | 184 | .PHONY: changes 185 | changes: 186 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 187 | @echo 188 | @echo "The overview file is in $(BUILDDIR)/changes." 189 | 190 | .PHONY: linkcheck 191 | linkcheck: 192 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 193 | @echo 194 | @echo "Link check complete; look for any errors in the above output " \ 195 | "or in $(BUILDDIR)/linkcheck/output.txt." 196 | 197 | .PHONY: doctest 198 | doctest: 199 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 200 | @echo "Testing of doctests in the sources finished, look at the " \ 201 | "results in $(BUILDDIR)/doctest/output.txt." 202 | 203 | .PHONY: coverage 204 | coverage: 205 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 206 | @echo "Testing of coverage in the sources finished, look at the " \ 207 | "results in $(BUILDDIR)/coverage/python.txt." 208 | 209 | .PHONY: xml 210 | xml: 211 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 212 | @echo 213 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 214 | 215 | .PHONY: pseudoxml 216 | pseudoxml: 217 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 218 | @echo 219 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 220 | 221 | .PHONY: dummy 222 | dummy: 223 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 224 | @echo 225 | @echo "Build finished. Dummy builder generates no files." 226 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # 2 | # CBSOData documentation build configuration file, created by 3 | # sphinx-quickstart on Sun Dec 11 20:22:38 2016. 4 | # 5 | # This file is execfile()d with the current directory set to its 6 | # containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | # If extensions (or modules to document with autodoc) are in another directory, 15 | # add these directories to sys.path here. If the directory is relative to the 16 | # documentation root, use os.path.abspath to make it absolute, like shown here. 17 | # 18 | import os 19 | import sys 20 | 21 | sys.path.insert(0, os.path.abspath("..")) 22 | 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon"] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ["_templates"] 37 | 38 | # The suffix(es) of source filenames. 39 | # You can specify multiple suffix as a list of string: 40 | # 41 | # source_suffix = ['.rst', '.md'] 42 | source_suffix = ".rst" 43 | 44 | # The encoding of source files. 45 | # 46 | # source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = "index" 50 | 51 | # General information about the project. 52 | project = "CBSOData" 53 | copyright = "2016, Jonathan de Bruin" 54 | author = "Jonathan de Bruin" 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = "1.3" 62 | # The full version, including alpha/beta/rc tags. 63 | release = "1.3" 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | # 68 | # This is also used if you do content translation via gettext catalogs. 69 | # Usually you set "language" from the command line for these cases. 70 | language = None 71 | 72 | # There are two options for replacing |today|: either, you set today to some 73 | # non-false value, then it is used: 74 | # 75 | # today = '' 76 | # 77 | # Else, today_fmt is used as the format for a strftime call. 78 | # 79 | # today_fmt = '%B %d, %Y' 80 | 81 | # List of patterns, relative to source directory, that match files and 82 | # directories to ignore when looking for source files. 83 | # This patterns also effect to html_static_path and html_extra_path 84 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 85 | 86 | # The reST default role (used for this markup: `text`) to use for all 87 | # documents. 88 | # 89 | # default_role = None 90 | 91 | # If true, '()' will be appended to :func: etc. cross-reference text. 92 | # 93 | # add_function_parentheses = True 94 | 95 | # If true, the current module name will be prepended to all description 96 | # unit titles (such as .. function::). 97 | # 98 | # add_module_names = True 99 | 100 | # If true, sectionauthor and moduleauthor directives will be shown in the 101 | # output. They are ignored by default. 102 | # 103 | # show_authors = False 104 | 105 | # The name of the Pygments (syntax highlighting) style to use. 106 | pygments_style = "sphinx" 107 | 108 | # A list of ignored prefixes for module index sorting. 109 | # modindex_common_prefix = [] 110 | 111 | # If true, keep warnings as "system message" paragraphs in the built documents. 112 | # keep_warnings = False 113 | 114 | # If true, `todo` and `todoList` produce output, else they produce nothing. 115 | todo_include_todos = False 116 | 117 | 118 | # -- Options for HTML output ---------------------------------------------- 119 | 120 | # The theme to use for HTML and HTML Help pages. See the documentation for 121 | # a list of builtin themes. 122 | # 123 | html_theme = "sphinx_rtd_theme" 124 | 125 | # Theme options are theme-specific and customize the look and feel of a theme 126 | # further. For a list of options available for each theme, see the 127 | # documentation. 128 | # 129 | # html_theme_options = {} 130 | 131 | # Add any paths that contain custom themes here, relative to this directory. 132 | # html_theme_path = [] 133 | 134 | # The name for this set of Sphinx documents. 135 | # " v documentation" by default. 136 | # 137 | # html_title = u'CBSOData v0.1' 138 | 139 | # A shorter title for the navigation bar. Default is the same as html_title. 140 | # 141 | # html_short_title = None 142 | 143 | # The name of an image file (relative to this directory) to place at the top 144 | # of the sidebar. 145 | # 146 | # html_logo = None 147 | 148 | # The name of an image file (relative to this directory) to use as a favicon of 149 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 150 | # pixels large. 151 | # 152 | # html_favicon = None 153 | 154 | # Add any paths that contain custom static files (such as style sheets) here, 155 | # relative to this directory. They are copied after the builtin static files, 156 | # so a file named "default.css" will overwrite the builtin "default.css". 157 | html_static_path = ["_static"] 158 | 159 | # Add any extra paths that contain custom files (such as robots.txt or 160 | # .htaccess) here, relative to this directory. These files are copied 161 | # directly to the root of the documentation. 162 | # 163 | # html_extra_path = [] 164 | 165 | # If not None, a 'Last updated on:' timestamp is inserted at every page 166 | # bottom, using the given strftime format. 167 | # The empty string is equivalent to '%b %d, %Y'. 168 | # 169 | # html_last_updated_fmt = None 170 | 171 | # If true, SmartyPants will be used to convert quotes and dashes to 172 | # typographically correct entities. 173 | # 174 | # html_use_smartypants = True 175 | 176 | # Custom sidebar templates, maps document names to template names. 177 | # 178 | # html_sidebars = {} 179 | 180 | # Additional templates that should be rendered to pages, maps page names to 181 | # template names. 182 | # 183 | # html_additional_pages = {} 184 | 185 | # If false, no module index is generated. 186 | # 187 | # html_domain_indices = True 188 | 189 | # If false, no index is generated. 190 | # 191 | # html_use_index = True 192 | 193 | # If true, the index is split into individual pages for each letter. 194 | # 195 | # html_split_index = False 196 | 197 | # If true, links to the reST sources are added to the pages. 198 | # 199 | # html_show_sourcelink = True 200 | 201 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 202 | # 203 | # html_show_sphinx = True 204 | 205 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 206 | # 207 | # html_show_copyright = True 208 | 209 | # If true, an OpenSearch description file will be output, and all pages will 210 | # contain a tag referring to it. The value of this option must be the 211 | # base URL from which the finished HTML is served. 212 | # 213 | # html_use_opensearch = '' 214 | 215 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 216 | # html_file_suffix = None 217 | 218 | # Language to be used for generating the HTML full-text search index. 219 | # Sphinx supports the following languages: 220 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 221 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' 222 | # 223 | # html_search_language = 'en' 224 | 225 | # A dictionary with options for the search language support, empty by default. 226 | # 'ja' uses this config value. 227 | # 'zh' user can custom change `jieba` dictionary path. 228 | # 229 | # html_search_options = {'type': 'default'} 230 | 231 | # The name of a javascript file (relative to the configuration directory) that 232 | # implements a search results scorer. If empty, the default will be used. 233 | # 234 | # html_search_scorer = 'scorer.js' 235 | 236 | # Output file base name for HTML help builder. 237 | htmlhelp_basename = "CBSODatadoc" 238 | 239 | # -- Options for LaTeX output --------------------------------------------- 240 | 241 | latex_elements = { 242 | # The paper size ('letterpaper' or 'a4paper'). 243 | # 244 | # 'papersize': 'letterpaper', 245 | # The font size ('10pt', '11pt' or '12pt'). 246 | # 247 | # 'pointsize': '10pt', 248 | # Additional stuff for the LaTeX preamble. 249 | # 250 | # 'preamble': '', 251 | # Latex figure (float) alignment 252 | # 253 | # 'figure_align': 'htbp', 254 | } 255 | 256 | # Grouping the document tree into LaTeX files. List of tuples 257 | # (source start file, target name, title, 258 | # author, documentclass [howto, manual, or own class]). 259 | latex_documents = [ 260 | ( 261 | master_doc, 262 | "CBSOData.tex", 263 | "CBSOData Documentation", 264 | "Jonathan de Bruin", 265 | "manual", 266 | ), 267 | ] 268 | 269 | # The name of an image file (relative to this directory) to place at the top of 270 | # the title page. 271 | # 272 | # latex_logo = None 273 | 274 | # For "manual" documents, if this is true, then toplevel headings are parts, 275 | # not chapters. 276 | # 277 | # latex_use_parts = False 278 | 279 | # If true, show page references after internal links. 280 | # 281 | # latex_show_pagerefs = False 282 | 283 | # If true, show URL addresses after external links. 284 | # 285 | # latex_show_urls = False 286 | 287 | # Documents to append as an appendix to all manuals. 288 | # 289 | # latex_appendices = [] 290 | 291 | # It false, will not define \strong, \code, itleref, \crossref ... but only 292 | # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added 293 | # packages. 294 | # 295 | # latex_keep_old_macro_names = True 296 | 297 | # If false, no module index is generated. 298 | # 299 | # latex_domain_indices = True 300 | 301 | 302 | # -- Options for manual page output --------------------------------------- 303 | 304 | # One entry per manual page. List of tuples 305 | # (source start file, name, description, authors, manual section). 306 | man_pages = [(master_doc, "CBSOData", "CBSOData Documentation", [author], 1)] 307 | 308 | # If true, show URL addresses after external links. 309 | # 310 | # man_show_urls = False 311 | 312 | 313 | # -- Options for Texinfo output ------------------------------------------- 314 | 315 | # Grouping the document tree into Texinfo files. List of tuples 316 | # (source start file, target name, title, author, 317 | # dir menu entry, description, category) 318 | texinfo_documents = [ 319 | ( 320 | master_doc, 321 | "CBSOData", 322 | "CBSOData Documentation", 323 | author, 324 | "CBSOData", 325 | "One line description of project.", 326 | "Miscellaneous", 327 | ), 328 | ] 329 | 330 | # Documents to append as an appendix to all manuals. 331 | # 332 | # texinfo_appendices = [] 333 | 334 | # If false, no module index is generated. 335 | # 336 | # texinfo_domain_indices = True 337 | 338 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 339 | # 340 | # texinfo_show_urls = 'footnote' 341 | 342 | # If true, do not generate a @detailmenu in the "Top" node's menu. 343 | # 344 | # texinfo_no_detailmenu = False 345 | 346 | 347 | ## Project settings 348 | 349 | # Napoleon 350 | napoleon_google_docstring = False 351 | napoleon_numpy_docstring = True 352 | napoleon_include_init_with_doc = False 353 | napoleon_include_private_with_doc = False 354 | napoleon_include_special_with_doc = True 355 | napoleon_use_admonition_for_examples = False 356 | napoleon_use_admonition_for_notes = False 357 | napoleon_use_admonition_for_references = False 358 | napoleon_use_ivar = False 359 | napoleon_use_param = True 360 | napoleon_use_rtype = True 361 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to PyCBS's documentation! 2 | ================================= 3 | 4 | Contents: 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | readme_link 10 | reference 11 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. epub3 to make an epub3 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. xml to make Docutils-native XML files 38 | echo. pseudoxml to make pseudoxml-XML files for display purposes 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | echo. coverage to run coverage check of the documentation if enabled 42 | echo. dummy to check syntax errors of document sources 43 | goto end 44 | ) 45 | 46 | if "%1" == "clean" ( 47 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 48 | del /q /s %BUILDDIR%\* 49 | goto end 50 | ) 51 | 52 | 53 | REM Check if sphinx-build is available and fallback to Python version if any 54 | %SPHINXBUILD% 1>NUL 2>NUL 55 | if errorlevel 9009 goto sphinx_python 56 | goto sphinx_ok 57 | 58 | :sphinx_python 59 | 60 | set SPHINXBUILD=python -m sphinx.__init__ 61 | %SPHINXBUILD% 2> nul 62 | if errorlevel 9009 ( 63 | echo. 64 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 65 | echo.installed, then set the SPHINXBUILD environment variable to point 66 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 67 | echo.may add the Sphinx directory to PATH. 68 | echo. 69 | echo.If you don't have Sphinx installed, grab it from 70 | echo.http://sphinx-doc.org/ 71 | exit /b 1 72 | ) 73 | 74 | :sphinx_ok 75 | 76 | 77 | if "%1" == "html" ( 78 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 79 | if errorlevel 1 exit /b 1 80 | echo. 81 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 82 | goto end 83 | ) 84 | 85 | if "%1" == "dirhtml" ( 86 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 87 | if errorlevel 1 exit /b 1 88 | echo. 89 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 90 | goto end 91 | ) 92 | 93 | if "%1" == "singlehtml" ( 94 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 95 | if errorlevel 1 exit /b 1 96 | echo. 97 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 98 | goto end 99 | ) 100 | 101 | if "%1" == "pickle" ( 102 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 103 | if errorlevel 1 exit /b 1 104 | echo. 105 | echo.Build finished; now you can process the pickle files. 106 | goto end 107 | ) 108 | 109 | if "%1" == "json" ( 110 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 111 | if errorlevel 1 exit /b 1 112 | echo. 113 | echo.Build finished; now you can process the JSON files. 114 | goto end 115 | ) 116 | 117 | if "%1" == "htmlhelp" ( 118 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 119 | if errorlevel 1 exit /b 1 120 | echo. 121 | echo.Build finished; now you can run HTML Help Workshop with the ^ 122 | .hhp project file in %BUILDDIR%/htmlhelp. 123 | goto end 124 | ) 125 | 126 | if "%1" == "qthelp" ( 127 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 128 | if errorlevel 1 exit /b 1 129 | echo. 130 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 131 | .qhcp project file in %BUILDDIR%/qthelp, like this: 132 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyCBS.qhcp 133 | echo.To view the help file: 134 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyCBS.ghc 135 | goto end 136 | ) 137 | 138 | if "%1" == "devhelp" ( 139 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 140 | if errorlevel 1 exit /b 1 141 | echo. 142 | echo.Build finished. 143 | goto end 144 | ) 145 | 146 | if "%1" == "epub" ( 147 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 148 | if errorlevel 1 exit /b 1 149 | echo. 150 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 151 | goto end 152 | ) 153 | 154 | if "%1" == "epub3" ( 155 | %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 156 | if errorlevel 1 exit /b 1 157 | echo. 158 | echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. 159 | goto end 160 | ) 161 | 162 | if "%1" == "latex" ( 163 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 164 | if errorlevel 1 exit /b 1 165 | echo. 166 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdf" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "latexpdfja" ( 181 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 182 | cd %BUILDDIR%/latex 183 | make all-pdf-ja 184 | cd %~dp0 185 | echo. 186 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 187 | goto end 188 | ) 189 | 190 | if "%1" == "text" ( 191 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 192 | if errorlevel 1 exit /b 1 193 | echo. 194 | echo.Build finished. The text files are in %BUILDDIR%/text. 195 | goto end 196 | ) 197 | 198 | if "%1" == "man" ( 199 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 200 | if errorlevel 1 exit /b 1 201 | echo. 202 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 203 | goto end 204 | ) 205 | 206 | if "%1" == "texinfo" ( 207 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 208 | if errorlevel 1 exit /b 1 209 | echo. 210 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 211 | goto end 212 | ) 213 | 214 | if "%1" == "gettext" ( 215 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 216 | if errorlevel 1 exit /b 1 217 | echo. 218 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 219 | goto end 220 | ) 221 | 222 | if "%1" == "changes" ( 223 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 224 | if errorlevel 1 exit /b 1 225 | echo. 226 | echo.The overview file is in %BUILDDIR%/changes. 227 | goto end 228 | ) 229 | 230 | if "%1" == "linkcheck" ( 231 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 232 | if errorlevel 1 exit /b 1 233 | echo. 234 | echo.Link check complete; look for any errors in the above output ^ 235 | or in %BUILDDIR%/linkcheck/output.txt. 236 | goto end 237 | ) 238 | 239 | if "%1" == "doctest" ( 240 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 241 | if errorlevel 1 exit /b 1 242 | echo. 243 | echo.Testing of doctests in the sources finished, look at the ^ 244 | results in %BUILDDIR%/doctest/output.txt. 245 | goto end 246 | ) 247 | 248 | if "%1" == "coverage" ( 249 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 250 | if errorlevel 1 exit /b 1 251 | echo. 252 | echo.Testing of coverage in the sources finished, look at the ^ 253 | results in %BUILDDIR%/coverage/python.txt. 254 | goto end 255 | ) 256 | 257 | if "%1" == "xml" ( 258 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 259 | if errorlevel 1 exit /b 1 260 | echo. 261 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 262 | goto end 263 | ) 264 | 265 | if "%1" == "pseudoxml" ( 266 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 267 | if errorlevel 1 exit /b 1 268 | echo. 269 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 270 | goto end 271 | ) 272 | 273 | if "%1" == "dummy" ( 274 | %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy 275 | if errorlevel 1 exit /b 1 276 | echo. 277 | echo.Build finished. Dummy builder generates no files. 278 | goto end 279 | ) 280 | 281 | :end 282 | -------------------------------------------------------------------------------- /docs/readme_link.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | .. automodule:: cbsodata 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx-rtd-theme 2 | requests 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "cbsodata" 3 | description = "Statistics Netherlands opendata API client for Python" 4 | authors = [ 5 | { name = "Jonathan de Bruin", email = "jonathandebruinos@gmail.com" } 6 | ] 7 | readme = "README.md" 8 | classifiers = [ 9 | "Development Status :: 5 - Production/Stable", 10 | "License :: OSI Approved :: MIT License", 11 | "Programming Language :: Python :: 3.8", 12 | "Programming Language :: Python :: 3.9", 13 | "Programming Language :: Python :: 3.10", 14 | "Programming Language :: Python :: 3.11", 15 | "Programming Language :: Python :: 3.12" 16 | ] 17 | license = {text = "MIT"} 18 | dependencies = ["requests"] 19 | dynamic = ["version"] 20 | requires-python = ">=3.8" 21 | 22 | [project.scripts] 23 | cbsodata = "cbsodata.__main__:main" 24 | cbs = "cbsodata.__main__:main" 25 | 26 | [project.optional-dependencies] 27 | lint = ["ruff"] 28 | test = ["pytest"] 29 | 30 | [project.urls] 31 | Homepage = "https://github.com/J535D165/cbsodata" 32 | Documentation = "http://cbsodata.readthedocs.io/" 33 | 34 | [build-system] 35 | build-backend = 'setuptools.build_meta' 36 | requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] 37 | 38 | [tool.setuptools] 39 | packages = ["cbsodata"] 40 | 41 | [tool.setuptools_scm] 42 | write_to = "cbsodata/_version.py" 43 | 44 | [tool.ruff] 45 | select = ["E", "F", "UP", "I", "B"] 46 | 47 | [tool.ruff.isort] 48 | force-single-line = true 49 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/J535D165/cbsodata/cbb5b33b01fce09ca9781478b41a3f5131a7544b/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_cbsodata.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | import requests 5 | 6 | import cbsodata 7 | 8 | datasets = ["82010NED", "80884ENG"] 9 | 10 | datasets_derden = ["47003NED", "47005NED"] 11 | 12 | 13 | @pytest.mark.parametrize("table_id", datasets) 14 | def test_info(table_id): 15 | info = cbsodata.get_info(table_id) 16 | 17 | assert isinstance(info, dict) 18 | 19 | 20 | @pytest.mark.parametrize("table_id", datasets) 21 | def test_download(table_id, tmpdir): 22 | cbsodata.download_data(table_id, dir=tmpdir) 23 | 24 | 25 | @pytest.mark.parametrize("table_id", ["00000AAA"]) 26 | def test_http_error(table_id): 27 | try: 28 | cbsodata.get_data(table_id) 29 | except requests.HTTPError: 30 | assert True 31 | else: 32 | raise AssertionError() 33 | 34 | 35 | def test_http_error_table_list(): 36 | try: 37 | cbsodata.get_table_list(catalog_url="test.cbs.nl") 38 | except requests.ConnectionError: 39 | assert True 40 | else: 41 | raise AssertionError() 42 | 43 | 44 | @pytest.mark.parametrize("table_id", datasets) 45 | def test_http_https_download(table_id, tmpdir): 46 | cbsodata.options["use_https"] = True 47 | cbsodata.download_data(table_id, dir=tmpdir) 48 | cbsodata.options["use_https"] = False 49 | cbsodata.download_data(table_id, dir=tmpdir) 50 | cbsodata.options["use_https"] = True 51 | 52 | 53 | @pytest.mark.parametrize("table_id", datasets) 54 | def test_download_and_store(table_id, tmpdir): 55 | cbsodata.download_data(table_id, dir=tmpdir) 56 | 57 | assert Path(tmpdir, "TableInfos.json").exists() 58 | 59 | 60 | @pytest.mark.parametrize("table_id", datasets) 61 | def test_get_data(table_id): 62 | cbsodata.get_data(table_id) 63 | 64 | 65 | @pytest.mark.parametrize("table_id", datasets) 66 | def test_info_values(table_id): 67 | info = cbsodata.get_info(table_id) 68 | 69 | # Check response is dict (not a list) 70 | assert isinstance(info, dict) 71 | 72 | # Check required keys are available 73 | assert "Description" in info.keys() 74 | assert "ID" in info.keys() 75 | assert "Identifier" in info.keys() 76 | 77 | 78 | def test_table_list(): 79 | assert len(cbsodata.get_table_list()) > 100 80 | 81 | 82 | def test_filters(): 83 | default_sel_filt = cbsodata.get_info("82070ENG")["DefaultSelection"] 84 | filters_and_selections = default_sel_filt.split("&") 85 | 86 | for fs in filters_and_selections: 87 | if fs.startswith("$filter="): 88 | filt = fs[8:] 89 | 90 | cbsodata.get_data("82070ENG", filters=filt) 91 | 92 | 93 | def test_select(): 94 | default_sel_filt = cbsodata.get_info("82070ENG")["DefaultSelection"] 95 | filters_and_selections = default_sel_filt.split("&") 96 | 97 | for fs in filters_and_selections: 98 | if fs.startswith("$select="): 99 | select = fs[8:] 100 | 101 | cbsodata.get_data("82070ENG", select=select) 102 | 103 | 104 | def test_select_list(): 105 | default_sel_filt = cbsodata.get_info("82070ENG")["DefaultSelection"] 106 | filters_and_selections = default_sel_filt.split("&") 107 | 108 | for fs in filters_and_selections: 109 | if fs.startswith("$select="): 110 | select = fs[8:] 111 | 112 | cbsodata.get_data("82070ENG", select=select.split(", ")) 113 | 114 | 115 | def test_select_subset(): 116 | default_sel_filt = cbsodata.get_info("82070ENG")["DefaultSelection"] 117 | filters_and_selections = default_sel_filt.split("&") 118 | 119 | for fs in filters_and_selections: 120 | if fs.startswith("$select="): 121 | select = fs[8:] 122 | 123 | select_list = select.split(", ") 124 | cbsodata.get_data("82070ENG", select=select_list[0:2]) 125 | 126 | 127 | def test_select_n_cols(): 128 | default_sel_filt = cbsodata.get_info("82070ENG")["DefaultSelection"] 129 | filters_and_selections = default_sel_filt.split("&") 130 | 131 | for fs in filters_and_selections: 132 | if fs.startswith("$select="): 133 | select = fs[8:] 134 | 135 | select_list = select.split(", ") 136 | data = cbsodata.get_data("82070ENG", select=select_list[0:2]) 137 | 138 | assert len(data[0].keys()) == 2 139 | assert len(data[5].keys()) == 2 140 | assert len(data[10].keys()) == 2 141 | 142 | 143 | @pytest.mark.parametrize("table_id", datasets_derden) 144 | def test_get_table_list_derden(table_id): 145 | # option 1 146 | print("global") 147 | cbsodata.options.catalog_url = "dataderden.cbs.nl" 148 | data_option1 = cbsodata.get_table_list() 149 | cbsodata.options.catalog_url = "opendata.cbs.nl" 150 | 151 | # option 2 152 | print("context") 153 | with cbsodata.catalog("dataderden.cbs.nl"): 154 | data_option2 = cbsodata.get_table_list() 155 | 156 | # option 3 157 | print("argument") 158 | data_option3 = cbsodata.get_table_list(catalog_url="dataderden.cbs.nl") 159 | 160 | assert len(data_option1[0].keys()) > 0 161 | 162 | for key in data_option1[0].keys(): 163 | assert data_option1[0][key] == data_option2[0][key] == data_option3[0][key] 164 | 165 | 166 | @pytest.mark.parametrize("table_id", datasets_derden) 167 | def test_get_data_derden(table_id): 168 | # option 1 169 | print("global") 170 | cbsodata.options.catalog_url = "dataderden.cbs.nl" 171 | data_option1 = cbsodata.get_data(table_id) 172 | cbsodata.options.catalog_url = "opendata.cbs.nl" 173 | 174 | # option 2 175 | print("context") 176 | with cbsodata.catalog("dataderden.cbs.nl"): 177 | data_option2 = cbsodata.get_data(table_id) 178 | 179 | # option 3 180 | print("argument") 181 | data_option3 = cbsodata.get_data(table_id, catalog_url="dataderden.cbs.nl") 182 | 183 | assert len(data_option1[0].keys()) > 0 184 | 185 | for key in data_option1[0].keys(): 186 | assert data_option1[0][key] == data_option2[0][key] == data_option3[0][key] 187 | --------------------------------------------------------------------------------