'
13 |
14 | hr_faded: '
'
15 | hr_shaded: '
'
--------------------------------------------------------------------------------
/settings.ini:
--------------------------------------------------------------------------------
1 | [DEFAULT]
2 | host = github
3 | lib_name = dtype_diet
4 | user = noklam
5 | description = Attempt to shrink Pandas `dtypes` without losing data so you have more RAM (and maybe more speed)
6 | keywords = pandas, optimization
7 | author = noklam
8 | author_email = mediumnok@gmail.com
9 | copyright = Chan Nok Lam
10 | branch = master
11 | version = 0.0.3
12 | min_python = 3.7
13 | audience = Developers
14 | language = English
15 | custom_sidebar = False
16 | license = MIT
17 | status = 2
18 | requirements = pandas>=1.0.0 tabulate
19 | nbs_path = .
20 | doc_path = docs
21 | doc_host = https://noklam.github.io
22 | doc_baseurl = /dtype_diet/
23 | git_url = https://github.com/noklam/dtype_diet/tree/master/
24 | lib_path = dtype_diet
25 | title = dtype_diet
26 | tst_flags = slow
27 | recursive = True
28 | black_formatting = False
29 | readme_nb = index.ipynb
30 | allowed_metadata_keys =
31 | allowed_cell_metadata_keys =
32 | jupyter_hooks = True
33 | clean_ids = True
34 | clear_all = False
35 | put_version_in_init = True
36 |
37 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: "3"
2 | services:
3 | fastai: &fastai
4 | restart: unless-stopped
5 | working_dir: /data
6 | image: fastai/codespaces
7 | logging:
8 | driver: json-file
9 | options:
10 | max-size: 50m
11 | stdin_open: true
12 | tty: true
13 | volumes:
14 | - .:/data/
15 |
16 | notebook:
17 | <<: *fastai
18 | command: bash -c "pip install -e . && jupyter notebook --allow-root --no-browser --ip=0.0.0.0 --port=8080 --NotebookApp.token='' --NotebookApp.password=''"
19 | ports:
20 | - "8080:8080"
21 |
22 | watcher:
23 | <<: *fastai
24 | command: watchmedo shell-command --command nbdev_build_docs --pattern *.ipynb --recursive --drop
25 | network_mode: host # for GitHub Codespaces https://github.com/features/codespaces/
26 |
27 | jekyll:
28 | <<: *fastai
29 | ports:
30 | - "4000:4000"
31 | command: >
32 | bash -c "cp -r docs_src docs
33 | && pip install .
34 | && nbdev_build_docs && cd docs
35 | && bundle i
36 | && chmod -R u+rwx . && bundle exec jekyll serve --host 0.0.0.0"
37 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Chan Nok Lam
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/docs/licenses/LICENSE:
--------------------------------------------------------------------------------
1 | /* This license pertains to the docs template, except for the Navgoco jQuery component. */
2 |
3 | The MIT License (MIT)
4 |
5 | Original theme: Copyright (c) 2016 Tom Johnson
6 | Modifications: Copyright (c) 2017 onwards fast.ai, Inc
7 |
8 | Permission is hereby granted, free of charge, to any person obtaining a copy
9 | of this software and associated documentation files (the "Software"), to deal
10 | in the Software without restriction, including without limitation the rights
11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 | copies of the Software, and to permit persons to whom the Software is
13 | furnished to do so, subject to the following conditions:
14 |
15 | The above copyright notice and this permission notice shall be included in all
16 | copies or substantial portions of the Software.
17 |
18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 | SOFTWARE.
25 |
--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 | on: [push, pull_request]
3 | jobs:
4 | build:
5 | runs-on: ubuntu-latest
6 | steps:
7 | - uses: actions/checkout@v1
8 | - uses: actions/setup-python@v1
9 | with:
10 | python-version: '3.7'
11 | architecture: 'x64'
12 | - name: Install the library
13 | run: |
14 | pip install nbdev jupyter
15 | pip install -e .
16 | - name: Read all notebooks
17 | run: |
18 | nbdev_read_nbs
19 | - name: Check if all notebooks are cleaned
20 | run: |
21 | echo "Check we are starting with clean git checkout"
22 | if [ -n "$(git status -uno -s)" ]; then echo "git status is not clean"; false; fi
23 | echo "Trying to strip out notebooks"
24 | nbdev_clean_nbs
25 | echo "Check that strip out was unnecessary"
26 | git status -s # display the status to see which nbs need cleaning up
27 | if [ -n "$(git status -uno -s)" ]; then echo -e "!!! Detected unstripped out notebooks\n!!!Remember to run nbdev_install_git_hooks"; false; fi
28 | - name: Check if there is no diff library/notebooks
29 | run: |
30 | if [ -n "$(nbdev_diff_nbs)" ]; then echo -e "!!! Detected difference between the notebooks and the library"; false; fi
31 | - name: Run tests
32 | run: |
33 | nbdev_test_nbs
34 |
--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | repository: noklam/dtype_diet
2 | output: web
3 | topnav_title: dtype_diet
4 | site_title: dtype_diet
5 | company_name: Chan Nok Lam
6 | description: One step to take your dataset to go on a diet.
7 | # Set to false to disable KaTeX math
8 | use_math: true
9 | # Add Google analytics id if you have one and want to use it here
10 | google_analytics:
11 | # See http://nbdev.fast.ai/search for help with adding Search
12 | google_search:
13 |
14 | host: 127.0.0.1
15 | # the preview server used. Leave as is.
16 | port: 4000
17 | # the port where the preview is rendered.
18 |
19 | exclude:
20 | - .idea/
21 | - .gitignore
22 | - vendor
23 |
24 | exclude: [vendor]
25 |
26 | highlighter: rouge
27 | markdown: kramdown
28 | kramdown:
29 | input: GFM
30 | auto_ids: true
31 | hard_wrap: false
32 | syntax_highlighter: rouge
33 |
34 | collections:
35 | tooltips:
36 | output: false
37 |
38 | defaults:
39 | -
40 | scope:
41 | path: ""
42 | type: "pages"
43 | values:
44 | layout: "page"
45 | comments: true
46 | search: true
47 | sidebar: home_sidebar
48 | topnav: topnav
49 | -
50 | scope:
51 | path: ""
52 | type: "tooltips"
53 | values:
54 | layout: "page"
55 | comments: true
56 | search: true
57 | tooltip: true
58 |
59 | sidebars:
60 | - home_sidebar
61 |
62 | theme: jekyll-theme-cayman
63 | baseurl: /dtype_diet/
--------------------------------------------------------------------------------
/.github/workflows/python-publish.yml:
--------------------------------------------------------------------------------
1 | # This workflow will upload a Python Package using Twine when a release is created
2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
3 |
4 | # This workflow uses actions that are not certified by GitHub.
5 | # They are provided by a third-party and are governed by
6 | # separate terms of service, privacy policy, and support
7 | # documentation.
8 |
9 | name: Upload Python Package to PyPI
10 |
11 | on:
12 | release:
13 | types: [published]
14 |
15 | # Allows you to run this workflow manually from the Actions tab
16 | workflow_dispatch:
17 |
18 |
19 | permissions:
20 | contents: read
21 |
22 | jobs:
23 | deploy:
24 |
25 | runs-on: ubuntu-22.04
26 |
27 | steps:
28 | - uses: actions/checkout@v3
29 |
30 | - name: Set up Python
31 | uses: actions/setup-python@v3
32 | with:
33 | python-version: '3.12.0'
34 |
35 | - name: Install dependencies
36 | run: |
37 | python -m pip install --upgrade build
38 |
39 | - name: Build package
40 | run: python -m build
41 |
42 | - name: Publish package
43 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
44 | with:
45 | user: __token__
46 | password: ${{ secrets.PYPI_API_TOKEN }}
47 |
--------------------------------------------------------------------------------
/docs/feed.xml:
--------------------------------------------------------------------------------
1 | ---
2 | search: exclude
3 | layout: none
4 | ---
5 |
6 |
7 |
8 |
9 | {{ site.title | xml_escape }}
10 | {{ site.description | xml_escape }}
11 | {{ site.time | date_to_rfc822 }}
14 | {{ site.time | date_to_rfc822 }}
15 | Jekyll v{{ jekyll.version }}
16 | {% for post in site.posts limit:10 %}
17 | -
18 |
{{ post.title | xml_escape }}
19 | {{ post.content | xml_escape }}
20 | {{ post.date | date_to_rfc822 }}
21 | {{ post.url | prepend: site.url }}
23 | {% for tag in post.tags %}
24 | {{ tag | xml_escape }}
25 | {% endfor %}
26 | {% for tag in page.tags %}
27 | {{ cat | xml_escape }}
28 | {% endfor %}
29 |
30 | {% endfor %}
31 |
32 |
33 |
--------------------------------------------------------------------------------
/docs/_includes/links.html:
--------------------------------------------------------------------------------
1 | {% comment %}Get links from each sidebar, as listed in the _config.yml file under sidebars{% endcomment %}
2 |
3 | {% for sidebar in site.sidebars %}
4 | {% for entry in site.data.sidebars[sidebar].entries %}
5 | {% for folder in entry.folders %}
6 | {% for folderitem in folder.folderitems %}
7 | {% if folderitem.url contains "html#" %}
8 | [{{folderitem.url | remove: "/" }}]: {{folderitem.url | remove: "/"}}
9 | {% else %}
10 | [{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}}
11 | {% endif %}
12 | {% for subfolders in folderitem.subfolders %}
13 | {% for subfolderitem in subfolders.subfolderitems %}
14 | [{{subfolderitem.url | remove: "/" | remove: ".html"}}]: {{subfolderitem.url | remove: "/"}}
15 | {% endfor %}
16 | {% endfor %}
17 | {% endfor %}
18 | {% endfor %}
19 | {% endfor %}
20 | {% endfor %}
21 |
22 |
23 | {% comment %} Get links from topnav {% endcomment %}
24 |
25 | {% for entry in site.data.topnav.topnav %}
26 | {% for item in entry.items %}
27 | {% if item.external_url == null %}
28 | [{{item.url | remove: "/" | remove: ".html"}}]: {{item.url | remove: "/"}}
29 | {% endif %}
30 | {% endfor %}
31 | {% endfor %}
32 |
33 | {% comment %}Get links from topnav dropdowns {% endcomment %}
34 |
35 | {% for entry in site.data.topnav.topnav_dropdowns %}
36 | {% for folder in entry.folders %}
37 | {% for folderitem in folder.folderitems %}
38 | {% if folderitem.external_url == null %}
39 | [{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}}
40 | {% endif %}
41 | {% endfor %}
42 | {% endfor %}
43 | {% endfor %}
44 |
45 |
--------------------------------------------------------------------------------
/docs/_includes/head_print.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
{% if page.homepage == true %} {{site.homepage_title}} {% elsif page.title %}{{ page.title }}{% endif %} | {{ site.site_title }}
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
23 |
24 |
29 |
--------------------------------------------------------------------------------
/docs/css/modern-business.css:
--------------------------------------------------------------------------------
1 | /*!
2 | * Start Bootstrap - Modern Business HTML Template (http://startbootstrap.com)
3 | * Code licensed under the Apache License v2.0.
4 | * For details, see http://www.apache.org/licenses/LICENSE-2.0.
5 | */
6 |
7 | /* Global Styles */
8 |
9 | html,
10 | body {
11 | height: 100%;
12 | }
13 |
14 | .img-portfolio {
15 | margin-bottom: 30px;
16 | }
17 |
18 | .img-hover:hover {
19 | opacity: 0.8;
20 | }
21 |
22 | /* Home Page Carousel */
23 |
24 | header.carousel {
25 | height: 50%;
26 | }
27 |
28 | header.carousel .item,
29 | header.carousel .item.active,
30 | header.carousel .carousel-inner {
31 | height: 100%;
32 | }
33 |
34 | header.carousel .fill {
35 | width: 100%;
36 | height: 100%;
37 | background-position: center;
38 | background-size: cover;
39 | }
40 |
41 | /* 404 Page Styles */
42 |
43 | .error-404 {
44 | font-size: 100px;
45 | }
46 |
47 | /* Pricing Page Styles */
48 |
49 | .price {
50 | display: block;
51 | font-size: 50px;
52 | line-height: 50px;
53 | }
54 |
55 | .price sup {
56 | top: -20px;
57 | left: 2px;
58 | font-size: 20px;
59 | }
60 |
61 | .period {
62 | display: block;
63 | font-style: italic;
64 | }
65 |
66 | /* Footer Styles */
67 |
68 | footer {
69 | margin: 50px 0;
70 | }
71 |
72 | /* Responsive Styles */
73 |
74 | @media(max-width:991px) {
75 | .client-img,
76 | .img-related {
77 | margin-bottom: 30px;
78 | }
79 | }
80 |
81 | @media(max-width:767px) {
82 | .img-portfolio {
83 | margin-bottom: 15px;
84 | }
85 |
86 | header.carousel .carousel {
87 | height: 70%;
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/docs/licenses/LICENSE-BSD-NAVGOCO.txt:
--------------------------------------------------------------------------------
1 | /* This license pertains to the Navgoco jQuery component used for the sidebar. */
2 |
3 | Copyright (c) 2013, Christodoulos Tsoulloftas, http://www.komposta.net
4 | All rights reserved.
5 |
6 | Redistribution and use in source and binary forms, with or without modification,
7 | are permitted provided that the following conditions are met:
8 |
9 | * Redistributions of source code must retain the above copyright notice,
10 | this list of conditions and the following disclaimer.
11 | * Redistributions in binary form must reproduce the above copyright notice,
12 | this list of conditions and the following disclaimer in the documentation
13 | and/or other materials provided with the distribution.
14 | * Neither the name of the
nor the names of its
15 | contributors may be used to endorse or promote products derived from this
16 | software without specific prior written permission.
17 |
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
22 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
27 | OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from pkg_resources import parse_version
2 | from configparser import ConfigParser
3 | import setuptools
4 | assert parse_version(setuptools.__version__)>=parse_version('36.2')
5 |
6 | # note: all settings are in settings.ini; edit there, not here
7 | config = ConfigParser(delimiters=['='])
8 | config.read('settings.ini')
9 | cfg = config['DEFAULT']
10 |
11 | cfg_keys = 'version description keywords author author_email'.split()
12 | expected = cfg_keys + "lib_name user branch license status min_python audience language".split()
13 | for o in expected: assert o in cfg, "missing expected setting: {}".format(o)
14 | setup_cfg = {o:cfg[o] for o in cfg_keys}
15 |
16 | licenses = {
17 | 'MIT': ('MIT License', 'OSI Approved :: MIT License'),
18 | }
19 | statuses = [ '1 - Planning', '2 - Pre-Alpha', '3 - Alpha',
20 | '4 - Beta', '5 - Production/Stable', '6 - Mature', '7 - Inactive' ]
21 | py_versions = '3.6 3.7 3.8 3.9 3.10 3.11'.split()
22 |
23 | requirements = cfg.get('requirements','').split()
24 | lic = licenses[cfg['license']]
25 | min_python = cfg['min_python']
26 |
27 | setuptools.setup(
28 | name = cfg['lib_name'],
29 | license = lic[0],
30 | classifiers = [
31 | 'Development Status :: ' + statuses[int(cfg['status'])],
32 | 'Intended Audience :: ' + cfg['audience'].title(),
33 | 'License :: ' + lic[1],
34 | 'Natural Language :: ' + cfg['language'].title(),
35 | ] + ['Programming Language :: Python :: '+o for o in py_versions[py_versions.index(min_python):]],
36 | url = cfg['git_url'],
37 | packages = setuptools.find_packages(),
38 | include_package_data = True,
39 | install_requires = requirements,
40 | dependency_links = cfg.get('dep_links','').split(),
41 | python_requires = '>=' + cfg['min_python'],
42 | long_description = open('README.md').read(),
43 | long_description_content_type = 'text/markdown',
44 | zip_safe = False,
45 | entry_points = { 'console_scripts': cfg.get('console_scripts','').split() },
46 | **setup_cfg)
47 |
48 |
--------------------------------------------------------------------------------
/docs/_layouts/page.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 |
12 |
13 | {% if page.simple_map == true %}
14 |
15 |
20 |
21 | {% include custom/{{page.map_name}}.html %}
22 |
23 | {% elsif page.complex_map == true %}
24 |
25 |
30 |
31 | {% include custom/{{page.map_name}}.html %}
32 |
33 | {% endif %}
34 |
35 |
36 |
37 | {% if page.summary %}
38 |
{{page.summary}}
39 | {% endif %}
40 |
41 | {% unless page.toc == false %}
42 | {% include toc.html %}
43 | {% endunless %}
44 |
45 |
46 | {% if site.github_editme_path %}
47 |
48 |
49 |
50 | {% endif %}
51 |
52 | {{content}}
53 |
54 |
65 |
66 |
Open in Colab Open in Colab
3 |
4 |
13 |
14 |
15 |
16 | {{item.title}} {{item.title}} {{item.title}}
38 | {{ folder.title }}
39 |
50 |
51 | {% endfor %}
52 | {% endfor %}
53 | {% if site.google_search %}
54 |
55 | {% include search_google_custom.html %}
56 |
57 | {% endif %}
58 |
59 |
60 |
63 |
--------------------------------------------------------------------------------
/docs/js/jquery.navgoco.min.js:
--------------------------------------------------------------------------------
1 | /*
2 | * jQuery Navgoco Menus Plugin v0.2.1 (2014-04-11)
3 | * https://github.com/tefra/navgoco
4 | *
5 | * Copyright (c) 2014 Chris T (@tefra)
6 | * BSD - https://github.com/tefra/navgoco/blob/master/LICENSE-BSD
7 | */
8 | !function(a){"use strict";var b=function(b,c,d){return this.el=b,this.$el=a(b),this.options=c,this.uuid=this.$el.attr("id")?this.$el.attr("id"):d,this.state={},this.init(),this};b.prototype={init:function(){var b=this;b._load(),b.$el.find("ul").each(function(c){var d=a(this);d.attr("data-index",c),b.options.save&&b.state.hasOwnProperty(c)?(d.parent().addClass(b.options.openClass),d.show()):d.parent().hasClass(b.options.openClass)?(d.show(),b.state[c]=1):d.hide()});var c=a("" + txt + " ";
66 | else if (this_level <= level){ // higher level than before; end parent ol
67 | for(i = this_level; i < level; i++) {
68 | html += "" + txt + " ";
71 | }
72 | else if (this_level > level) { // lower level than before; expand the previous to contain a ol
73 | for(i = this_level; i > level; i--) {
74 | html += "<"+settings.listType+">"+((i-level == 2) ? "" : " ")
75 | }
76 | html += "" + txt + " ";
77 | }
78 | level = this_level; // update for the next one
79 | });
80 | html += "";
81 | if (!settings.noBackToTopLinks) {
82 | $(document).on('click', '.back-to-top', function() {
83 | $(window).scrollTop(0);
84 | window.location.hash = '';
85 | });
86 | }
87 |
88 | render[settings.showEffect]();
89 | };
90 | })(jQuery);
91 |
--------------------------------------------------------------------------------
/docs/_layouts/default.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | {% include head.html %}
5 |
41 |
46 |
57 | {% if page.datatable == true %}
58 |
59 |
83 |
84 |
85 |
86 | {% assign content_col_size = "col-md-12" %}
87 | {% unless page.hide_sidebar %}
88 |
89 |
92 | {% assign content_col_size = "col-md-9" %}
93 | {% endunless %}
94 |
95 |
96 |
97 | {{content}}
98 |
99 |
100 |
101 |
102 |
103 |
104 |
{{ page.title }} | {{ site.site_title }}
7 | =0}var self=this;self.matches=function(string,crit){return"string"!=typeof string?!1:(string=string.trim(),doMatch(string,crit))}}module.exports=new LiteralSearchStrategy},{}],4:[function(require,module){module.exports=function(){function findMatches(store,crit,strategy){for(var data=store.get(),i=0;i{title} ',noResultsText:"No results found",limit:10,fuzzy:!1};self.init=function(_opt){validateOptions(_opt),assignOptions(_opt),isJSON(opt.dataSource)?initWithJSON(opt.dataSource):initWithURL(opt.dataSource)}}var Searcher=require("./Searcher"),Templater=require("./Templater"),Store=require("./Store"),JSONLoader=require("./JSONLoader"),searcher=new Searcher,templater=new Templater,store=new Store,jsonLoader=new JSONLoader;window.SimpleJekyllSearch=new SimpleJekyllSearch}(window,document)},{"./JSONLoader":1,"./Searcher":4,"./Store":5,"./Templater":6}]},{},[7]);
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # dtype_diet
2 | > Attempt to shrink Pandas `dtypes` without losing data so you have more RAM (and maybe more speed)
3 |
4 |
5 | This file will become your README and also the index of your documentation.
6 |
7 | ## Install
8 |
9 | `pip install dtype_diet`
10 |
11 | # Documentation
12 | https://noklam.github.io/dtype_diet/
13 |
14 | ## How to use
15 |
16 | > This is a fork of https://github.com/ianozsvald/dtype_diet to continue supoprt and develop the library with approval from the original author @ianozsvald.
17 |
18 | This tool checks each column to see if larger dtypes (e.g. 8 byte `float64` and `int64`) could be shrunk to smaller `dtypes` without causing any data loss.
19 | Dropping an 8 byte type to a 4 (or 2 or 1 byte) type will keep halving the RAM requirement for that column. Categoricals are proposed for `object` columns which can bring significant speed and RAM benefits.
20 |
21 |
22 | Here's an minimal example with 3 lines of code running on a Kaggle dataset showing a reduction of 957 -> 85MB, you can find the notebook in the [repository](https://github.com/noklam/dtype_diet/01_example.ipynb):
23 |
24 | ```python
25 | #slow
26 | # sell_prices.csv.zip
27 | # Source data: https://www.kaggle.com/c/m5-forecasting-uncertainty/
28 | import pandas as pd
29 | from dtype_diet import report_on_dataframe, optimize_dtypes
30 | df = pd.read_csv('data/sell_prices.csv')
31 | proposed_df = report_on_dataframe(df, unit="MB")
32 | new_df = optimize_dtypes(df, proposed_df)
33 | print(f'Original df memory: {df.memory_usage(deep=True).sum()/1024/1024} MB')
34 | print(f'Propsed df memory: {new_df.memory_usage(deep=True).sum()/1024/1024} MB')
35 | ```
36 |
37 | Original df memory: 957.5197134017944 MB
38 | Propsed df memory: 85.09655094146729 MB
39 |
40 |
41 | ```python
42 | #slow
43 | proposed_df
44 | ```
45 |
46 |
47 |
48 |
49 |
50 |
63 |
64 |
65 |
66 | Current dtype
68 | Proposed dtype
69 | Current Memory (MB)
70 | Proposed Memory (MB)
71 | Ram Usage Improvement (MB)
72 | Ram Usage Improvement (%)
73 |
74 |
75 | Column
76 |
83 |
84 |
85 |
86 | store_id
87 | object
88 | category
89 | 203763.920410
90 | 3340.907715
91 | 200423.012695
92 | 98.360403
93 |
94 |
95 | item_id
96 | object
97 | category
98 | 233039.977539
99 | 6824.677734
100 | 226215.299805
101 | 97.071456
102 |
103 |
104 | wm_yr_wk
105 | int64
106 | int16
107 | 26723.191406
108 | 6680.844727
109 | 20042.346680
110 | 74.999825
111 |
112 |
113 | sell_price
114 | float64
115 | None
116 | 26723.191406
117 | NaN
118 | NaN
119 | NaN
120 |
121 |
122 |
123 |
\n",
45 | "RangeIndex: 6841121 entries, 0 to 6841120\n",
46 | "Data columns (total 4 columns):\n",
47 | " # Column Dtype \n",
48 | "--- ------ ----- \n",
49 | " 0 store_id object \n",
50 | " 1 item_id object \n",
51 | " 2 wm_yr_wk int64 \n",
52 | " 3 sell_price float64\n",
53 | "dtypes: float64(1), int64(1), object(2)\n",
54 | "memory usage: 957.5 MB\n"
55 | ]
56 | }
57 | ],
58 | "source": [
59 | "#slow\n",
60 | "# sell_prices.csv.zip \n",
61 | "# Source data: https://www.kaggle.com/c/m5-forecasting-uncertainty/\n",
62 | "df = pd.read_csv('data/sell_prices.csv')\n",
63 | "df.info(memory_usage='deep')"
64 | ]
65 | },
66 | {
67 | "cell_type": "code",
68 | "execution_count": null,
69 | "metadata": {},
70 | "outputs": [],
71 | "source": [
72 | "#slow\n",
73 | "proposed_df = report_on_dataframe(df, unit=\"MB\")"
74 | ]
75 | },
76 | {
77 | "cell_type": "code",
78 | "execution_count": null,
79 | "metadata": {},
80 | "outputs": [
81 | {
82 | "data": {
83 | "text/html": [
84 | "\n",
85 | "\n",
98 | "
\n",
99 | " \n",
100 | " \n",
101 | " Current dtype \n",
103 | " Proposed dtype \n",
104 | " Current Memory (MB) \n",
105 | " Proposed Memory (MB) \n",
106 | " Ram Usage Improvement (MB) \n",
107 | " Ram Usage Improvement (%) \n",
108 | " \n",
109 | " \n",
110 | " Column \n",
111 | " \n",
118 | " \n",
119 | " \n",
120 | " \n",
121 | " store_id \n",
122 | " object \n",
123 | " category \n",
124 | " 203763.920410 \n",
125 | " 3340.907715 \n",
126 | " 200423.012695 \n",
127 | " 98.360403 \n",
128 | " \n",
129 | " \n",
130 | " item_id \n",
131 | " object \n",
132 | " category \n",
133 | " 233039.977539 \n",
134 | " 6824.677734 \n",
135 | " 226215.299805 \n",
136 | " 97.071456 \n",
137 | " \n",
138 | " \n",
139 | " wm_yr_wk \n",
140 | " int64 \n",
141 | " int16 \n",
142 | " 26723.191406 \n",
143 | " 6680.844727 \n",
144 | " 20042.346680 \n",
145 | " 74.999825 \n",
146 | " \n",
147 | " \n",
148 | " sell_price \n",
149 | " float64 \n",
150 | " None \n",
151 | " 26723.191406 \n",
152 | " NaN \n",
153 | " NaN \n",
154 | " NaN \n",
155 | " \n",
156 | " \n",
157 | "
\n",
158 | "
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 | {% raw %}
33 |
34 |
35 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
<class 'pandas.core.frame.DataFrame'>
56 | RangeIndex: 6841121 entries, 0 to 6841120
57 | Data columns (total 4 columns):
58 | # Column Dtype
59 | --- ------ -----
60 | 0 store_id object
61 | 1 item_id object
62 | 2 wm_yr_wk int64
63 | 3 sell_price float64
64 | dtypes: float64(1), int64(1), object(2)
65 | memory usage: 957.5 MB
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 | {% endraw %}
75 |
76 | {% raw %}
77 |
78 |
91 | {% endraw %}
92 |
93 | {% raw %}
94 |
95 |
96 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
128 |
129 |
130 |
131 | Current dtype
133 | Proposed dtype
134 | Current Memory (MB)
135 | Proposed Memory (MB)
136 | Ram Usage Improvement (MB)
137 | Ram Usage Improvement (%)
138 |
139 |
140 | Column
141 |
148 |
149 |
150 |
151 | store_id
152 | object
153 | category
154 | 203763.920410
155 | 3340.907715
156 | 200423.012695
157 | 98.360403
158 |
159 |
160 | item_id
161 | object
162 | category
163 | 233039.977539
164 | 6824.677734
165 | 226215.299805
166 | 97.071456
167 |
168 |
169 | wm_yr_wk
170 | int64
171 | int16
172 | 26723.191406
173 | 6680.844727
174 | 20042.346680
175 | 74.999825
176 |
177 |
178 | sell_price
179 | float64
180 | None
181 | 26723.191406
182 | NaN
183 | NaN
184 | NaN
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 | {% endraw %}
198 |
199 |
200 |
201 |
It shows potential dtypes for conversion, you should review if it will cause overflow issue in the future and modify accordingly if needed.
202 |
203 |
204 |
205 |
206 | {% raw %}
207 |
208 |
221 | {% endraw %}
222 |
223 |
224 |
225 |
optimize_dtypes take your df and the proposed_df as an argument to convert the dataframe to the proposed dtypes.
226 |
227 |
228 |
229 |
230 | {% raw %}
231 |
232 |
233 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
Original df memory: 957.5197134017944 MB
252 | Propsed df memory: 85.09655094146729 MB
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 | {% endraw %}
262 |
263 |
\n",
108 | "\n",
121 | "
\n",
122 | " \n",
123 | " \n",
124 | " Current dtype \n",
126 | " Proposed dtype \n",
127 | " Current Memory (MB) \n",
128 | " Proposed Memory (MB) \n",
129 | " Ram Usage Improvement (MB) \n",
130 | " Ram Usage Improvement (%) \n",
131 | " \n",
132 | " \n",
133 | " Column \n",
134 | " \n",
141 | " \n",
142 | " \n",
143 | " \n",
144 | " store_id \n",
145 | " object \n",
146 | " category \n",
147 | " 203763.920410 \n",
148 | " 3340.907715 \n",
149 | " 200423.012695 \n",
150 | " 98.360403 \n",
151 | " \n",
152 | " \n",
153 | " item_id \n",
154 | " object \n",
155 | " category \n",
156 | " 233039.977539 \n",
157 | " 6824.677734 \n",
158 | " 226215.299805 \n",
159 | " 97.071456 \n",
160 | " \n",
161 | " \n",
162 | " wm_yr_wk \n",
163 | " int64 \n",
164 | " int16 \n",
165 | " 26723.191406 \n",
166 | " 6680.844727 \n",
167 | " 20042.346680 \n",
168 | " 74.999825 \n",
169 | " \n",
170 | " \n",
171 | " sell_price \n",
172 | " float64 \n",
173 | " None \n",
174 | " 26723.191406 \n",
175 | " NaN \n",
176 | " NaN \n",
177 | " NaN \n",
178 | " \n",
179 | " \n",
180 | "
\n",
181 | "
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 |
33 |
34 |
This file will become your README and also the index of your documentation.
35 |
36 |
37 |
38 |
39 |
45 |
46 |
47 |
pip install dtype_diet
48 |
49 |
50 |
51 |
52 |
59 |
65 |
66 |
67 |
This is a fork of https://github.com/ianozsvald/dtype_diet to continue supoprt and develop the library with approval from the original author @ianozsvald.
68 |
69 |
This tool checks each column to see if larger dtypes (e.g. 8 byte float64 and int64) could be shrunk to smaller dtypes without causing any data loss.
70 | Dropping an 8 byte type to a 4 (or 2 or 1 byte) type will keep halving the RAM requirement for that column. Categoricals are proposed for object columns which can bring significant speed and RAM benefits.
71 |
Here's an minimal example with 3 lines of code running on a Kaggle dataset showing a reduction of 957 -> 85MB, you can find the notebook in the repository :
72 |
73 |
74 |
75 |
76 | {% raw %}
77 |
78 |
79 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
Original df memory: 957.5197134017944 MB
105 | Propsed df memory: 85.09655094146729 MB
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 | {% endraw %}
115 |
116 | {% raw %}
117 |
118 |
119 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
151 |
152 |
153 |
154 | Current dtype
156 | Proposed dtype
157 | Current Memory (MB)
158 | Proposed Memory (MB)
159 | Ram Usage Improvement (MB)
160 | Ram Usage Improvement (%)
161 |
162 |
163 | Column
164 |
171 |
172 |
173 |
174 | store_id
175 | object
176 | category
177 | 203763.920410
178 | 3340.907715
179 | 200423.012695
180 | 98.360403
181 |
182 |
183 | item_id
184 | object
185 | category
186 | 233039.977539
187 | 6824.677734
188 | 226215.299805
189 | 97.071456
190 |
191 |
192 | wm_yr_wk
193 | int64
194 | int16
195 | 26723.191406
196 | 6680.844727
197 | 20042.346680
198 | 74.999825
199 |
200 |
201 | sell_price
202 | float64
203 | None
204 | 26723.191406
205 | NaN
206 | NaN
207 | NaN
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 | {% endraw %}
221 |
222 |
223 |
224 |
Recommendations:
225 |
226 | Run report_on_dataframe(your_df) to get recommendations
227 | Run optimize_dtypes(df, proposed_df) to convert to recommeded dtypes.
228 | Consider if Categoricals will save you RAM (see Caveats below)
229 | Consider if f32 or f16 will be useful (see Caveats - f32 is probably a reasonable choice unless you have huge ranges of floats)
230 | Consider if int32, int16, int8 will be useful (see Caveats - overflow may be an issue)
231 | Look at https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.convert_dtypes.html which recommends Pandas nullable dtype alternatives (e.g. to avoid promoting an int64 with NaN items to float64, instead you get Int64 with NaNs and no data loss)
232 | Look at Extension arrays like https://github.com/JDASoftwareGroup/rle-array (thanks @repererum for the tweet )
233 |
234 |
Look at report_on_dataframe(your_df) to get a printed report - no changes are made to your dataframe.
235 |
236 |
237 |
238 |
239 |
240 |
241 |
Caveats
242 | reduced numeric ranges might lead to overflow (TODO document)
243 | category dtype can have unexpected effects e.g. need for observed=True in groupby (TODO document)
244 | f16 is likely to be simulated on modern hardware so calculations will be 2-3* slower than on f32 or f64
245 | we could do with a link that explains binary representation of float & int for those wanting to learn more
246 |
247 |
Development Contributors
251 |
Local Setup
252 |
$ conda create -n dtype_diet python=3.8 pandas jupyter pyarrow pytest
253 | $ conda activate dtype_diet
254 |
Release
255 |
make release
256 |
Contributing The repository is developed with nbdev, a system for developing library with notebook.
257 |
Make sure you run this if you want to contribute to the library. For details, please refer to nbdev documentation (https://github.com/fastai/nbdev )
258 |
259 |
nbdev_install_git_hooks
260 |
Some other useful commands
261 |
262 |
nbdev_build_docs
263 | nbdev_build_lib
264 | nbdev_test_nbs
265 |
266 |
267 |
268 |
269 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 | {% raw %}
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
count_errors(serSeries, new_dtype
44 |
45 |
After converting ser to new dtype, count whether items have isclose()
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 | {% endraw %}
56 |
57 | {% raw %}
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
map_dtypes_to_choices(serSeries, optimizestr)
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 | {% endraw %}
80 |
81 | {% raw %}
82 |
83 |
84 |
85 |
86 | {% endraw %}
87 |
88 | {% raw %}
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
get_smallest_valid_conversion(serSeries, optimizestr)
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 | {% endraw %}
111 |
112 | {% raw %}
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
get_improvement(as_typeAsTypecurrent_nbytesint)
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 | {% endraw %}
135 |
136 | {% raw %}
137 |
138 |
139 |
140 |
141 | {% endraw %}
142 |
143 | {% raw %}
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
report_on_dataframe(dfDataFrame, unitstr='MB'optimizestr='memory'
155 |
156 |
[Report on columns that might be converted]
157 | Args:
158 | df ([type]): [description]
159 | unit (str, optional): [byte, MB, GB]. Defaults to "MB".
160 | optimize (str, optional): [memory, computation]. Defaults to memory.
161 | [memory]: The lowest memory dtype for float is fp16.
162 | [computation]: The lowest memory dtype for float is fp32.
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 | {% endraw %}
173 |
174 | {% raw %}
175 |
176 |
177 |
178 |
179 | {% endraw %}
180 |
181 | {% raw %}
182 |
183 |
199 | {% endraw %}
200 |
201 | {% raw %}
202 |
203 |
204 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
236 |
237 |
238 |
239 | Current dtype
241 | Proposed dtype
242 | Current Memory (MB)
243 | Proposed Memory (MB)
244 | Ram Usage Improvement (MB)
245 | Ram Usage Improvement (%)
246 |
247 |
248 | Column
249 |
256 |
257 |
258 |
259 | store_id
260 | object
261 | category
262 | 203763.920410
263 | 3340.907715
264 | 200423.012695
265 | 98.360403
266 |
267 |
268 | item_id
269 | object
270 | category
271 | 233039.977539
272 | 6824.677734
273 | 226215.299805
274 | 97.071456
275 |
276 |
277 | wm_yr_wk
278 | int64
279 | int16
280 | 26723.191406
281 | 6680.844727
282 | 20042.346680
283 | 74.999825
284 |
285 |
286 | sell_price
287 | float64
288 | None
289 | 26723.191406
290 | NaN
291 | NaN
292 | NaN
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 | {% endraw %}
306 |
307 |
308 |
309 |
report_on_dataframecurrent values of the data, you should still be careful about overflow for further transformation.
310 |
311 |
312 |
313 |
314 | {% raw %}
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
optimize_dtypes(dfDataFrame, proposed_dfDataFrame)
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 | {% endraw %}
337 |
338 | {% raw %}
339 |
340 |
341 |
342 |
343 | {% endraw %}
344 |
345 | {% raw %}
346 |
347 |
348 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
Given a dataframe, check for lowest possible conversions:
381 | convert_dtypes does a slightly different job:
382 | a b c d e str_a str_b
383 | 0 0 256 65536 1100 100101 hello 0
384 | 1 0 256 65536 1100 100101 hello 1
385 | 2 0 256 65536 1100 100101 hello 2
386 | 3 0 256 65536 1100 100101 hello 3
387 | 4 0 256 65536 1100 100101 hello 4
388 | .. .. ... ... ... ... ... ...
389 | 95 0 256 65536 1100 100101 hello 95
390 | 96 0 256 65536 1100 100101 hello 96
391 | 97 0 256 65536 1100 100101 hello 97
392 | 98 0 256 65536 1100 100101 hello 98
393 | 99 0 256 65536 1100 100101 hello 99
394 |
395 | [100 rows x 7 columns]
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 | {% endraw %}
405 |
406 |
\n",
218 | "\n",
231 | "
\n",
232 | " \n",
233 | " \n",
234 | " Current dtype \n",
236 | " Proposed dtype \n",
237 | " Current Memory (MB) \n",
238 | " Proposed Memory (MB) \n",
239 | " Ram Usage Improvement (MB) \n",
240 | " Ram Usage Improvement (%) \n",
241 | " \n",
242 | " \n",
243 | " Column \n",
244 | " \n",
251 | " \n",
252 | " \n",
253 | " \n",
254 | " store_id \n",
255 | " object \n",
256 | " category \n",
257 | " 203763.920410 \n",
258 | " 3340.907715 \n",
259 | " 200423.012695 \n",
260 | " 98.360403 \n",
261 | " \n",
262 | " \n",
263 | " item_id \n",
264 | " object \n",
265 | " category \n",
266 | " 233039.977539 \n",
267 | " 6824.677734 \n",
268 | " 226215.299805 \n",
269 | " 97.071456 \n",
270 | " \n",
271 | " \n",
272 | " wm_yr_wk \n",
273 | " int64 \n",
274 | " int16 \n",
275 | " 26723.191406 \n",
276 | " 6680.844727 \n",
277 | " 20042.346680 \n",
278 | " 74.999825 \n",
279 | " \n",
280 | " \n",
281 | " sell_price \n",
282 | " float64 \n",
283 | " None \n",
284 | " 26723.191406 \n",
285 | " NaN \n",
286 | " NaN \n",
287 | " NaN \n",
288 | " \n",
289 | " \n",
290 | "
\n",
291 | "