'
13 |
14 | hr_faded: '
'
15 | hr_shaded: '
'
--------------------------------------------------------------------------------
/archive_nbs/single_profile.py:
--------------------------------------------------------------------------------
1 | import torch_xla.debug.metrics as met
2 |
3 | import fastai_xla_extensions.core
4 | from fastai2.vision.all import *
5 | from my_timesaver_utils.profiling import *
6 | path = untar_data(URLs.PETS)/'images'
7 | Path.BASE_PATH = path; path.ls()
8 | print(f'running on default_device() & cuda is {torch.cuda.is_available()}')
9 |
10 | img = PILImage.create(path/'Abyssinian_1.jpg')
11 | resize = Resize(size=200)
12 | img2 = resize(img,split_idx=0)
13 |
14 |
15 |
16 |
17 | timg2 = TensorImage(array(img2)).permute(2,0,1).float()/255.
18 |
19 | def batch_ex(bs, device): return TensorImage(timg2[None].to(device).expand(bs, *timg2.shape))
20 |
21 |
22 | b768_img = batch_ex(768, default_device()); (b768_img.shape, b768_img.device)
23 |
24 |
25 | flip_tfm = Flip(p=1.0)
26 | # run without profile
27 | run_with_profile = True
28 | F.grid_sample = profile_call(F.grid_sample) if run_with_profile else F.grid_sample
29 |
30 | @profile_call
31 | def mtest(b_img):
32 | #set_trace()
33 | new_b_img = flip_tfm(b_img)
34 | return new_b_img
35 |
36 | clear_prof_data()
37 | print("--- 10 image tensor loops:")
38 | for i in range(10):
39 | print("--- ---------------------------------")
40 | new_b768_img = mtest(b768_img)
41 | print("--- ")
42 | print_prof_data()
43 |
44 | print(met.metrics_report())
45 |
--------------------------------------------------------------------------------
/docs/licenses/LICENSE:
--------------------------------------------------------------------------------
1 | /* This license pertains to the docs template, except for the Navgoco jQuery component. */
2 |
3 | The MIT License (MIT)
4 |
5 | Original theme: Copyright (c) 2016 Tom Johnson
6 | Modifications: Copyright (c) 2017 onwards fast.ai, Inc
7 |
8 | Permission is hereby granted, free of charge, to any person obtaining a copy
9 | of this software and associated documentation files (the "Software"), to deal
10 | in the Software without restriction, including without limitation the rights
11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 | copies of the Software, and to permit persons to whom the Software is
13 | furnished to do so, subject to the following conditions:
14 |
15 | The above copyright notice and this permission notice shall be included in all
16 | copies or substantial portions of the Software.
17 |
18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 | SOFTWARE.
25 |
--------------------------------------------------------------------------------
/docs/feed.xml:
--------------------------------------------------------------------------------
1 | ---
2 | search: exclude
3 | layout: none
4 | ---
5 |
6 |
7 |
8 |
9 | {{ site.title | xml_escape }}
10 | {{ site.description | xml_escape }}
11 | {{ site.time | date_to_rfc822 }}
14 | {{ site.time | date_to_rfc822 }}
15 | Jekyll v{{ jekyll.version }}
16 | {% for post in site.posts limit:10 %}
17 | -
18 |
{{ post.title | xml_escape }}
19 | {{ post.content | xml_escape }}
20 | {{ post.date | date_to_rfc822 }}
21 | {{ post.url | prepend: site.url }}
23 | {% for tag in post.tags %}
24 | {{ tag | xml_escape }}
25 | {% endfor %}
26 | {% for tag in page.tags %}
27 | {{ cat | xml_escape }}
28 | {% endfor %}
29 |
30 | {% endfor %}
31 |
32 |
33 |
--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | repository: butchland/fastai_xla_extensions
2 | output: web
3 | topnav_title: fastai_xla_extensions
4 | site_title: fastai_xla_extensions
5 | company_name: Butch Landingin
6 | description: A library to extend fastai to run on TPUs using pytorch-xla
7 | # Set to false to disable KaTeX math
8 | use_math: true
9 | # Add Google analytics id if you have one and want to use it here
10 | google_analytics:
11 | # See http://nbdev.fast.ai/search for help with adding Search
12 | google_search:
13 |
14 | host: 127.0.0.1
15 | # the preview server used. Leave as is.
16 | port: 4000
17 | # the port where the preview is rendered.
18 |
19 | exclude:
20 | - .idea/
21 | - .gitignore
22 | - vendor
23 |
24 | exclude: [vendor]
25 |
26 | highlighter: rouge
27 | markdown: kramdown
28 | kramdown:
29 | input: GFM
30 | auto_ids: true
31 | hard_wrap: false
32 | syntax_highlighter: rouge
33 |
34 | collections:
35 | tooltips:
36 | output: false
37 |
38 | defaults:
39 | -
40 | scope:
41 | path: ""
42 | type: "pages"
43 | values:
44 | layout: "page"
45 | comments: true
46 | search: true
47 | sidebar: home_sidebar
48 | topnav: topnav
49 | -
50 | scope:
51 | path: ""
52 | type: "tooltips"
53 | values:
54 | layout: "page"
55 | comments: true
56 | search: true
57 | tooltip: true
58 |
59 | sidebars:
60 | - home_sidebar
61 | permalink: pretty
62 |
63 | theme: jekyll-theme-cayman
64 | baseurl: /fastai_xla_extensions/
--------------------------------------------------------------------------------
/docs/_data/sidebars/home_sidebar.yml:
--------------------------------------------------------------------------------
1 |
2 | #################################################
3 | ### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
4 | #################################################
5 | # Instead edit ../../sidebar.json
6 | entries:
7 | - folders:
8 | - folderitems:
9 | - output: web,pdf
10 | title: Overview
11 | url: /
12 | - output: web,pdf
13 | title: Core XLA extensions
14 | url: core
15 | - output: web,pdf
16 | title: Utils
17 | url: utils
18 | - output: web,pdf
19 | title: CIFAR Loader
20 | url: cifar_loader
21 | - output: web,pdf
22 | title: Miscellaneous Utilities
23 | url: misc_utils
24 | - output: web,pdf
25 | title: 'Multi Core XLA Base '
26 | url: multi_core.base
27 | - output: web,pdf
28 | title: Torch Compatible Utilities
29 | url: multi_core.torch_compat
30 | - output: web,pdf
31 | title: Multi Core XLA Learner extensions
32 | url: multi_core.learner
33 | - output: web,pdf
34 | title: Multi Core Callback XLA Extensions
35 | url: multi_core.callback
36 | - output: web,pdf
37 | title: Multi Core LR Find XLA Extensions
38 | url: multi_core.lr_find
39 | - output: web,pdf
40 | title: 'Multi Core XLA Inference '
41 | url: multi_core.inference
42 | - output: web,pdf
43 | title: Development Setup
44 | url: dev_setup
45 | output: web
46 | title: fastai_xla_extensions
47 | output: web
48 | title: Sidebar
49 |
--------------------------------------------------------------------------------
/fastai_xla_extensions/utils.py:
--------------------------------------------------------------------------------
1 | # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/01_utils.ipynb (unless otherwise specified).
2 |
3 | __all__ = ['xla_imported', 'print_aten_ops']
4 |
5 | # Internal Cell
6 | try:
7 | import torch_xla
8 | except ImportError:
9 | pass
10 |
11 | # Cell
12 | import sys
13 |
14 | def xla_imported():
15 | "Check whether the `torch_xla` module has been successfully imported"
16 | return 'torch_xla' in sys.modules
17 |
18 | # Internal Cell
19 | if xla_imported():
20 | import torch_xla.debug.metrics as met
21 |
22 | # Cell
23 | def print_aten_ops():
24 | "print out xla aten operations (from xla debug metrics report `torch_xla.debug.metrics`)"
25 | # import torch_xla.debug.metrics as met
26 | from io import StringIO
27 | import sys
28 |
29 | class Capturing(list):
30 | def __enter__(self):
31 | self._stdout = sys.stdout
32 | sys.stdout = self._stringio = StringIO()
33 | return self
34 | def __exit__(self, *args):
35 | self.extend(self._stringio.getvalue().splitlines())
36 | del self._stringio # free up some memory
37 | sys.stdout = self._stdout
38 |
39 | out = met.metrics_report()
40 | if out.find("aten::"):
41 | print_now=False
42 | lines = out.split("\n")
43 | for l in lines:
44 | if print_now:
45 | print_now=False
46 | print(l)
47 | if l.find("aten::")>-1:
48 | print("needs lowering:", l)
49 | print_now=True
--------------------------------------------------------------------------------
/docs/_includes/links.html:
--------------------------------------------------------------------------------
1 | {% comment %}Get links from each sidebar, as listed in the _config.yml file under sidebars{% endcomment %}
2 |
3 | {% for sidebar in site.sidebars %}
4 | {% for entry in site.data.sidebars[sidebar].entries %}
5 | {% for folder in entry.folders %}
6 | {% for folderitem in folder.folderitems %}
7 | {% if folderitem.url contains "html#" %}
8 | [{{folderitem.url | remove: "/" }}]: {{folderitem.url | remove: "/"}}
9 | {% else %}
10 | [{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}}
11 | {% endif %}
12 | {% for subfolders in folderitem.subfolders %}
13 | {% for subfolderitem in subfolders.subfolderitems %}
14 | [{{subfolderitem.url | remove: "/" | remove: ".html"}}]: {{subfolderitem.url | remove: "/"}}
15 | {% endfor %}
16 | {% endfor %}
17 | {% endfor %}
18 | {% endfor %}
19 | {% endfor %}
20 | {% endfor %}
21 |
22 |
23 | {% comment %} Get links from topnav {% endcomment %}
24 |
25 | {% for entry in site.data.topnav.topnav %}
26 | {% for item in entry.items %}
27 | {% if item.external_url == null %}
28 | [{{item.url | remove: "/" | remove: ".html"}}]: {{item.url | remove: "/"}}
29 | {% endif %}
30 | {% endfor %}
31 | {% endfor %}
32 |
33 | {% comment %}Get links from topnav dropdowns {% endcomment %}
34 |
35 | {% for entry in site.data.topnav.topnav_dropdowns %}
36 | {% for folder in entry.folders %}
37 | {% for folderitem in folder.folderitems %}
38 | {% if folderitem.external_url == null %}
39 | [{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}}
40 | {% endif %}
41 | {% endfor %}
42 | {% endfor %}
43 | {% endfor %}
44 |
45 |
--------------------------------------------------------------------------------
/docs/css/modern-business.css:
--------------------------------------------------------------------------------
1 | /*!
2 | * Start Bootstrap - Modern Business HTML Template (http://startbootstrap.com)
3 | * Code licensed under the Apache License v2.0.
4 | * For details, see http://www.apache.org/licenses/LICENSE-2.0.
5 | */
6 |
7 | /* Global Styles */
8 |
9 | html,
10 | body {
11 | height: 100%;
12 | }
13 |
14 | .img-portfolio {
15 | margin-bottom: 30px;
16 | }
17 |
18 | .img-hover:hover {
19 | opacity: 0.8;
20 | }
21 |
22 | /* Home Page Carousel */
23 |
24 | header.carousel {
25 | height: 50%;
26 | }
27 |
28 | header.carousel .item,
29 | header.carousel .item.active,
30 | header.carousel .carousel-inner {
31 | height: 100%;
32 | }
33 |
34 | header.carousel .fill {
35 | width: 100%;
36 | height: 100%;
37 | background-position: center;
38 | background-size: cover;
39 | }
40 |
41 | /* 404 Page Styles */
42 |
43 | .error-404 {
44 | font-size: 100px;
45 | }
46 |
47 | /* Pricing Page Styles */
48 |
49 | .price {
50 | display: block;
51 | font-size: 50px;
52 | line-height: 50px;
53 | }
54 |
55 | .price sup {
56 | top: -20px;
57 | left: 2px;
58 | font-size: 20px;
59 | }
60 |
61 | .period {
62 | display: block;
63 | font-style: italic;
64 | }
65 |
66 | /* Footer Styles */
67 |
68 | footer {
69 | margin: 50px 0;
70 | }
71 |
72 | /* Responsive Styles */
73 |
74 | @media(max-width:991px) {
75 | .client-img,
76 | .img-related {
77 | margin-bottom: 30px;
78 | }
79 | }
80 |
81 | @media(max-width:767px) {
82 | .img-portfolio {
83 | margin-bottom: 15px;
84 | }
85 |
86 | header.carousel .carousel {
87 | height: 70%;
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/docs/_includes/head_print.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
{% if page.homepage == true %} {{site.homepage_title}} {% elsif page.title %}{{ page.title }}{% endif %} | {{ site.site_title }}
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
23 |
24 |
29 |
--------------------------------------------------------------------------------
/docs/licenses/LICENSE-BSD-NAVGOCO.txt:
--------------------------------------------------------------------------------
1 | /* This license pertains to the Navgoco jQuery component used for the sidebar. */
2 |
3 | Copyright (c) 2013, Christodoulos Tsoulloftas, http://www.komposta.net
4 | All rights reserved.
5 |
6 | Redistribution and use in source and binary forms, with or without modification,
7 | are permitted provided that the following conditions are met:
8 |
9 | * Redistributions of source code must retain the above copyright notice,
10 | this list of conditions and the following disclaimer.
11 | * Redistributions in binary form must reproduce the above copyright notice,
12 | this list of conditions and the following disclaimer in the documentation
13 | and/or other materials provided with the distribution.
14 | * Neither the name of the
nor the names of its
15 | contributors may be used to endorse or promote products derived from this
16 | software without specific prior written permission.
17 |
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
22 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
27 | OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
/docs/_layouts/page.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 |
9 |
10 | {% if page.simple_map == true %}
11 |
12 |
17 |
18 | {% include custom/{{page.map_name}}.html %}
19 |
20 | {% elsif page.complex_map == true %}
21 |
22 |
27 |
28 | {% include custom/{{page.map_name}}.html %}
29 |
30 | {% endif %}
31 |
32 |
33 |
34 | {% if page.summary %}
35 |
{{page.summary}}
36 | {% endif %}
37 |
38 | {% unless page.toc == false %}
39 | {% include toc.html %}
40 | {% endunless %}
41 |
42 |
43 | {% if site.github_editme_path %}
44 |
45 |
46 |
47 | {% endif %}
48 |
49 | {{content}}
50 |
51 |
62 |
63 |
3 |
4 |
13 |
14 |
15 |
16 | {{item.title}} {{item.title}} {{item.title}}
38 | {{ folder.title }}
39 |
50 |
51 | {% endfor %}
52 | {% endfor %}
53 | {% if site.google_search %}
54 |
55 | {% include search_google_custom.html %}
56 |
57 | {% endif %}
58 |
59 |
60 |
63 |
--------------------------------------------------------------------------------
/docs/js/jquery.navgoco.min.js:
--------------------------------------------------------------------------------
1 | /*
2 | * jQuery Navgoco Menus Plugin v0.2.1 (2014-04-11)
3 | * https://github.com/tefra/navgoco
4 | *
5 | * Copyright (c) 2014 Chris T (@tefra)
6 | * BSD - https://github.com/tefra/navgoco/blob/master/LICENSE-BSD
7 | */
8 | !function(a){"use strict";var b=function(b,c,d){return this.el=b,this.$el=a(b),this.options=c,this.uuid=this.$el.attr("id")?this.$el.attr("id"):d,this.state={},this.init(),this};b.prototype={init:function(){var b=this;b._load(),b.$el.find("ul").each(function(c){var d=a(this);d.attr("data-index",c),b.options.save&&b.state.hasOwnProperty(c)?(d.parent().addClass(b.options.openClass),d.show()):d.parent().hasClass(b.options.openClass)?(d.show(),b.state[c]=1):d.hide()});var c=a("
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
_BaseOptimizer patches
42 |
43 |
44 |
45 | {% raw %}
46 |
47 |
48 |
49 |
50 | {% endraw %}
51 |
52 | {% raw %}
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
_BaseOptimizer.__getstate__()
64 |
65 |
Pickling opt state should include param_groups and defaults
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
_BaseOptimizer.__setstate__(data
76 |
77 |
Pickling opt state should include param_groups and defaults
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 | {% endraw %}
88 |
89 |
90 |
91 |
Patch the fastai.optimizer._BaseOptimizer __getstate__ and __setstate__ methods which are used in pickling fastai optimizers.
92 |
This should fix the bug where running the learner on multiple TPU cores on XLA triggers an error in which the method _fetch_gradients(optimizer) fails in the statement for param_group in optimizer.__getstate__()['param_groups']: in the torch_xla.core.xla_model module.
93 |
The patch modifies the copy constructor to include the param_groups and defaults.
94 |
95 |
96 |
97 |
98 |
" + txt + " ";
66 | else if (this_level <= level){ // higher level than before; end parent ol
67 | for(i = this_level; i < level; i++) {
68 | html += "" + txt + " ";
71 | }
72 | else if (this_level > level) { // lower level than before; expand the previous to contain a ol
73 | for(i = this_level; i > level; i--) {
74 | html += "<"+settings.listType+">"+((i-level == 2) ? "" : " ")
75 | }
76 | html += "" + txt + " ";
77 | }
78 | level = this_level; // update for the next one
79 | });
80 | html += "";
81 | if (!settings.noBackToTopLinks) {
82 | $(document).on('click', '.back-to-top', function() {
83 | $(window).scrollTop(0);
84 | window.location.hash = '';
85 | });
86 | }
87 |
88 | render[settings.showEffect]();
89 | };
90 | })(jQuery);
91 |
--------------------------------------------------------------------------------
/docs/_layouts/default.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | {% include head.html %}
5 |
41 |
46 |
57 | {% if page.datatable == true %}
58 |
59 |
83 |
84 |
85 |
86 | {% assign content_col_size = "col-md-12" %}
87 | {% unless page.hide_sidebar %}
88 |
89 |
92 | {% assign content_col_size = "col-md-9" %}
93 | {% endunless %}
94 |
95 |
96 |
97 | {{content}}
98 |
99 |
100 |
101 |
102 |
103 |
104 |
{{ page.title }} | {{ site.site_title }}
7 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
Utilities used by other modules
42 |
43 |
44 |
45 |
46 |
47 | {% raw %}
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
xla_imported()
59 |
60 |
Check whether the torch_xla module has been successfully imported
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 | {% endraw %}
71 |
72 | {% raw %}
73 |
74 |
75 |
76 |
77 | {% endraw %}
78 |
79 |
80 |
81 |
xla_importedtorch_xla module has been successfully imported.
82 |
83 |
84 |
85 |
86 | {% raw %}
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
print_aten_ops()
98 |
99 |
print out xla aten operations (from xla debug metrics report torch_xla.debug.metrics)
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 | {% endraw %}
110 |
111 | {% raw %}
112 |
113 |
114 |
115 |
116 | {% endraw %}
117 |
118 |
119 |
120 |
One of the problems we have hit testing different models and transforms is that sometimes it is slower on TPUs compared to running on CPUs, but this happens because we hit operations on Pytorch XLA that are only handled by the CPU and not by the accelerator.
121 |
print_aten_opsstdout, so the only way to get that info is capture it.
122 |
123 |
124 |
125 |
126 | {% raw %}
127 |
128 |
142 | {% endraw %}
143 |
144 |
=0}var self=this;self.matches=function(string,crit){return"string"!=typeof string?!1:(string=string.trim(),doMatch(string,crit))}}module.exports=new LiteralSearchStrategy},{}],4:[function(require,module){module.exports=function(){function findMatches(store,crit,strategy){for(var data=store.get(),i=0;i{title} ',noResultsText:"No results found",limit:10,fuzzy:!1};self.init=function(_opt){validateOptions(_opt),assignOptions(_opt),isJSON(opt.dataSource)?initWithJSON(opt.dataSource):initWithURL(opt.dataSource)}}var Searcher=require("./Searcher"),Templater=require("./Templater"),Store=require("./Store"),JSONLoader=require("./JSONLoader"),searcher=new Searcher,templater=new Templater,store=new Store,jsonLoader=new JSONLoader;window.SimpleJekyllSearch=new SimpleJekyllSearch}(window,document)},{"./JSONLoader":1,"./Searcher":4,"./Store":5,"./Templater":6}]},{},[7]);
2 |
--------------------------------------------------------------------------------
/nbs/02b_misc_utils.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "#default_exp misc_utils"
10 | ]
11 | },
12 | {
13 | "cell_type": "markdown",
14 | "metadata": {},
15 | "source": [
16 | "# Miscellaneous Utilities"
17 | ]
18 | },
19 | {
20 | "cell_type": "markdown",
21 | "metadata": {},
22 | "source": [
23 | "\n",
62 | " \n",
63 | " \n",
64 | " epoch \n",
65 | " train_loss \n",
66 | " valid_loss \n",
67 | " time \n",
68 | " \n",
69 | " \n",
70 | " \n",
71 | " \n",
72 | " 0 \n",
73 | " 0.943764 \n",
74 | " 0.945623 \n",
75 | " 00:08 \n",
76 | " \n",
77 | " \n",
78 | " 1 \n",
79 | " 0.850762 \n",
80 | " 0.866062 \n",
81 | " 00:08 \n",
82 | " \n",
83 | " \n",
84 | " 2 \n",
85 | " 0.735538 \n",
86 | " 0.828996 \n",
87 | " 00:08 \n",
88 | " \n",
89 | " \n",
90 | " 3 \n",
91 | " 0.578858 \n",
92 | " 0.816693 \n",
93 | " 00:08 \n",
94 | " \n",
95 | " \n",
96 | " 4 \n",
97 | " 0.469008 \n",
98 | " 0.817968 \n",
99 | " 00:08 \n",
100 | " \n",
101 | " \n",
102 | "
"
103 | ],
104 | "text/plain": [
105 | ""
106 | ]
107 | },
108 | "metadata": {},
109 | "output_type": "display_data"
110 | }
111 | ],
112 | "source": [
113 | "learn.fit_one_cycle(5, 5e-3, wd=0.1)"
114 | ]
115 | },
116 | {
117 | "cell_type": "markdown",
118 | "metadata": {},
119 | "source": [
120 | "### Interpretation"
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": 10,
126 | "metadata": {},
127 | "outputs": [
128 | {
129 | "data": {
130 | "text/plain": [
131 | "array(['Star Wars (1977)', 'Contact (1997)', 'Fargo (1996)',\n",
132 | " 'Return of the Jedi (1983)', 'Liar Liar (1997)',\n",
133 | " 'English Patient, The (1996)', 'Scream (1996)', 'Toy Story (1995)',\n",
134 | " 'Air Force One (1997)', 'Independence Day (ID4) (1996)'],\n",
135 | " dtype=object)"
136 | ]
137 | },
138 | "execution_count": 10,
139 | "metadata": {},
140 | "output_type": "execute_result"
141 | }
142 | ],
143 | "source": [
144 | "g = ratings.groupby(\"title\")['rating'].count()\n",
145 | "top_movies = g.sort_values(ascending=False).index.values[:1000]\n",
146 | "top_movies[:10]"
147 | ]
148 | },
149 | {
150 | "cell_type": "code",
151 | "execution_count": 11,
152 | "metadata": {},
153 | "outputs": [
154 | {
155 | "data": {
156 | "text/plain": [
157 | "torch.Size([1000])"
158 | ]
159 | },
160 | "execution_count": 11,
161 | "metadata": {},
162 | "output_type": "execute_result"
163 | }
164 | ],
165 | "source": [
166 | "movie_bias = learn.model.bias(top_movies, is_item=True)\n",
167 | "movie_bias.shape"
168 | ]
169 | },
170 | {
171 | "cell_type": "code",
172 | "execution_count": 16,
173 | "metadata": {},
174 | "outputs": [],
175 | "source": [
176 | "mean_ratings = ratings.groupby(\"title\")['rating'].mean()\n",
177 | "movie_ratings = [(b, i, mean_ratings.loc[i]) for i,b in zip(top_movies,movie_bias)]"
178 | ]
179 | },
180 | {
181 | "cell_type": "code",
182 | "execution_count": 17,
183 | "metadata": {},
184 | "outputs": [
185 | {
186 | "data": {
187 | "text/plain": [
188 | "[(tensor(-0.3798),\n",
189 | " 'Children of the Corn: The Gathering (1996)',\n",
190 | " 1.3157894736842106),\n",
191 | " (tensor(-0.2619), 'Mortal Kombat: Annihilation (1997)', 1.9534883720930232),\n",
192 | " (tensor(-0.2598), 'Bio-Dome (1996)', 1.903225806451613),\n",
193 | " (tensor(-0.2545), \"McHale's Navy (1997)\", 2.1884057971014492),\n",
194 | " (tensor(-0.2445), 'Showgirls (1995)', 1.9565217391304348),\n",
195 | " (tensor(-0.2393), 'Free Willy 3: The Rescue (1997)', 1.7407407407407407),\n",
196 | " (tensor(-0.2383), 'Leave It to Beaver (1997)', 1.8409090909090908),\n",
197 | " (tensor(-0.2355), 'Crow: City of Angels, The (1996)', 1.9487179487179487),\n",
198 | " (tensor(-0.2269),\n",
199 | " 'Lawnmower Man 2: Beyond Cyberspace (1996)',\n",
200 | " 1.7142857142857142),\n",
201 | " (tensor(-0.2221), 'Beautician and the Beast, The (1997)', 2.313953488372093),\n",
202 | " (tensor(-0.2202), 'Barb Wire (1996)', 1.9333333333333333),\n",
203 | " (tensor(-0.2080), 'Cable Guy, The (1996)', 2.339622641509434),\n",
204 | " (tensor(-0.2038), 'Beverly Hills Ninja (1997)', 2.3125),\n",
205 | " (tensor(-0.2036), 'Striptease (1996)', 2.2388059701492535),\n",
206 | " (tensor(-0.1995), 'Island of Dr. Moreau, The (1996)', 2.1578947368421053)]"
207 | ]
208 | },
209 | "execution_count": 17,
210 | "metadata": {},
211 | "output_type": "execute_result"
212 | }
213 | ],
214 | "source": [
215 | "item0 = lambda o:o[0]\n",
216 | "sorted(movie_ratings, key=item0)[:15]"
217 | ]
218 | },
219 | {
220 | "cell_type": "code",
221 | "execution_count": null,
222 | "metadata": {},
223 | "outputs": [],
224 | "source": []
225 | }
226 | ],
227 | "metadata": {
228 | "kernelspec": {
229 | "display_name": "Python 3",
230 | "language": "python",
231 | "name": "python3"
232 | },
233 | "language_info": {
234 | "codemirror_mode": {
235 | "name": "ipython",
236 | "version": 3
237 | },
238 | "file_extension": ".py",
239 | "mimetype": "text/x-python",
240 | "name": "python",
241 | "nbconvert_exporter": "python",
242 | "pygments_lexer": "ipython3",
243 | "version": "3.7.6"
244 | }
245 | },
246 | "nbformat": 4,
247 | "nbformat_minor": 4
248 | }
249 |
--------------------------------------------------------------------------------
/fastai_xla_extensions/cifar_loader.py:
--------------------------------------------------------------------------------
1 | # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/02_cifar_loader.ipynb (unless otherwise specified).
2 |
3 | __all__ = ['download_torch_dsets', 'load_torch_items', 'load_classes', 'CifarNP2ImageTransform', 'Int2TensorTransform',
4 | 'CifarImageTransform', 'CifarImage2FloatTransform', 'make_torch_tfms', 'CifarTupleTransform', 'TupleTorchDS',
5 | 'make_cifar_item_tfm', 'i2t_tfm', 'cfnp2img_tfm', 'cfimg_tfm', 'cfimg2float_tfm', 'make_cifar_tls',
6 | 'make_cifar_dl', 'make_fastai_cifar_dls']
7 |
8 | # Cell
9 | def download_torch_dsets(path, torch_dset):
10 | """Download cifar10 datasets using torchvision utils
11 |
12 | Arguments:
13 | path (pathlib.Path): path to download the dataset (aka root)
14 | """
15 | train_dataset = torch_dset(
16 | root=path,
17 | train=True,
18 | download=True
19 |
20 | )
21 | test_dataset = torch_dset(
22 | root=path,
23 | train=False,
24 | download=True,
25 | )
26 | return train_dataset,test_dataset
27 |
28 | # Internal Cell
29 | import numpy as np
30 | import torchvision.datasets.utils as tv_utils
31 | import torchvision.datasets.cifar as cifar_dsets
32 | import pickle
33 |
34 | # Cell
35 | def load_torch_items(downloaded_list, path, check=False):
36 | """loads cifar test/train items into tuple(data, target)
37 |
38 | scrobbled together from torch.data.utils.datasets.CIFAR10 code
39 | https://pytorch.org/docs/stable/_modules/torchvision/datasets/cifar.html#CIFAR10
40 |
41 | Arguments:
42 | downloaded_list : a list of file names with their checksum, see CIFAR10.train_list or CIFAR10.test_list.
43 | path (pathlib.Path): the root path where the dataset was downloaded
44 | check(bool, optional): whether to perform an integrity check on the downloaded files (default: False)
45 | """
46 | data = []
47 | targets = []
48 | # now load the picked numpy arrays
49 | for file_name, checksum in downloaded_list:
50 | file_path = path/cifar_dsets.CIFAR10.base_folder/file_name
51 | if check and not tv_utils.check_integrity(file_path, checksum):
52 | raise RuntimeError(
53 | f'Data checksum failed for file:{file_path} checksum:{checksum}')
54 | with open(file_path, 'rb') as f:
55 | entry = pickle.load(f, encoding='latin1')
56 | data.append(entry['data'])
57 | if 'labels' in entry:
58 | targets.extend(entry['labels'])
59 | else:
60 | targets.extend(entry['fine_labels'])
61 |
62 | data = np.vstack(data).reshape(-1, 3, 32, 32)
63 | data = data.transpose((0, 2, 3, 1)) # convert to HWC
64 |
65 | return data, targets
66 |
67 | # Cell
68 | # TODO: incorporate list of classes into dataloaders vocab and decodes
69 |
70 | from fastcore.foundation import L
71 | def load_classes(path):
72 | """Load classes to used to map categories to target labels"""
73 | base_folder = cifar_dsets.CIFAR10.base_folder
74 | meta = cifar_dsets.CIFAR10.meta
75 | file_path = path/base_folder/meta['filename']
76 | if not tv_utils.check_integrity(file_path, meta['md5']):
77 | raise RuntimeError('Dataset metadata file not found or corrupted.' +
78 | ' You can use download=True to download it')
79 | data = {}
80 | with open(file_path, 'rb') as infile:
81 | data = pickle.load(infile, encoding='latin1')
82 | classes = data[meta['key']]
83 | # class_for i, _class in enumerato_idx = {_class: i te(classes)}
84 | return L(classes)
85 |
86 | # Internal Cell
87 | from fastcore.transform import Transform
88 | import torchvision.transforms.functional as TVF
89 | import torch
90 | import PIL
91 |
92 | # Cell
93 | class CifarNP2ImageTransform(Transform):
94 | def encodes(self, o:np.ndarray) -> None:
95 | return PIL.Image.fromarray(o)
96 |
97 | # Cell
98 | class Int2TensorTransform(Transform):
99 | def encodes(self, o: int) -> None:
100 | return torch.tensor(o)
101 |
102 | # Cell
103 | class CifarImageTransform(Transform):
104 | def encodes(self, o: PIL.Image) -> None:
105 | return TVF.to_tensor(o)
106 |
107 | # Cell
108 | class CifarImage2FloatTransform(Transform):
109 | def encodes(self, o: torch.Tensor) -> None:
110 | return o.float().div_(255.)
111 |
112 | # Internal Cell
113 | import torchvision as thv
114 |
115 | # Cell
116 | def make_torch_tfms():
117 | norm = thv.transforms.Normalize(
118 | mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))
119 | transform_train = thv.transforms.Compose([
120 | thv.transforms.RandomCrop(32, padding=4),
121 | thv.transforms.RandomHorizontalFlip(),
122 | thv.transforms.ToTensor(),
123 | norm,
124 | ])
125 | transform_test = thv.transforms.Compose([
126 | thv.transforms.ToTensor(),
127 | norm,
128 | ])
129 | return transform_train, transform_test
130 |
131 | # Internal Cell
132 | from fastcore.transform import ItemTransform
133 | from fastcore.basics import store_attr
134 |
135 | # Cell
136 | class CifarTupleTransform(ItemTransform):
137 | def __init__(self, x_tfm, y_tfm):
138 | store_attr()
139 | def encodes(self,xy):
140 | return [self.x_tfm(xy[0]), self.y_tfm(xy[1])]
141 |
142 | # Internal Cell
143 | import torch.utils.data as th_data
144 | from torch.utils.data import Dataset
145 |
146 | # Cell
147 | # TODO: Use TupleTorchDS to create torch dataloaders
148 |
149 | class TupleTorchDS(th_data.Dataset):
150 | def __init__(self, items, x_tfm=None, y_tfm=None):
151 | store_attr()
152 |
153 | def __len__(self):
154 | return len(self.items)
155 |
156 | def __getitem__(self, index):
157 | x,y = self.items[index]
158 | x = self.x_tfm(x) if self.x_tfm is not None else x
159 | y = self.y_tfm(y) if self.y_tfm is not None else y
160 | return (x,y)
161 |
162 | # Internal Cell
163 | from fastcore.transform import Pipeline
164 |
165 | # Cell
166 | i2t_tfm = Int2TensorTransform() # cnvt int -> torch.tensor
167 | cfnp2img_tfm = CifarNP2ImageTransform() # cnvt ndarray -> PIL.Image
168 | cfimg_tfm = CifarImageTransform() # cnvt PIL.Image -> torch.tensor
169 | cfimg2float_tfm = CifarImage2FloatTransform() # cnvt tensor int -> float + div 255
170 |
171 | def make_cifar_item_tfm(th_img_tfms=None):
172 | img_tfms = [cfnp2img_tfm]
173 | if th_img_tfms is not None:
174 | # assumes th_img_tfms incl ToTensor (cnvt2 PIL.Image -> tensor + div by 255)
175 | img_tfms += [th_img_tfms]
176 | else:
177 | img_tfms += [cfimg_tfm, cfimg2float_tfm]
178 |
179 | return CifarTupleTransform(x_tfm=Pipeline(img_tfms), y_tfm=i2t_tfm)
180 |
181 | # Internal Cell
182 | from fastai.data.core import TfmdDL
183 | from fastai.data.core import TfmdLists
184 | from fastcore.foundation import L
185 |
186 | # Cell
187 | def make_cifar_tls(file_list, path, item_tfm, check=True):
188 | data, targets = load_torch_items(file_list, path, check=check)
189 | item_tuples = L(data,targets).zip()
190 | tls = TfmdLists(item_tuples,[item_tfm])
191 | return tls
192 |
193 | # Cell
194 | def make_cifar_dl(file_list, path, th_img_tfms=None, check=True, bs=64, **kwargs):
195 | item_tfm = make_cifar_item_tfm(th_img_tfms)
196 | tls = make_cifar_tls(file_list, path, item_tfm, check=check)
197 | dl = TfmdDL(tls,bs=bs, **kwargs)
198 | return dl
199 |
200 | # Internal Cell
201 | from fastai.data.core import DataLoaders
202 |
203 | # Cell
204 | def make_fastai_cifar_dls(path, bs=64, check=True, device=None, **kwargs):
205 | train_tfm, test_tfm = make_torch_tfms()
206 | train_dl = make_cifar_dl(
207 | cifar_dsets.CIFAR10.train_list,
208 | path,
209 | train_tfm,
210 | check=check, bs=bs,
211 | shuffle=True)
212 | test_dl = make_cifar_dl(
213 | cifar_dsets.CIFAR10.test_list,
214 | path,
215 | test_tfm,
216 | check=check, bs=bs,
217 | shuffle=False)
218 | dls = DataLoaders(train_dl, test_dl, device=device)
219 | return dls
--------------------------------------------------------------------------------
/docs/Gemfile.lock:
--------------------------------------------------------------------------------
1 | GEM
2 | remote: https://rubygems.org/
3 | specs:
4 | activesupport (6.0.3.2)
5 | concurrent-ruby (~> 1.0, >= 1.0.2)
6 | i18n (>= 0.7, < 2)
7 | minitest (~> 5.1)
8 | tzinfo (~> 1.1)
9 | zeitwerk (~> 2.2, >= 2.2.2)
10 | addressable (2.8.0)
11 | public_suffix (>= 2.0.2, < 5.0)
12 | coffee-script (2.4.1)
13 | coffee-script-source
14 | execjs
15 | coffee-script-source (1.11.1)
16 | colorator (1.1.0)
17 | commonmarker (0.17.13)
18 | ruby-enum (~> 0.5)
19 | concurrent-ruby (1.1.6)
20 | dnsruby (1.61.4)
21 | simpleidn (~> 0.1)
22 | em-websocket (0.5.1)
23 | eventmachine (>= 0.12.9)
24 | http_parser.rb (~> 0.6.0)
25 | ethon (0.12.0)
26 | ffi (>= 1.3.0)
27 | eventmachine (1.2.7)
28 | execjs (2.7.0)
29 | faraday (1.0.1)
30 | multipart-post (>= 1.2, < 3)
31 | ffi (1.13.1)
32 | forwardable-extended (2.6.0)
33 | gemoji (3.0.1)
34 | github-pages (207)
35 | github-pages-health-check (= 1.16.1)
36 | jekyll (= 3.9.0)
37 | jekyll-avatar (= 0.7.0)
38 | jekyll-coffeescript (= 1.1.1)
39 | jekyll-commonmark-ghpages (= 0.1.6)
40 | jekyll-default-layout (= 0.1.4)
41 | jekyll-feed (= 0.13.0)
42 | jekyll-gist (= 1.5.0)
43 | jekyll-github-metadata (= 2.13.0)
44 | jekyll-mentions (= 1.5.1)
45 | jekyll-optional-front-matter (= 0.3.2)
46 | jekyll-paginate (= 1.1.0)
47 | jekyll-readme-index (= 0.3.0)
48 | jekyll-redirect-from (= 0.15.0)
49 | jekyll-relative-links (= 0.6.1)
50 | jekyll-remote-theme (= 0.4.1)
51 | jekyll-sass-converter (= 1.5.2)
52 | jekyll-seo-tag (= 2.6.1)
53 | jekyll-sitemap (= 1.4.0)
54 | jekyll-swiss (= 1.0.0)
55 | jekyll-theme-architect (= 0.1.1)
56 | jekyll-theme-cayman (= 0.1.1)
57 | jekyll-theme-dinky (= 0.1.1)
58 | jekyll-theme-hacker (= 0.1.1)
59 | jekyll-theme-leap-day (= 0.1.1)
60 | jekyll-theme-merlot (= 0.1.1)
61 | jekyll-theme-midnight (= 0.1.1)
62 | jekyll-theme-minimal (= 0.1.1)
63 | jekyll-theme-modernist (= 0.1.1)
64 | jekyll-theme-primer (= 0.5.4)
65 | jekyll-theme-slate (= 0.1.1)
66 | jekyll-theme-tactile (= 0.1.1)
67 | jekyll-theme-time-machine (= 0.1.1)
68 | jekyll-titles-from-headings (= 0.5.3)
69 | jemoji (= 0.11.1)
70 | kramdown (= 2.3.0)
71 | kramdown-parser-gfm (= 1.1.0)
72 | liquid (= 4.0.3)
73 | mercenary (~> 0.3)
74 | minima (= 2.5.1)
75 | nokogiri (>= 1.10.4, < 2.0)
76 | rouge (= 3.19.0)
77 | terminal-table (~> 1.4)
78 | github-pages-health-check (1.16.1)
79 | addressable (~> 2.3)
80 | dnsruby (~> 1.60)
81 | octokit (~> 4.0)
82 | public_suffix (~> 3.0)
83 | typhoeus (~> 1.3)
84 | html-pipeline (2.13.0)
85 | activesupport (>= 2)
86 | nokogiri (>= 1.4)
87 | http_parser.rb (0.6.0)
88 | i18n (0.9.5)
89 | concurrent-ruby (~> 1.0)
90 | jekyll (3.9.0)
91 | addressable (~> 2.4)
92 | colorator (~> 1.0)
93 | em-websocket (~> 0.5)
94 | i18n (~> 0.7)
95 | jekyll-sass-converter (~> 1.0)
96 | jekyll-watch (~> 2.0)
97 | kramdown (>= 1.17, < 3)
98 | liquid (~> 4.0)
99 | mercenary (~> 0.3.3)
100 | pathutil (~> 0.9)
101 | rouge (>= 1.7, < 4)
102 | safe_yaml (~> 1.0)
103 | jekyll-avatar (0.7.0)
104 | jekyll (>= 3.0, < 5.0)
105 | jekyll-coffeescript (1.1.1)
106 | coffee-script (~> 2.2)
107 | coffee-script-source (~> 1.11.1)
108 | jekyll-commonmark (1.3.1)
109 | commonmarker (~> 0.14)
110 | jekyll (>= 3.7, < 5.0)
111 | jekyll-commonmark-ghpages (0.1.6)
112 | commonmarker (~> 0.17.6)
113 | jekyll-commonmark (~> 1.2)
114 | rouge (>= 2.0, < 4.0)
115 | jekyll-default-layout (0.1.4)
116 | jekyll (~> 3.0)
117 | jekyll-feed (0.13.0)
118 | jekyll (>= 3.7, < 5.0)
119 | jekyll-gist (1.5.0)
120 | octokit (~> 4.2)
121 | jekyll-github-metadata (2.13.0)
122 | jekyll (>= 3.4, < 5.0)
123 | octokit (~> 4.0, != 4.4.0)
124 | jekyll-mentions (1.5.1)
125 | html-pipeline (~> 2.3)
126 | jekyll (>= 3.7, < 5.0)
127 | jekyll-optional-front-matter (0.3.2)
128 | jekyll (>= 3.0, < 5.0)
129 | jekyll-paginate (1.1.0)
130 | jekyll-readme-index (0.3.0)
131 | jekyll (>= 3.0, < 5.0)
132 | jekyll-redirect-from (0.15.0)
133 | jekyll (>= 3.3, < 5.0)
134 | jekyll-relative-links (0.6.1)
135 | jekyll (>= 3.3, < 5.0)
136 | jekyll-remote-theme (0.4.1)
137 | addressable (~> 2.0)
138 | jekyll (>= 3.5, < 5.0)
139 | rubyzip (>= 1.3.0)
140 | jekyll-sass-converter (1.5.2)
141 | sass (~> 3.4)
142 | jekyll-seo-tag (2.6.1)
143 | jekyll (>= 3.3, < 5.0)
144 | jekyll-sitemap (1.4.0)
145 | jekyll (>= 3.7, < 5.0)
146 | jekyll-swiss (1.0.0)
147 | jekyll-theme-architect (0.1.1)
148 | jekyll (~> 3.5)
149 | jekyll-seo-tag (~> 2.0)
150 | jekyll-theme-cayman (0.1.1)
151 | jekyll (~> 3.5)
152 | jekyll-seo-tag (~> 2.0)
153 | jekyll-theme-dinky (0.1.1)
154 | jekyll (~> 3.5)
155 | jekyll-seo-tag (~> 2.0)
156 | jekyll-theme-hacker (0.1.1)
157 | jekyll (~> 3.5)
158 | jekyll-seo-tag (~> 2.0)
159 | jekyll-theme-leap-day (0.1.1)
160 | jekyll (~> 3.5)
161 | jekyll-seo-tag (~> 2.0)
162 | jekyll-theme-merlot (0.1.1)
163 | jekyll (~> 3.5)
164 | jekyll-seo-tag (~> 2.0)
165 | jekyll-theme-midnight (0.1.1)
166 | jekyll (~> 3.5)
167 | jekyll-seo-tag (~> 2.0)
168 | jekyll-theme-minimal (0.1.1)
169 | jekyll (~> 3.5)
170 | jekyll-seo-tag (~> 2.0)
171 | jekyll-theme-modernist (0.1.1)
172 | jekyll (~> 3.5)
173 | jekyll-seo-tag (~> 2.0)
174 | jekyll-theme-primer (0.5.4)
175 | jekyll (> 3.5, < 5.0)
176 | jekyll-github-metadata (~> 2.9)
177 | jekyll-seo-tag (~> 2.0)
178 | jekyll-theme-slate (0.1.1)
179 | jekyll (~> 3.5)
180 | jekyll-seo-tag (~> 2.0)
181 | jekyll-theme-tactile (0.1.1)
182 | jekyll (~> 3.5)
183 | jekyll-seo-tag (~> 2.0)
184 | jekyll-theme-time-machine (0.1.1)
185 | jekyll (~> 3.5)
186 | jekyll-seo-tag (~> 2.0)
187 | jekyll-titles-from-headings (0.5.3)
188 | jekyll (>= 3.3, < 5.0)
189 | jekyll-watch (2.2.1)
190 | listen (~> 3.0)
191 | jemoji (0.11.1)
192 | gemoji (~> 3.0)
193 | html-pipeline (~> 2.2)
194 | jekyll (>= 3.0, < 5.0)
195 | kramdown (2.3.0)
196 | rexml
197 | kramdown-parser-gfm (1.1.0)
198 | kramdown (~> 2.0)
199 | liquid (4.0.3)
200 | listen (3.2.1)
201 | rb-fsevent (~> 0.10, >= 0.10.3)
202 | rb-inotify (~> 0.9, >= 0.9.10)
203 | mercenary (0.3.6)
204 | mini_portile2 (2.8.0)
205 | minima (2.5.1)
206 | jekyll (>= 3.5, < 5.0)
207 | jekyll-feed (~> 0.9)
208 | jekyll-seo-tag (~> 2.1)
209 | minitest (5.14.1)
210 | multipart-post (2.1.1)
211 | nokogiri (1.13.6)
212 | mini_portile2 (~> 2.8.0)
213 | racc (~> 1.4)
214 | octokit (4.18.0)
215 | faraday (>= 0.9)
216 | sawyer (~> 0.8.0, >= 0.5.3)
217 | pathutil (0.16.2)
218 | forwardable-extended (~> 2.6)
219 | public_suffix (3.1.1)
220 | racc (1.6.0)
221 | rb-fsevent (0.10.4)
222 | rb-inotify (0.10.1)
223 | ffi (~> 1.0)
224 | rexml (3.2.5)
225 | rouge (3.19.0)
226 | ruby-enum (0.8.0)
227 | i18n
228 | rubyzip (2.3.0)
229 | safe_yaml (1.0.5)
230 | sass (3.7.4)
231 | sass-listen (~> 4.0.0)
232 | sass-listen (4.0.0)
233 | rb-fsevent (~> 0.9, >= 0.9.4)
234 | rb-inotify (~> 0.9, >= 0.9.7)
235 | sawyer (0.8.2)
236 | addressable (>= 2.3.5)
237 | faraday (> 0.8, < 2.0)
238 | simpleidn (0.1.1)
239 | unf (~> 0.1.4)
240 | terminal-table (1.8.0)
241 | unicode-display_width (~> 1.1, >= 1.1.1)
242 | thread_safe (0.3.6)
243 | typhoeus (1.4.0)
244 | ethon (>= 0.9.0)
245 | tzinfo (1.2.10)
246 | thread_safe (~> 0.1)
247 | unf (0.1.4)
248 | unf_ext
249 | unf_ext (0.0.7.7)
250 | unicode-display_width (1.7.0)
251 | zeitwerk (2.4.0)
252 |
253 | PLATFORMS
254 | ruby
255 |
256 | DEPENDENCIES
257 | github-pages
258 | jekyll (~> 3.9)
259 |
260 | BUNDLED WITH
261 | 2.1.4
262 |
--------------------------------------------------------------------------------
/archive_nbs/TabularTrainingGPU.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "from fastai2.tabular.all import *"
10 | ]
11 | },
12 | {
13 | "cell_type": "code",
14 | "execution_count": 2,
15 | "metadata": {},
16 | "outputs": [],
17 | "source": [
18 | "path = untar_data(URLs.ADULT_SAMPLE)\n",
19 | "df = pd.read_csv(path/'adult.csv')"
20 | ]
21 | },
22 | {
23 | "cell_type": "code",
24 | "execution_count": 3,
25 | "metadata": {},
26 | "outputs": [],
27 | "source": [
28 | "dls = TabularDataLoaders.from_csv(path/'adult.csv', path=path, y_names=\"salary\",\n",
29 | " cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race'],\n",
30 | " cont_names = ['age', 'fnlwgt', 'education-num'],\n",
31 | " procs = [Categorify, FillMissing, Normalize])"
32 | ]
33 | },
34 | {
35 | "cell_type": "code",
36 | "execution_count": 4,
37 | "metadata": {},
38 | "outputs": [
39 | {
40 | "data": {
41 | "text/html": [
42 | "\n",
43 | " \n",
44 | " \n",
45 | " epoch \n",
46 | " train_loss \n",
47 | " valid_loss \n",
48 | " accuracy \n",
49 | " time \n",
50 | " \n",
51 | " \n",
52 | " \n",
53 | " \n",
54 | " 0 \n",
55 | " 0.369606 \n",
56 | " 0.363110 \n",
57 | " 0.827856 \n",
58 | " 00:04 \n",
59 | " \n",
60 | " \n",
61 | " 1 \n",
62 | " 0.367178 \n",
63 | " 0.364534 \n",
64 | " 0.828010 \n",
65 | " 00:04 \n",
66 | " \n",
67 | " \n",
68 | " 2 \n",
69 | " 0.348837 \n",
70 | " 0.367921 \n",
71 | " 0.827088 \n",
72 | " 00:04 \n",
73 | " \n",
74 | " \n",
75 | " 3 \n",
76 | " 0.355421 \n",
77 | " 0.364824 \n",
78 | " 0.830313 \n",
79 | " 00:04 \n",
80 | " \n",
81 | " \n",
82 | " 4 \n",
83 | " 0.352137 \n",
84 | " 0.357688 \n",
85 | " 0.834459 \n",
86 | " 00:04 \n",
87 | " \n",
88 | " \n",
89 | " 5 \n",
90 | " 0.344102 \n",
91 | " 0.360992 \n",
92 | " 0.836456 \n",
93 | " 00:04 \n",
94 | " \n",
95 | " \n",
96 | " 6 \n",
97 | " 0.341461 \n",
98 | " 0.357141 \n",
99 | " 0.838298 \n",
100 | " 00:04 \n",
101 | " \n",
102 | " \n",
103 | " 7 \n",
104 | " 0.330545 \n",
105 | " 0.355759 \n",
106 | " 0.837377 \n",
107 | " 00:04 \n",
108 | " \n",
109 | " \n",
110 | " 8 \n",
111 | " 0.343082 \n",
112 | " 0.357329 \n",
113 | " 0.836609 \n",
114 | " 00:04 \n",
115 | " \n",
116 | " \n",
117 | " 9 \n",
118 | " 0.327605 \n",
119 | " 0.357438 \n",
120 | " 0.835688 \n",
121 | " 00:04 \n",
122 | " \n",
123 | " \n",
124 | " 10 \n",
125 | " 0.320859 \n",
126 | " 0.357013 \n",
127 | " 0.836302 \n",
128 | " 00:04 \n",
129 | " \n",
130 | " \n",
131 | " 11 \n",
132 | " 0.320642 \n",
133 | " 0.360160 \n",
134 | " 0.833692 \n",
135 | " 00:04 \n",
136 | " \n",
137 | " \n",
138 | " 12 \n",
139 | " 0.330805 \n",
140 | " 0.360279 \n",
141 | " 0.836456 \n",
142 | " 00:04 \n",
143 | " \n",
144 | " \n",
145 | " 13 \n",
146 | " 0.321315 \n",
147 | " 0.359108 \n",
148 | " 0.835534 \n",
149 | " 00:04 \n",
150 | " \n",
151 | " \n",
152 | " 14 \n",
153 | " 0.330493 \n",
154 | " 0.358782 \n",
155 | " 0.836302 \n",
156 | " 00:04 \n",
157 | " \n",
158 | " \n",
159 | "
"
160 | ],
161 | "text/plain": [
162 | ""
163 | ]
164 | },
165 | "metadata": {},
166 | "output_type": "display_data"
167 | }
168 | ],
169 | "source": [
170 | "learn = tabular_learner(dls, metrics=accuracy)\n",
171 | "learn.fit_one_cycle(15)"
172 | ]
173 | },
174 | {
175 | "cell_type": "code",
176 | "execution_count": 5,
177 | "metadata": {},
178 | "outputs": [
179 | {
180 | "data": {
181 | "text/html": [],
182 | "text/plain": [
183 | ""
184 | ]
185 | },
186 | "metadata": {},
187 | "output_type": "display_data"
188 | },
189 | {
190 | "data": {
191 | "text/plain": [
192 | "( workclass education marital-status occupation relationship race \\\n",
193 | " 0 5.0 8.0 3.0 0.0 6.0 5.0 \n",
194 | " \n",
195 | " education-num_na age fnlwgt education-num salary \n",
196 | " 0 1.0 0.762793 -0.837396 0.752865 1.0 ,\n",
197 | " tensor(1),\n",
198 | " tensor([0.4151, 0.5849]))"
199 | ]
200 | },
201 | "execution_count": 5,
202 | "metadata": {},
203 | "output_type": "execute_result"
204 | }
205 | ],
206 | "source": [
207 | "learn.predict(df.iloc[0])\n"
208 | ]
209 | },
210 | {
211 | "cell_type": "code",
212 | "execution_count": 6,
213 | "metadata": {},
214 | "outputs": [
215 | {
216 | "data": {
217 | "text/html": [],
218 | "text/plain": [
219 | ""
220 | ]
221 | },
222 | "metadata": {},
223 | "output_type": "display_data"
224 | },
225 | {
226 | "data": {
227 | "text/plain": [
228 | "(tensor([[4.1512e-01, 5.8488e-01],\n",
229 | " [1.9511e-01, 8.0489e-01],\n",
230 | " [9.9947e-01, 5.3267e-04],\n",
231 | " ...,\n",
232 | " [5.7765e-01, 4.2235e-01],\n",
233 | " [6.7786e-01, 3.2214e-01],\n",
234 | " [6.7229e-01, 3.2771e-01]]),\n",
235 | " None)"
236 | ]
237 | },
238 | "execution_count": 6,
239 | "metadata": {},
240 | "output_type": "execute_result"
241 | }
242 | ],
243 | "source": [
244 | "test_df = df.copy()\n",
245 | "test_df.drop(['salary'], axis=1, inplace=True)\n",
246 | "dl = learn.dls.test_dl(test_df)\n",
247 | "###\n",
248 | "learn.get_preds(dl=dl)"
249 | ]
250 | },
251 | {
252 | "cell_type": "code",
253 | "execution_count": null,
254 | "metadata": {},
255 | "outputs": [],
256 | "source": []
257 | }
258 | ],
259 | "metadata": {
260 | "kernelspec": {
261 | "display_name": "Python 3",
262 | "language": "python",
263 | "name": "python3"
264 | },
265 | "language_info": {
266 | "codemirror_mode": {
267 | "name": "ipython",
268 | "version": 3
269 | },
270 | "file_extension": ".py",
271 | "mimetype": "text/x-python",
272 | "name": "python",
273 | "nbconvert_exporter": "python",
274 | "pygments_lexer": "ipython3",
275 | "version": "3.7.6"
276 | }
277 | },
278 | "nbformat": 4,
279 | "nbformat_minor": 4
280 | }
281 |
--------------------------------------------------------------------------------
/fastai_xla_extensions/multi_core/torch_compat.py:
--------------------------------------------------------------------------------
1 | # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03a_multi_core.torch_compat.ipynb (unless otherwise specified).
2 |
3 | __all__ = ['TfmdTorchDS', 'to_list', 'has_setup', 'run_setups', 'TorchDatasetBuilder', 'VocabularyMapper',
4 | 'make_torch_dataloaders', 'FileNamePatternLabeller']
5 |
6 | # Internal Cell
7 | from ..utils import xla_imported
8 | from .base import *
9 | from ..misc_utils import *
10 |
11 |
12 | # Internal Cell
13 | try:
14 | import torch_xla
15 | except ImportError:
16 | pass
17 |
18 | # Internal Cell
19 | if xla_imported():
20 | import torch_xla.core.xla_model as xm
21 | import torch_xla.distributed.xla_multiprocessing as xmp
22 |
23 | # Internal Cell
24 | from fastcore.basics import patch_to
25 | import torch
26 | import torch.utils.data as th_data
27 | from fastcore.foundation import L
28 | from pathlib import Path
29 | from fastcore.transform import Pipeline
30 | from fastai.data.core import DataLoaders
31 | from pathlib import Path
32 | from fastai.torch_core import find_bs, TensorBase
33 | from fastai.torch_core import TensorBase
34 | from fastcore.xtras import is_listy
35 | import torch.utils.hooks
36 | import torch.utils.data.distributed as th_distrib
37 |
38 | # Cell
39 | class TfmdTorchDS(th_data.Dataset):
40 | "A torch dataset compatible holder for items with x and y transforms"
41 | def __init__(self, items, x_tfm=None, y_tfm=None):
42 | self.items = items
43 | self.x_tfm = x_tfm
44 | self.y_tfm = y_tfm
45 |
46 | def __len__(self):
47 | return len(self.items)
48 |
49 | def __getitem__(self, index):
50 | item = self.items[index]
51 | x = self.x_tfm(item) if self.x_tfm is not None else item
52 | y = self.y_tfm(item) if self.y_tfm is not None else item
53 | return (x,y)
54 |
55 | # Internal Cell
56 | import torchvision as thv
57 | from operator import itemgetter
58 | from fastcore.imports import noop
59 |
60 | # Cell
61 | def to_list(o):
62 | "return item o as a list (unchanged if o is already a list and empty list if o is None)"
63 | return [] if o is None else [o] if not is_listy(o) else o
64 |
65 | def has_setup(tfms):
66 | """returns last index if at least 1 `tfm` in `tfms` has a method `setup` else return -1"""
67 | setups = L(tfms).attrgot('setup',None).argwhere(noop) # get indexes where tfm has `setup` attribute
68 | return -1 if len(setups) == 0 else setups[-1]
69 |
70 | def run_setups(tfms, items):
71 | """run tfm setups including tfm for all items"""
72 | indx = has_setup(tfms)
73 | if indx == -1: # no setup found
74 | return
75 |
76 | for i,tfm in enumerate(tfms):
77 | if hasattr(tfm,'setup'):
78 | tfm.setup(items)
79 | if i < indx:
80 | # tfm items to be fed into next tfm
81 | items = [tfm(item) for item in items]
82 |
83 | # Cell
84 | class TorchDatasetBuilder:
85 | "build torch compatible train and test datasets with transforms"
86 | def __init__(self, source, get_items, splitter,
87 | x_tfms, y_tfms,
88 | x_type_tfms=None,
89 | x_train_tfms=None, x_test_tfms=None,
90 | do_setup=False):
91 | self.source = source
92 | self.get_items = get_items
93 | self.splitter = splitter
94 | self.do_setup = do_setup
95 | self.x_tfms = to_list(x_tfms)
96 | self.y_tfms = to_list(y_tfms)
97 | self.x_type_tfms = to_list(x_type_tfms)
98 | self.x_train_tfms = to_list(x_train_tfms)
99 | self.x_test_tfms = to_list(x_test_tfms)
100 |
101 | def setup(self, items, do_setup=None, setup_x=False):
102 | self.do_setup = do_setup if do_setup is not None else self.do_setup
103 | if self.do_setup:
104 | all_x_tfms = [*self.x_type_tfms, *self.x_train_tfms, *self.x_tfms]
105 | if setup_x:
106 | run_setups(all_x_tfms, items)
107 | run_setups(self.y_tfms, items)
108 | self.do_setup = False
109 |
110 | def get_datasets(self, do_setup=None):
111 | self.do_setup = do_setup if do_setup is not None else self.do_setup
112 |
113 | items = self.get_items(self.source) if self.get_items is not None else self.source
114 |
115 | train_idxs, test_idxs = self.splitter(items)
116 |
117 | train_items = itemgetter(*train_idxs)(items)
118 | test_items = itemgetter(*test_idxs)(items)
119 | self.setup(train_items)
120 | allx_test_tfms = [*self.x_type_tfms, *self.x_test_tfms, *self.x_tfms]
121 | allx_train_tfms = [*self.x_type_tfms, *self.x_train_tfms, *self.x_tfms]
122 | train_x_tfm = thv.transforms.Compose(allx_train_tfms)
123 | test_x_tfm = thv.transforms.Compose(allx_test_tfms)
124 | y_tfm = thv.transforms.Compose(self.y_tfms)
125 | train_ds = TfmdTorchDS(train_items, x_tfm=train_x_tfm, y_tfm=y_tfm)
126 | test_ds = TfmdTorchDS(test_items, x_tfm=test_x_tfm, y_tfm=y_tfm)
127 | return train_ds, test_ds
128 |
129 | # Cell
130 | from fastai.data.transforms import CategoryMap
131 |
132 | class VocabularyMapper:
133 | """A simplified version of the fastai Categorize Transform"""
134 | def __init__(self, vocab=None):
135 | self.vocab = vocab
136 | self.c = 0
137 | def setup(self, items):
138 | self.vocab = CategoryMap(items)
139 | self.c = len(self.vocab)
140 | def __call__(self, o):
141 | if self.vocab is None: return o
142 | try:
143 | return torch.tensor(self.vocab.o2i[o])
144 | except KeyError as e:
145 | raise KeyError(f"Label '{o}' was not included in the training dataset") from e
146 |
147 | # Cell
148 | def make_torch_dataloaders(train_dataset, test_dataset,
149 | rank,
150 | world_size,
151 | bs,
152 | num_workers=4,
153 | distrib=True,
154 | sync_valid=False):
155 | "make torch-based distributed dataloaders from torch compatible datasets"
156 | if distrib:
157 | train_sampler = th_distrib.DistributedSampler(
158 | train_dataset,
159 | num_replicas=world_size,
160 | rank=rank,
161 | shuffle=True)
162 | train_loader = th_data.DataLoader(
163 | train_dataset,
164 | batch_size=bs,
165 | sampler=train_sampler,
166 | # shuffle=True,
167 | num_workers=num_workers,
168 | drop_last=True)
169 |
170 | if sync_valid:
171 | test_sampler = th_distrib.DistributedSampler(
172 | test_dataset,
173 | num_replicas=world_size,
174 | rank=rank,
175 | shuffle=False)
176 |
177 | test_loader = th_data.DataLoader(
178 | test_dataset,
179 | batch_size=bs,
180 | sampler=test_sampler,
181 | # shuffle=False,
182 | num_workers=num_workers,
183 | drop_last=True)
184 | else:
185 | test_loader = th_data.DataLoader(
186 | test_dataset,
187 | batch_size=bs,
188 | shuffle=False,
189 | num_workers=num_workers,
190 | drop_last=True)
191 |
192 | else:
193 | train_loader = th_data.DataLoader(
194 | train_dataset,
195 | batch_size=bs,
196 | # sampler=train_sampler,
197 | shuffle=True,
198 | num_workers=num_workers,
199 | drop_last=True)
200 |
201 | test_loader = th_data.DataLoader(
202 | test_dataset,
203 | batch_size=bs,
204 | shuffle=False,
205 | num_workers=num_workers,
206 | drop_last=True)
207 | dataloaders = DataLoaders(train_loader, test_loader, device=None)
208 | return dataloaders
209 |
210 | # Internal Cell
211 | import re
212 |
213 | # Cell
214 | class FileNamePatternLabeller:
215 | "Delayed action version of fastai RegexLabeller with file name selection"
216 | def __init__(self, pat_str, match=False):
217 | self.pat_str = pat_str
218 | self.match = match
219 | self.matcher = None
220 | self.pat = None
221 | def __call__(self, f):
222 | if isinstance(f,str):
223 | f = Path(f)
224 | o = f.name
225 | if self.pat is None:
226 | self.pat = re.compile(self.pat_str)
227 | self.matcher = self.pat.match if self.match else self.pat.search
228 | res = self.matcher(o)
229 | assert res, f'Failed to find "{self.pat}" in {o}'
230 | return res.group(1)
--------------------------------------------------------------------------------
/fastai_xla_extensions/multi_core/inference.py:
--------------------------------------------------------------------------------
1 | # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03e_multi_core.inference.ipynb (unless otherwise specified).
2 |
3 | __all__ = ['setup_inference_args', 'save_pred_results', 'xla_run_inference', 'reload_pred_results',
4 | 'prep_inference_args']
5 |
6 | # Cell
7 | try:
8 | import torch_xla
9 | except ImportError:
10 | pass
11 |
12 | # Cell
13 |
14 | from fastai.vision.all import *
15 | from ..utils import xla_imported
16 | from ..misc_utils import *
17 | from ..core import XLAOptCallback
18 | from .base import *
19 | from .learner import *
20 | from .callback import *
21 |
22 | # Cell
23 |
24 | if xla_imported():
25 | import torch_xla.core.xla_model as xm
26 | import torch_xla.distributed.parallel_loader as pl
27 | import torch_xla.distributed.xla_multiprocessing as xmp
28 |
29 | # Cell
30 | from fastai.vision.all import *
31 |
32 |
33 | # Cell
34 | from fastai.learner import _ConstantFunc
35 | # from fastcore.basics import patch
36 | # from fastai.learner import Learner
37 |
38 | @patch
39 | def inner_get_preds(self:Learner, ds_idx=1, dl=None, with_input=False, with_decoded=False, with_loss=False, act=None,
40 | inner=False, reorder=True, cbs=None, **kwargs):
41 |
42 | xla_rank = getattr(self,'xla_rank',None)
43 | if xla_rank is None:
44 | return
45 |
46 | if dl is None:
47 | dl = self.dls[ds_idx].new(shuffled=False, drop_last=False)
48 | else:
49 | try: len(dl)
50 | except TypeError as e:
51 | raise TypeError("`dl` is something other than a single `DataLoader` object")
52 | if not isinstance(dl, TPUDistributedDL):
53 | world_size = kwargs.pop('world_size', xm.xrt_world_size())
54 | seed = kwargs.pop('dl_seed',42)
55 | dl = TPUDistributedDL(dl, xla_rank, world_size=world_size, seed=seed)
56 |
57 | if reorder and hasattr(dl, 'get_idxs'):
58 | idxs = dl.dl.get_idxs()
59 | dl = dl.new(get_idxs = _ConstantFunc(idxs))
60 | rank_idxs = dl.get_idxs()
61 | rank_idxs_len = len(rank_idxs)
62 |
63 | #handle save_preds and save_targs across ranks
64 | save_preds = kwargs.pop('save_preds',None)
65 | if save_preds is not None:
66 | if isinstance(save_preds, str):
67 | kwargs['save_preds'] = Path(save_preds + str(xla_rank)) # add rank to filename
68 | elif isinstance(save_preds, Path):
69 | kwargs['save_preds'] = Path(str(save_preds) + str(xla_rank))
70 | kwargs['save_preds'].mkdir(parents=True,exist_ok=True)
71 | save_targs = kwargs.pop('save_targs',None)
72 | if save_targs is not None:
73 | if isinstance(save_targs, str):
74 | kwargs['save_targs'] = Path(save_targs + str(xla_rank)) # add rank to filename
75 | elif isinstance(save_preds, Path):
76 | kwargs['save_targs'] = Path(str(save_targs) + str(xla_rank))
77 | kwargs['save_targs'].mkdir(parents=True,exist_ok=True)
78 |
79 | cb = GatherPredsCallback(with_input=with_input, with_loss=with_loss, **kwargs)
80 | ctx_mgrs = self.validation_context(cbs=L(cbs)+[cb], inner=inner)
81 | if with_loss:
82 | ctx_mgrs.append(self.loss_not_reduced())
83 |
84 | with ContextManagers(ctx_mgrs):
85 | self._do_epoch_validate(dl=dl)
86 |
87 | if act is None:
88 | act = getattr(self.loss_func, 'activation', noop)
89 |
90 | res = cb.all_tensors()
91 |
92 | pred_i = 1 if with_input else 0
93 | if res[pred_i] is not None:
94 | if act != noop:
95 | # compute activation on tpu device and detach after
96 | tmp_pred = res[pred_i].to(xm.xla_device())
97 | tmp_res = act(tmp_pred)
98 | res[pred_i] = self.to_detach(tmp_res)
99 |
100 | if with_decoded:
101 | res.insert(pred_i+2, getattr(self.loss_func, 'decodes', noop)(res[pred_i]))
102 |
103 | if reorder and hasattr(dl, 'get_idxs'):
104 | t_idxs = tensor(rank_idxs)
105 | start_idx = xla_rank * rank_idxs_len
106 | t_idxs = t_idxs - tensor(start_idx) # broadcast
107 | sorted_idxs = t_idxs.argsort()
108 | res = nested_reorder(res, sorted_idxs )
109 |
110 | return tuple(res)
111 | self._end_cleanup()
112 |
113 |
114 | # Cell
115 |
116 | def setup_inference_args(rank, inference_args):
117 | master_cbs = ifnone(inference_args.pop('master_cbs', None),[])
118 | return inference_args, master_cbs
119 |
120 |
121 | # Cell
122 |
123 | import pickle
124 | def save_pred_results(rank, results):
125 | fn = f'preds{rank}.pkl'
126 | fn = Path(fn)
127 | with open(fn,'wb') as f:
128 | pickle.dump(results, f)
129 |
130 | # Cell
131 |
132 | def xla_run_inference(rank, learner_args, add_args, inference_args, ctrl_args):
133 | sync_valid = True
134 | learner = make_xla_child_learner(rank, sync_valid, learner_args, add_args, ctrl_args)
135 | pred_args, master_cbs = setup_inference_args(rank, inference_args)
136 |
137 | if rank == 0 and len(master_cbs) > 0:
138 | learner.add_cbs(master_cbs)
139 |
140 | # learner.synced_cancel.before_fit()
141 |
142 | if rank == 0:
143 | learner.sync_recorder.orig_logger = learner.logger
144 |
145 | results = learner.inner_get_preds(**pred_args)
146 | xm.rendezvous('xla_run_inference')
147 |
148 | save_pred_results(rank, results)
149 | xm.mark_step()
150 |
151 |
152 | # Cell
153 | from fastcore.foundation import L
154 |
155 | def reload_pred_results(num_files, n_samples):
156 | all_preds = L()
157 | for rank in range(num_files):
158 | fn = f'preds{rank}.pkl'
159 |
160 | fn = Path(fn)
161 | if fn.is_file():
162 | with open(fn,'rb') as f:
163 | rank_preds = pickle.load(f)
164 | all_preds.append(rank_preds)
165 | else:
166 | raise RuntimeException(f'Missing preds file for rank {rank}')
167 |
168 | for rank in range(num_files):
169 | fn = f'preds{rank}.pkl'
170 | fn = Path(fn)
171 | fn.unlink()
172 |
173 | n_items = len(all_preds[0]) # num items per preds
174 |
175 | all_res = []
176 | for i in range(n_items):
177 | items = all_preds.itemgot(i)
178 |
179 | if isinstance(items[0], torch.Tensor):
180 | all_items = torch.cat(tuple(items))
181 | elif is_listy(items[0]):
182 | all_items = [*items]
183 | else:
184 | all_items = items
185 | all_res.append(all_items)
186 | res = []
187 | for i, pred in enumerate(all_res):
188 | pred = pred[:n_samples] # take only first
189 | res.append(pred)
190 | return res
191 |
192 |
193 |
194 | # Cell
195 |
196 | @patch
197 | def pre_xla_inference(self:Learner):
198 | ctrl_args = {}
199 | progress_removed = False
200 | if 'progress' in L(self.cbs).attrgot('name'):
201 | self.remove_cbs(ProgressCallback)
202 | progress_removed = True
203 | ctrl_args['use_progress'] = progress_removed
204 | return ctrl_args
205 |
206 | # Cell
207 |
208 | @patch
209 | def post_xla_inference(self:Learner, ctrl_args):
210 | if ctrl_args['use_progress']:
211 | self.add_cbs(ProgressCallback)
212 | self.recorder.reload_attrs()
213 |
214 | # Cell
215 |
216 | def prep_inference_args(**kwargs):
217 | return kwargs
218 |
219 | # Cell
220 |
221 | #export
222 |
223 | @patch
224 | @delegates(Learner.get_preds, but='num_cores,start_method,master_cbs')
225 | def xla_get_preds(self:Learner, ds_idx=1, dl=None,
226 | with_input=False, with_decoded=False,
227 | with_loss=False, act=None, inner=False,
228 | reorder=True, cbs=None, num_cores=8,
229 | start_method='fork', master_cbs=None,**kwargs):
230 | ctrl_args = self.pre_xla_inference()
231 | learner_args, add_args = self.pack_learner_args()
232 |
233 | inference_args = prep_inference_args(ds_idx=ds_idx, dl=dl,
234 | with_input=with_input, with_decoded=with_decoded,
235 | with_loss=with_loss,
236 | act=act, inner=inner,
237 | reorder=reorder,
238 | cbs=cbs, master_cbs=master_cbs, **kwargs)
239 | if dl:
240 | n_results = len(dl.dataset)
241 | else:
242 | n_results = len(self.dls.loaders[ds_idx].dataset)
243 |
244 | xmp.spawn(xla_run_inference,
245 | args=(learner_args, add_args, inference_args, ctrl_args),
246 | nprocs=num_cores,
247 | start_method=start_method)
248 |
249 | all_results = reload_pred_results(num_cores, n_results)
250 | self.post_xla_inference(ctrl_args)
251 | return all_results
252 |
--------------------------------------------------------------------------------
/nbs/99_dev_setup.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "#default_exp dev_setup"
10 | ]
11 | },
12 | {
13 | "cell_type": "markdown",
14 | "metadata": {},
15 | "source": [
16 | "# Development Setup"
17 | ]
18 | },
19 | {
20 | "cell_type": "markdown",
21 | "metadata": {},
22 | "source": [
23 | "[ 
![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
24 | ]
25 | },
26 | {
27 | "cell_type": "code",
28 | "execution_count": null,
29 | "metadata": {},
30 | "outputs": [
31 | {
32 | "name": "stdout",
33 | "output_type": "stream",
34 | "text": [
35 | "\u001b[K |████████████████████████████████| 133.6MB 100kB/s \n",
36 | "\u001b[K |████████████████████████████████| 61kB 3.3MB/s \n",
37 | "\u001b[?25h"
38 | ]
39 | }
40 | ],
41 | "source": [
42 | "#hide\n",
43 | "#colab\n",
44 | "!pip install -Uqq cloud-tpu-client==0.10 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.7-cp36-cp36m-linux_x86_64.whl"
45 | ]
46 | },
47 | {
48 | "cell_type": "code",
49 | "execution_count": null,
50 | "metadata": {},
51 | "outputs": [
52 | {
53 | "name": "stdout",
54 | "output_type": "stream",
55 | "text": [
56 | "Updating fastai...\n",
57 | "Done.\n"
58 | ]
59 | }
60 | ],
61 | "source": [
62 | "#hide\n",
63 | "#colab\n",
64 | "!curl -s https://course19.fast.ai/setup/colab | bash"
65 | ]
66 | },
67 | {
68 | "cell_type": "code",
69 | "execution_count": null,
70 | "metadata": {},
71 | "outputs": [
72 | {
73 | "name": "stdout",
74 | "output_type": "stream",
75 | "text": [
76 | "\u001b[K |████████████████████████████████| 194kB 6.5MB/s \n",
77 | "\u001b[K |████████████████████████████████| 61kB 7.4MB/s \n",
78 | "\u001b[?25h"
79 | ]
80 | }
81 | ],
82 | "source": [
83 | "#hide\n",
84 | "#colab\n",
85 | "# !pip install -Uqq git+https://github.com/fastai/fastai.git \n",
86 | "!pip install -Uqq fastai==2.3.0"
87 | ]
88 | },
89 | {
90 | "cell_type": "code",
91 | "execution_count": null,
92 | "metadata": {},
93 | "outputs": [],
94 | "source": [
95 | "#hide\n",
96 | "#colab\n",
97 | "!pip install -qqq nbdev"
98 | ]
99 | },
100 | {
101 | "cell_type": "code",
102 | "execution_count": null,
103 | "metadata": {},
104 | "outputs": [
105 | {
106 | "name": "stdout",
107 | "output_type": "stream",
108 | "text": [
109 | "torch==1.7.0+cu101\n",
110 | "torch-xla==1.7\n",
111 | "torchsummary==1.5.1\n",
112 | "torchtext==0.3.1\n",
113 | "torchvision==0.8.1+cu101\n",
114 | "fastai==2.2.5\n",
115 | "fastcore==1.3.19\n",
116 | "fastdtw==0.3.4\n",
117 | "fastprogress==1.0.0\n",
118 | "fastrlock==0.5\n"
119 | ]
120 | }
121 | ],
122 | "source": [
123 | "#hide\n",
124 | "!pip freeze | grep torch\n",
125 | "!pip freeze | grep fast\n",
126 | "!pip freeze | grep nbdev"
127 | ]
128 | },
129 | {
130 | "cell_type": "markdown",
131 | "metadata": {},
132 | "source": [
133 | "## `_BaseOptimizer` patches\n"
134 | ]
135 | },
136 | {
137 | "cell_type": "code",
138 | "execution_count": null,
139 | "metadata": {},
140 | "outputs": [],
141 | "source": [
142 | "#hide\n",
143 | "#colab\n",
144 | "from nbdev.showdoc import *"
145 | ]
146 | },
147 | {
148 | "cell_type": "code",
149 | "execution_count": null,
150 | "metadata": {},
151 | "outputs": [],
152 | "source": [
153 | "#export\n",
154 | "from fastai.optimizer import _BaseOptimizer\n",
155 | "from fastcore.basics import patch\n",
156 | "\n",
157 | "@patch\n",
158 | "def __getstate__(self:_BaseOptimizer):\n",
159 | " \"Pickling opt state should include `param_groups` and `defaults` \"\n",
160 | " d = {\n",
161 | " 'state': self.state_dict(),\n",
162 | " 'param_groups': self.param_groups,\n",
163 | " }\n",
164 | " if hasattr(self,'defaults'):\n",
165 | " d['defaults'] = self.defaults\n",
166 | " return d\n",
167 | "\n",
168 | "@patch\n",
169 | "def __setstate__(self:_BaseOptimizer, data):\n",
170 | " \"Pickling opt state should include `param_groups` and `defaults` \"\n",
171 | "\n",
172 | " if 'defaults' in data:\n",
173 | " self.defaults = data['defaults']\n",
174 | " self.load_state_dict(data['state'])\n",
175 | " self.param_groups = data['param_groups']"
176 | ]
177 | },
178 | {
179 | "cell_type": "code",
180 | "execution_count": null,
181 | "metadata": {},
182 | "outputs": [],
183 | "source": [
184 | "#hide_input\n",
185 | "#colab\n",
186 | "show_doc(_BaseOptimizer.__getstate__)\n",
187 | "show_doc(_BaseOptimizer.__setstate__)"
188 | ]
189 | },
190 | {
191 | "cell_type": "markdown",
192 | "metadata": {},
193 | "source": [
194 | "\n",
195 | "Patch the `fastai.optimizer._BaseOptimizer` `__getstate__` and `__setstate__` methods which are used in pickling fastai optimizers. \n",
196 | "\n",
197 | "This should fix the bug where running the learner on multiple TPU cores on XLA triggers an error in which the method `_fetch_gradients(optimizer)` fails in the statement `for param_group in optimizer.__getstate__()['param_groups']:` in the `torch_xla.core.xla_model` module. \n",
198 | "\n",
199 | "The patch modifies the copy constructor to include the param_groups and defaults."
200 | ]
201 | },
202 | {
203 | "cell_type": "code",
204 | "execution_count": null,
205 | "metadata": {},
206 | "outputs": [],
207 | "source": [
208 | "#hide\n",
209 | "#TODO add tests for pickling optimizers"
210 | ]
211 | }
212 | ],
213 | "metadata": {
214 | "kernelspec": {
215 | "display_name": "Python 3 (ipykernel)",
216 | "language": "python",
217 | "name": "python3"
218 | }
219 | },
220 | "nbformat": 4,
221 | "nbformat_minor": 4
222 | }
223 |
--------------------------------------------------------------------------------
/nbs/01_utils.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "#default_exp utils"
10 | ]
11 | },
12 | {
13 | "cell_type": "markdown",
14 | "metadata": {},
15 | "source": [
16 | "# Utils"
17 | ]
18 | },
19 | {
20 | "cell_type": "markdown",
21 | "metadata": {},
22 | "source": [
23 | "