'
13 |
14 | hr_faded: '
'
15 | hr_shaded: '
'
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: "3"
2 | services:
3 | fastai: &fastai
4 | restart: unless-stopped
5 | working_dir: /data
6 | image: fastai/codespaces
7 | logging:
8 | driver: json-file
9 | options:
10 | max-size: 50m
11 | stdin_open: true
12 | tty: true
13 | volumes:
14 | - .:/data/
15 |
16 | notebook:
17 | <<: *fastai
18 | command: bash -c "pip install -e . && jupyter notebook --allow-root --no-browser --ip=0.0.0.0 --port=8080 --NotebookApp.token='' --NotebookApp.password=''"
19 | ports:
20 | - "8080:8080"
21 |
22 | watcher:
23 | <<: *fastai
24 | command: watchmedo shell-command --command nbdev_build_docs --pattern *.ipynb --recursive --drop
25 | network_mode: host # for GitHub Codespaces https://github.com/features/codespaces/
26 |
27 | jekyll:
28 | <<: *fastai
29 | ports:
30 | - "4000:4000"
31 | command: >
32 | bash -c "cp -r docs_src docs
33 | && pip install .
34 | && nbdev_build_docs && cd docs
35 | && bundle i
36 | && chmod -R u+rwx . && bundle exec jekyll serve --host 0.0.0.0"
37 |
--------------------------------------------------------------------------------
/docs/licenses/LICENSE:
--------------------------------------------------------------------------------
1 | /* This license pertains to the docs template, except for the Navgoco jQuery component. */
2 |
3 | The MIT License (MIT)
4 |
5 | Original theme: Copyright (c) 2016 Tom Johnson
6 | Modifications: Copyright (c) 2017 onwards fast.ai, Inc
7 |
8 | Permission is hereby granted, free of charge, to any person obtaining a copy
9 | of this software and associated documentation files (the "Software"), to deal
10 | in the Software without restriction, including without limitation the rights
11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 | copies of the Software, and to permit persons to whom the Software is
13 | furnished to do so, subject to the following conditions:
14 |
15 | The above copyright notice and this permission notice shall be included in all
16 | copies or substantial portions of the Software.
17 |
18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 | SOFTWARE.
25 |
--------------------------------------------------------------------------------
/source_nbs/02_special_tokens.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "# default_exp special_tokens\n",
10 | "%load_ext autoreload\n",
11 | "%autoreload 2"
12 | ]
13 | },
14 | {
15 | "cell_type": "markdown",
16 | "metadata": {},
17 | "source": [
18 | "# Special Tokens\n"
19 | ]
20 | },
21 | {
22 | "cell_type": "code",
23 | "execution_count": null,
24 | "metadata": {},
25 | "outputs": [],
26 | "source": [
27 | "# export\n",
28 | "BOS_TOKEN = '[unused98]'\n",
29 | "EOS_TOKEN = '[unused99]'\n",
30 | "CLS_TOKEN = '[CLS]'\n",
31 | "SPACE_TOKEN = '[unused1]'\n",
32 | "UNK_TOKEN = '[UNK]'\n",
33 | "SPECIAL_TOKENS = [BOS_TOKEN, EOS_TOKEN, CLS_TOKEN, SPACE_TOKEN, UNK_TOKEN]\n",
34 | "TRAIN = 'train'\n",
35 | "EVAL = 'eval'\n",
36 | "PREDICT = 'infer'\n",
37 | "MODAL_LIST = ['image', 'others']"
38 | ]
39 | },
40 | {
41 | "cell_type": "code",
42 | "execution_count": null,
43 | "metadata": {},
44 | "outputs": [],
45 | "source": []
46 | }
47 | ],
48 | "metadata": {
49 | "kernelspec": {
50 | "display_name": "Python 3",
51 | "language": "python",
52 | "name": "python3"
53 | }
54 | },
55 | "nbformat": 4,
56 | "nbformat_minor": 2
57 | }
58 |
--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | repository: JayYip/m3tl
2 | output: web
3 | topnav_title: m3tl
4 | site_title: m3tl
5 | company_name: jayyip
6 | description: BERT for Multi-task Learning
7 | # Set to false to disable KaTeX math
8 | use_math: true
9 | # Add Google analytics id if you have one and want to use it here
10 | google_analytics:
11 | # See http://nbdev.fast.ai/search for help with adding Search
12 | google_search:
13 |
14 | host: 127.0.0.1
15 | # the preview server used. Leave as is.
16 | port: 4000
17 | # the port where the preview is rendered.
18 |
19 | exclude:
20 | - .idea/
21 | - .gitignore
22 | - vendor
23 |
24 | exclude: [vendor]
25 |
26 | highlighter: rouge
27 | markdown: kramdown
28 | kramdown:
29 | input: GFM
30 | auto_ids: true
31 | hard_wrap: false
32 | syntax_highlighter: rouge
33 |
34 | collections:
35 | tooltips:
36 | output: false
37 |
38 | defaults:
39 | -
40 | scope:
41 | path: ""
42 | type: "pages"
43 | values:
44 | layout: "page"
45 | comments: true
46 | search: true
47 | sidebar: home_sidebar
48 | topnav: topnav
49 | -
50 | scope:
51 | path: ""
52 | type: "tooltips"
53 | values:
54 | layout: "page"
55 | comments: true
56 | search: true
57 | tooltip: true
58 |
59 | sidebars:
60 | - home_sidebar
61 |
62 | theme: jekyll-theme-cayman
63 | baseurl: /m3tl/
--------------------------------------------------------------------------------
/docs/feed.xml:
--------------------------------------------------------------------------------
1 | ---
2 | search: exclude
3 | layout: none
4 | ---
5 |
6 |
7 |
8 |
9 | {{ site.title | xml_escape }}
10 | {{ site.description | xml_escape }}
11 | {{ site.time | date_to_rfc822 }}
14 | {{ site.time | date_to_rfc822 }}
15 | Jekyll v{{ jekyll.version }}
16 | {% for post in site.posts limit:10 %}
17 | -
18 |
{{ post.title | xml_escape }}
19 | {{ post.content | xml_escape }}
20 | {{ post.date | date_to_rfc822 }}
21 | {{ post.url | prepend: site.url }}
23 | {% for tag in post.tags %}
24 | {{ tag | xml_escape }}
25 | {% endfor %}
26 | {% for tag in page.tags %}
27 | {{ cat | xml_escape }}
28 | {% endfor %}
29 |
30 | {% endfor %}
31 |
32 |
33 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 |
2 | import codecs
3 | from setuptools import setup, find_packages
4 | from pkg_resources import parse_version
5 | from configparser import ConfigParser
6 | import setuptools
7 | assert parse_version(setuptools.__version__) >= parse_version('36.2')
8 |
9 | # note: all settings are in settings.ini; edit there, not here
10 | config = ConfigParser(delimiters=['='])
11 | config.read('settings.ini')
12 | cfg = config['DEFAULT']
13 |
14 | with codecs.open('README.md', 'r', 'utf8') as reader:
15 | long_description = reader.read()
16 |
17 |
18 | with codecs.open('requirements.txt', 'r', 'utf8') as reader:
19 | install_requires = list(map(lambda x: x.strip(), reader.readlines()))
20 |
21 |
22 | setup(
23 | name='m3tl',
24 | version=cfg['version'],
25 | packages=find_packages(),
26 | url='https://github.com/JayYip/m3tl',
27 | license='MIT',
28 | author='Jay Yip',
29 | author_email='junpang.yip@gmail.com',
30 | description='BERT for Multi-task Learning',
31 | long_description_content_type='text/markdown',
32 | long_description=long_description,
33 | python_requires='>=3.5.0',
34 | install_requires=install_requires,
35 | classifiers=(
36 | "Programming Language :: Python :: 3.6",
37 | "Programming Language :: Python :: 3.5",
38 | "License :: OSI Approved :: MIT License",
39 | "Operating System :: OS Independent",
40 | "Intended Audience :: Developers",
41 | ),
42 | )
43 |
--------------------------------------------------------------------------------
/docs/_includes/links.html:
--------------------------------------------------------------------------------
1 | {% comment %}Get links from each sidebar, as listed in the _config.yml file under sidebars{% endcomment %}
2 |
3 | {% for sidebar in site.sidebars %}
4 | {% for entry in site.data.sidebars[sidebar].entries %}
5 | {% for folder in entry.folders %}
6 | {% for folderitem in folder.folderitems %}
7 | {% if folderitem.url contains "html#" %}
8 | [{{folderitem.url | remove: "/" }}]: {{folderitem.url | remove: "/"}}
9 | {% else %}
10 | [{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}}
11 | {% endif %}
12 | {% for subfolders in folderitem.subfolders %}
13 | {% for subfolderitem in subfolders.subfolderitems %}
14 | [{{subfolderitem.url | remove: "/" | remove: ".html"}}]: {{subfolderitem.url | remove: "/"}}
15 | {% endfor %}
16 | {% endfor %}
17 | {% endfor %}
18 | {% endfor %}
19 | {% endfor %}
20 | {% endfor %}
21 |
22 |
23 | {% comment %} Get links from topnav {% endcomment %}
24 |
25 | {% for entry in site.data.topnav.topnav %}
26 | {% for item in entry.items %}
27 | {% if item.external_url == null %}
28 | [{{item.url | remove: "/" | remove: ".html"}}]: {{item.url | remove: "/"}}
29 | {% endif %}
30 | {% endfor %}
31 | {% endfor %}
32 |
33 | {% comment %}Get links from topnav dropdowns {% endcomment %}
34 |
35 | {% for entry in site.data.topnav.topnav_dropdowns %}
36 | {% for folder in entry.folders %}
37 | {% for folderitem in folder.folderitems %}
38 | {% if folderitem.external_url == null %}
39 | [{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}}
40 | {% endif %}
41 | {% endfor %}
42 | {% endfor %}
43 | {% endfor %}
44 |
45 |
--------------------------------------------------------------------------------
/docs/_includes/head_print.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
{% if page.homepage == true %} {{site.homepage_title}} {% elsif page.title %}{{ page.title }}{% endif %} | {{ site.site_title }}
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
23 |
24 |
29 |
--------------------------------------------------------------------------------
/docs/css/modern-business.css:
--------------------------------------------------------------------------------
1 | /*!
2 | * Start Bootstrap - Modern Business HTML Template (http://startbootstrap.com)
3 | * Code licensed under the Apache License v2.0.
4 | * For details, see http://www.apache.org/licenses/LICENSE-2.0.
5 | */
6 |
7 | /* Global Styles */
8 |
9 | html,
10 | body {
11 | height: 100%;
12 | }
13 |
14 | .img-portfolio {
15 | margin-bottom: 30px;
16 | }
17 |
18 | .img-hover:hover {
19 | opacity: 0.8;
20 | }
21 |
22 | /* Home Page Carousel */
23 |
24 | header.carousel {
25 | height: 50%;
26 | }
27 |
28 | header.carousel .item,
29 | header.carousel .item.active,
30 | header.carousel .carousel-inner {
31 | height: 100%;
32 | }
33 |
34 | header.carousel .fill {
35 | width: 100%;
36 | height: 100%;
37 | background-position: center;
38 | background-size: cover;
39 | }
40 |
41 | /* 404 Page Styles */
42 |
43 | .error-404 {
44 | font-size: 100px;
45 | }
46 |
47 | /* Pricing Page Styles */
48 |
49 | .price {
50 | display: block;
51 | font-size: 50px;
52 | line-height: 50px;
53 | }
54 |
55 | .price sup {
56 | top: -20px;
57 | left: 2px;
58 | font-size: 20px;
59 | }
60 |
61 | .period {
62 | display: block;
63 | font-style: italic;
64 | }
65 |
66 | /* Footer Styles */
67 |
68 | footer {
69 | margin: 50px 0;
70 | }
71 |
72 | /* Responsive Styles */
73 |
74 | @media(max-width:991px) {
75 | .client-img,
76 | .img-related {
77 | margin-bottom: 30px;
78 | }
79 | }
80 |
81 | @media(max-width:767px) {
82 | .img-portfolio {
83 | margin-bottom: 15px;
84 | }
85 |
86 | header.carousel .carousel {
87 | height: 70%;
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/docs/licenses/LICENSE-BSD-NAVGOCO.txt:
--------------------------------------------------------------------------------
1 | /* This license pertains to the Navgoco jQuery component used for the sidebar. */
2 |
3 | Copyright (c) 2013, Christodoulos Tsoulloftas, http://www.komposta.net
4 | All rights reserved.
5 |
6 | Redistribution and use in source and binary forms, with or without modification,
7 | are permitted provided that the following conditions are met:
8 |
9 | * Redistributions of source code must retain the above copyright notice,
10 | this list of conditions and the following disclaimer.
11 | * Redistributions in binary form must reproduce the above copyright notice,
12 | this list of conditions and the following disclaimer in the documentation
13 | and/or other materials provided with the distribution.
14 | * Neither the name of the
nor the names of its
15 | contributors may be used to endorse or promote products derived from this
16 | software without specific prior written permission.
17 |
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
22 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
27 | OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
/docs/sidebar.json:
--------------------------------------------------------------------------------
1 | {
2 | "m3tl": {
3 | "Overview": "/",
4 | "Tutorial": "tutorial.html",
5 | "": {
6 | "Problem Types": {
7 | "Classification": "1_problem_type_cls.html",
8 | "Multi-Label Classification": "2_problem_type_multi_cls.html",
9 | "Sequence Labeling": "3_problem_type_seq_tag.html",
10 | "Masked Language Model": "4_problem_type_masklm.html",
11 | "NSP+MLM(Deprecated)": "6_problem_type_pretrain.html",
12 | "Regression": "7_problem_type_regression.html",
13 | "Vector Fitting": "8_problem_type_vector_fit.html",
14 | "Pre-masked Masked Language Model": "9_problem_type_premask_mlm.html",
15 | "Contrastive Learning": "10_problem_type_contrast_learning.html"
16 | },
17 | "MTL Models": {
18 | "MTLBase": "15-00_mtl_model_base.html",
19 | "MMoE": "15-01_mtl_model_mmoe.html"
20 | },
21 | "Loss Combination Strategy": {
22 | "LossCombinationStrategyBase": "16-00_loss_combination_strategy.html"
23 | },
24 | "Embedding Layers": {
25 | "EmbeddingLayerBase": "18-00_embedding_layer.html"
26 | }
27 | },
28 | "Params": "params.html",
29 | "Run Bert Multitask Learning": "run_bert_multitask.html",
30 | "Multitask Learning Model": "model_fn.html",
31 | "Utils": "utils.html",
32 | "Special Tokens": "special_tokens.html",
33 | "Bert Utils": "bert_utils.html",
34 | "Create Bert Features": "create_bert_features.html",
35 | "Preprocessing Decorator": "preproc_decorator.html",
36 | "Read and Write TFRecord": "read_write_tfrecord.html",
37 | "Function to Create Datasets": "input_fn.html",
38 | "Pre-defined Problems": "predefined_problems_test.html",
39 | "Body Modeling": "modeling.html",
40 | "Top Models": "top.html",
41 | "Test Base": "test_base.html"
42 | }
43 | }
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 | .pytest_cache/
49 |
50 | # Translations
51 | *.mo
52 | *.pot
53 |
54 | # Django stuff:
55 | *.log
56 | local_settings.py
57 | db.sqlite3
58 |
59 | # Flask stuff:
60 | instance/
61 | .webassets-cache
62 |
63 | # Scrapy stuff:
64 | .scrapy
65 |
66 | # Sphinx documentation
67 | docs/_build/
68 |
69 | # PyBuilder
70 | target/
71 |
72 | # Jupyter Notebook
73 | .ipynb_checkpoints
74 |
75 | # pyenv
76 | .python-version
77 |
78 | # celery beat schedule file
79 | celerybeat-schedule
80 |
81 | # SageMath parsed files
82 | *.sage.py
83 |
84 | # Environments
85 | .env
86 | .venv
87 | env/
88 | venv/
89 | ENV/
90 | env.bak/
91 | venv.bak/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
106 | tmp/
107 | **/.DS_Store
108 |
109 | **/Boson*
110 |
111 | data/ctb8.0
112 | data/ontonotes
113 | .vscode
114 | .idea
115 | data/
116 | *_experiments*
117 | chinese_L-12_H-768_A-12/
118 | models/
119 | test.py
120 | cov.xml
--------------------------------------------------------------------------------
/docs/_layouts/page.html:
--------------------------------------------------------------------------------
1 | ---
2 | layout: default
3 | ---
4 |
5 |
12 |
13 | {% if page.simple_map == true %}
14 |
15 |
20 |
21 | {% include custom/{{page.map_name}}.html %}
22 |
23 | {% elsif page.complex_map == true %}
24 |
25 |
30 |
31 | {% include custom/{{page.map_name}}.html %}
32 |
33 | {% endif %}
34 |
35 |
36 |
37 | {% if page.summary %}
38 |
{{page.summary}}
39 | {% endif %}
40 |
41 | {% unless page.toc == false %}
42 | {% include toc.html %}
43 | {% endunless %}
44 |
45 |
46 | {% if site.github_editme_path %}
47 |
48 |
49 |
50 | {% endif %}
51 |
52 | {{content}}
53 |
54 |
65 |
66 |
Open in Colab Open in Colab
3 |
4 |
13 |
14 |
15 |
16 | {{item.title}} {{item.title}} {{item.title}}
38 | {{ folder.title }}
39 |
50 |
51 | {% endfor %}
52 | {% endfor %}
53 | {% if site.google_search %}
54 |
55 | {% include search_google_custom.html %}
56 |
57 | {% endif %}
58 |
59 |
60 |
63 |
--------------------------------------------------------------------------------
/docs/js/jquery.navgoco.min.js:
--------------------------------------------------------------------------------
1 | /*
2 | * jQuery Navgoco Menus Plugin v0.2.1 (2014-04-11)
3 | * https://github.com/tefra/navgoco
4 | *
5 | * Copyright (c) 2014 Chris T (@tefra)
6 | * BSD - https://github.com/tefra/navgoco/blob/master/LICENSE-BSD
7 | */
8 | !function(a){"use strict";var b=function(b,c,d){return this.el=b,this.$el=a(b),this.options=c,this.uuid=this.$el.attr("id")?this.$el.attr("id"):d,this.state={},this.init(),this};b.prototype={init:function(){var b=this;b._load(),b.$el.find("ul").each(function(c){var d=a(this);d.attr("data-index",c),b.options.save&&b.state.hasOwnProperty(c)?(d.parent().addClass(b.options.openClass),d.show()):d.parent().hasClass(b.options.openClass)?(d.show(),b.state[c]=1):d.hide()});var c=a("
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 | {% raw %}
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
BaseParams()
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 | {% endraw %}
55 |
56 | {% raw %}
57 |
58 |
78 | {% endraw %}
79 |
80 | {% raw %}
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
StaticBatchParams() :: BaseParams
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 | {% endraw %}
103 |
104 | {% raw %}
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
DynamicBatchSizeParams() :: BaseParams
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 | {% endraw %}
127 |
128 | {% raw %}
129 |
130 |
131 |
132 |
133 | {% endraw %}
134 |
135 |
" + txt + " ";
66 | else if (this_level <= level){ // higher level than before; end parent ol
67 | for(i = this_level; i < level; i++) {
68 | html += "" + txt + " ";
71 | }
72 | else if (this_level > level) { // lower level than before; expand the previous to contain a ol
73 | for(i = this_level; i > level; i--) {
74 | html += "<"+settings.listType+">"+((i-level == 2) ? "" : " ")
75 | }
76 | html += "" + txt + " ";
77 | }
78 | level = this_level; // update for the next one
79 | });
80 | html += "";
81 | if (!settings.noBackToTopLinks) {
82 | $(document).on('click', '.back-to-top', function() {
83 | $(window).scrollTop(0);
84 | window.location.hash = '';
85 | });
86 | }
87 |
88 | render[settings.showEffect]();
89 | };
90 | })(jQuery);
91 |
--------------------------------------------------------------------------------
/docs/_layouts/default.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | {% include head.html %}
5 |
41 |
46 |
57 | {% if page.datatable == true %}
58 |
59 |
83 |
84 |
85 |
86 | {% assign content_col_size = "col-md-12" %}
87 | {% unless page.hide_sidebar %}
88 |
89 |
92 | {% assign content_col_size = "col-md-9" %}
93 | {% endunless %}
94 |
95 |
96 |
97 | {{content}}
98 |
99 |
100 |
101 |
102 |
103 |
104 |
{{ page.title }} | {{ site.site_title }}
7 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 | {% raw %}
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
TestBase()
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 | {% endraw %}
55 |
56 | {% raw %}
57 |
58 |
59 |
60 |
61 | {% endraw %}
62 |
63 | {% raw %}
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
PysparkTestBase() :: TestBase
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 | {% endraw %}
86 |
87 | {% raw %}
88 |
89 |
90 |
91 |
92 | {% endraw %}
93 |
94 | {% raw %}
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
create_dummy_features_hidden_features(batch_size1hidden_dim768sample_featuresdict=Noneproblemstr=None
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 | {% endraw %}
117 |
118 | {% raw %}
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
test_top_layer(top_classproblemstr, paramsParamssample_featuresdict, hidden_dimint, test_batch_size_listlist=None**kwargs )
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 | {% endraw %}
141 |
142 | {% raw %}
143 |
144 |
145 |
146 |
147 | {% endraw %}
148 |
149 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 | {% raw %}
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
truncate_seq_pair(tokens_atokens_btargetmax_lengthrngNoneis_seqFalse
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 | {% endraw %}
55 |
56 | {% raw %}
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
punc_augument(raw_inputsparams
68 |
69 |
This code is dedicated in memory of a special time.
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 | {% endraw %}
80 |
81 | {% raw %}
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
create_instances_from_document(all_documentsdocument_indexmax_seq_lengthshort_seq_probmasked_lm_probmax_predictions_per_seqvocab_wordsrng
93 |
94 |
Creates TrainingInstance
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 | {% endraw %}
105 |
106 | {% raw %}
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
create_masked_lm_predictions(tokensmasked_lm_probmax_predictions_per_seqvocab_wordsrng
118 |
119 |
Creates the predictions for the masked LM objective.
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 | {% endraw %}
130 |
131 | {% raw %}
132 |
133 |
134 |
135 |
136 | {% endraw %}
137 |
138 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 |
33 |
34 |
M ulti-M odal M ulti-T ask L earning
35 |
Install
36 |
MASKED
37 |
What is it This is a project that uses transformers(based on huggingface transformers) as base model to do multi-modal multi-task learning .
38 |
Why do I need this Multi-task learning(MTL) is gaining more and more attention, especially in deep learning era. It is widely used in NLP, CV, recommendation, etc. However, MTL usually involves complicated data preprocessing, task managing and task interaction. Other open-source projects, like TencentNLP and PyText, supports MTL but in a naive way and it's not straightforward to implement complicated MTL algorithm. In this project, we try to make writing MTL model as easy as single task learning model and further extend MTL to multi-modal multi-task learning. To do so, we expose following MTL related programable module to user:
39 |
40 | problem sampling strategy
41 | loss combination strategy
42 | gradient surgery
43 | model after base model(transformers)
44 |
45 |
Apart from programable modules, we also provide various built-in SOTA MTL algorithms.
46 |
In a word, you can use this project to:
47 |
48 | implement complicated MTL algorithm
49 | do SOTA MTL without diving into details
50 | do multi-modal learning
51 |
52 |
And since we use transformers as base model, you get all the benefits that you can get from transformers!
53 |
What type of problems are supported?
54 |
55 |
56 |
57 | {% raw %}
58 |
59 |
60 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
`cls`: Classification
81 | `multi_cls`: Multi-Label Classification
82 | `seq_tag`: Sequence Labeling
83 | `masklm`: Masked Language Model
84 | `pretrain`: NSP+MLM(Deprecated)
85 | `regression`: Regression
86 | `vector_fit`: Vector Fitting
87 | `premask_mlm`: Pre-masked Masked Language Model
88 | `contrastive_learning`: Contrastive Learning
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 | {% endraw %}
98 |
99 |
100 |
101 |
Get Started Please see tutorials.
102 |
103 |
104 |
105 |
106 |
=0}var self=this;self.matches=function(string,crit){return"string"!=typeof string?!1:(string=string.trim(),doMatch(string,crit))}}module.exports=new LiteralSearchStrategy},{}],4:[function(require,module){module.exports=function(){function findMatches(store,crit,strategy){for(var data=store.get(),i=0;i{title} ',noResultsText:"No results found",limit:10,fuzzy:!1};self.init=function(_opt){validateOptions(_opt),assignOptions(_opt),isJSON(opt.dataSource)?initWithJSON(opt.dataSource):initWithURL(opt.dataSource)}}var Searcher=require("./Searcher"),Templater=require("./Templater"),Store=require("./Store"),JSONLoader=require("./JSONLoader"),searcher=new Searcher,templater=new Templater,store=new Store,jsonLoader=new JSONLoader;window.SimpleJekyllSearch=new SimpleJekyllSearch}(window,document)},{"./JSONLoader":1,"./Searcher":4,"./Store":5,"./Templater":6}]},{},[7]);
2 |
--------------------------------------------------------------------------------
/m3tl/problem_types/pretrain.py:
--------------------------------------------------------------------------------
1 | # AUTOGENERATED! DO NOT EDIT! File to edit: source_nbs/12_6_problem_type_pretrain.ipynb (unless otherwise specified).
2 |
3 | __all__ = ['PreTrain', 'pretrain_get_or_make_label_encoder_fn', 'pretrain_label_handling_fn']
4 |
5 | # Cell
6 | from typing import Dict, List, Tuple
7 |
8 | from loguru import logger
9 | import tensorflow as tf
10 | import transformers
11 | from ..base_params import BaseParams
12 | from .utils import empty_tensor_handling_loss
13 | from ..special_tokens import PREDICT
14 | from ..utils import (LabelEncoder, gather_indexes, get_phase,
15 | variable_summaries)
16 | from transformers import TFSharedEmbeddings
17 |
18 |
19 | # Cell
20 |
21 | class PreTrain(tf.keras.Model):
22 | def __init__(self, params: BaseParams, problem_name: str, input_embeddings: tf.Tensor=None, share_embedding=True):
23 | super(PreTrain, self).__init__(name=problem_name)
24 | self.problem_name = problem_name
25 | self.params = params
26 | self.nsp = transformers.models.bert.modeling_tf_bert.TFBertNSPHead(
27 | self.params.bert_config)
28 |
29 | if share_embedding is False:
30 | self.vocab_size = self.params.bert_config.vocab_size
31 | self.share_embedding = False
32 | else:
33 | word_embedding_weight = input_embeddings.word_embeddings
34 | self.vocab_size = word_embedding_weight.shape[0]
35 | embedding_size = word_embedding_weight.shape[-1]
36 | share_valid = (self.params.bert_config.hidden_size ==
37 | embedding_size)
38 | if not share_valid and self.params.share_embedding:
39 | logger.warning(
40 | 'Share embedding is enabled but hidden_size != embedding_size')
41 | self.share_embedding = self.params.share_embedding & share_valid
42 |
43 | if self.share_embedding:
44 | self.share_embedding_layer = TFSharedEmbeddings(
45 | vocab_size=word_embedding_weight.shape[0], hidden_size=word_embedding_weight.shape[1])
46 | self.share_embedding_layer.build([1])
47 | self.share_embedding_layer.weight = word_embedding_weight
48 | else:
49 | self.share_embedding_layer = tf.keras.layers.Dense(self.vocab_size)
50 |
51 | def call(self,
52 | inputs: Tuple[Dict[str, Dict[str, tf.Tensor]], Dict[str, Dict[str, tf.Tensor]]]) -> Tuple[tf.Tensor, tf.Tensor]:
53 | mode = get_phase()
54 | features, hidden_features = inputs
55 |
56 | # compute logits
57 | nsp_logits = self.nsp(hidden_features['pooled'])
58 |
59 | # masking is done inside the model
60 | seq_hidden_feature = hidden_features['seq']
61 | if mode != PREDICT:
62 | positions = features['masked_lm_positions']
63 |
64 | # gather_indexes will flatten the seq hidden_states, we need to reshape
65 | # back to 3d tensor
66 | input_tensor = gather_indexes(seq_hidden_feature, positions)
67 | shape_tensor = tf.shape(positions)
68 | shape_list = tf.concat(
69 | [shape_tensor, [seq_hidden_feature.shape.as_list()[-1]]], axis=0)
70 | input_tensor = tf.reshape(input_tensor, shape=shape_list)
71 | # set_shape to determin rank
72 | input_tensor.set_shape(
73 | [None, None, seq_hidden_feature.shape.as_list()[-1]])
74 | else:
75 | input_tensor = seq_hidden_feature
76 | if self.share_embedding:
77 | mlm_logits = self.share_embedding_layer(
78 | input_tensor, mode='linear')
79 | else:
80 | mlm_logits = self.share_embedding_layer(input_tensor)
81 |
82 | if self.params.detail_log:
83 | for weight_variable in self.weights:
84 | variable_summaries(weight_variable, self.problem_name)
85 |
86 | if mode != PREDICT:
87 | nsp_labels = features['next_sentence_label_ids']
88 | mlm_labels = features['masked_lm_ids']
89 | mlm_labels.set_shape([None, None])
90 | # compute loss
91 | nsp_loss = empty_tensor_handling_loss(
92 | nsp_labels, nsp_logits,
93 | tf.keras.losses.sparse_categorical_crossentropy)
94 | mlm_loss_layer = transformers.modeling_tf_utils.TFMaskedLanguageModelingLoss()
95 | # mlm_loss = tf.reduce_mean(
96 | # mlm_loss_layer.compute_loss(mlm_labels, mlm_logits))
97 |
98 | # add a useless from_logits argument to match the function signature of keras losses.
99 | def loss_fn_wrapper(labels, logits, from_logits=True):
100 | return mlm_loss_layer.compute_loss(labels, logits)
101 | mlm_loss = empty_tensor_handling_loss(
102 | mlm_labels,
103 | mlm_logits,
104 | loss_fn_wrapper
105 | )
106 | loss = nsp_loss + mlm_loss
107 | self.add_loss(loss)
108 |
109 | return (tf.sigmoid(nsp_logits), tf.nn.softmax(mlm_logits))
110 |
111 | # Cell
112 | def pretrain_get_or_make_label_encoder_fn(params: BaseParams, problem: str, mode: str, label_list: List[str], *args, **kwargs) -> LabelEncoder:
113 | params.set_problem_info(problem=problem, info_name='num_classes', info=1)
114 | return None
115 |
116 | # Cell
117 | def pretrain_label_handling_fn(target, label_encoder=None, tokenizer=None, decoding_length=None, *args, **kwargs):
118 | return None, None
119 |
120 |
--------------------------------------------------------------------------------
/m3tl/problem_types/masklm.py:
--------------------------------------------------------------------------------
1 | # AUTOGENERATED! DO NOT EDIT! File to edit: source_nbs/12_4_problem_type_masklm.ipynb (unless otherwise specified).
2 |
3 | __all__ = ['MaskLM', 'masklm_get_or_make_label_encoder_fn', 'masklm_label_handling_fn']
4 |
5 | # Cell
6 | import pickle
7 | from typing import List
8 |
9 | import tensorflow as tf
10 | from loguru import logger
11 | from ..base_params import BaseParams
12 | from .utils import (empty_tensor_handling_loss,
13 | nan_loss_handling, pad_to_shape)
14 | from ..special_tokens import PREDICT
15 | from ..utils import (LabelEncoder, gather_indexes,
16 | get_label_encoder_save_path, get_phase,
17 | load_transformer_tokenizer, need_make_label_encoder)
18 | from transformers import TFSharedEmbeddings
19 |
20 |
21 | # Cell
22 | class MaskLM(tf.keras.Model):
23 | """Multimodal MLM top layer.
24 | """
25 |
26 | def __init__(self, params: BaseParams, problem_name: str, input_embeddings: tf.keras.layers.Layer=None, share_embedding=True) -> None:
27 | super(MaskLM, self).__init__(name=problem_name)
28 | self.params = params
29 | self.problem_name = problem_name
30 |
31 | if share_embedding is False:
32 | self.vocab_size = self.params.bert_config.vocab_size
33 | self.share_embedding = False
34 | else:
35 | self.vocab_size = input_embeddings.shape[0]
36 | embedding_size = input_embeddings.shape[-1]
37 | share_valid = (self.params.bert_config.hidden_size ==
38 | embedding_size)
39 | if not share_valid and self.params.share_embedding:
40 | logger.warning(
41 | 'Share embedding is enabled but hidden_size != embedding_size')
42 | self.share_embedding = self.params.share_embedding & share_valid
43 |
44 | if self.share_embedding:
45 | self.share_embedding_layer = TFSharedEmbeddings(
46 | vocab_size=self.vocab_size, hidden_size=input_embeddings.shape[1])
47 | self.share_embedding_layer.build([1])
48 | self.share_embedding_layer.weight = input_embeddings
49 | else:
50 | self.share_embedding_layer = tf.keras.layers.Dense(self.vocab_size)
51 |
52 | def call(self, inputs):
53 | mode = get_phase()
54 | features, hidden_features = inputs
55 |
56 | # masking is done inside the model
57 | seq_hidden_feature = hidden_features['seq']
58 | if mode != PREDICT:
59 | positions = features['masked_lm_positions']
60 |
61 | # gather_indexes will flatten the seq hidden_states, we need to reshape
62 | # back to 3d tensor
63 | input_tensor = gather_indexes(seq_hidden_feature, positions)
64 | shape_tensor = tf.shape(positions)
65 | shape_list = tf.concat([shape_tensor, [seq_hidden_feature.shape.as_list()[-1]]], axis=0)
66 | input_tensor = tf.reshape(input_tensor, shape=shape_list)
67 | # set_shape to determin rank
68 | input_tensor.set_shape(
69 | [None, None, seq_hidden_feature.shape.as_list()[-1]])
70 | else:
71 | input_tensor = seq_hidden_feature
72 | if self.share_embedding:
73 | mlm_logits = self.share_embedding_layer(
74 | input_tensor, mode='linear')
75 | else:
76 | mlm_logits = self.share_embedding_layer(input_tensor)
77 | if mode != PREDICT:
78 | mlm_labels = features['masked_lm_ids']
79 | mlm_labels.set_shape([None, None])
80 | mlm_labels = pad_to_shape(from_tensor=mlm_labels, to_tensor=mlm_logits, axis=1)
81 | # compute loss
82 | mlm_loss = empty_tensor_handling_loss(
83 | mlm_labels,
84 | mlm_logits,
85 | tf.keras.losses.sparse_categorical_crossentropy
86 | )
87 | loss = nan_loss_handling(mlm_loss)
88 | self.add_loss(loss)
89 |
90 | return tf.nn.softmax(mlm_logits)
91 |
92 | # Cell
93 | def masklm_get_or_make_label_encoder_fn(params: BaseParams, problem: str, mode: str, label_list: List[str], *args, **kwargs) -> LabelEncoder:
94 |
95 | le_path = get_label_encoder_save_path(params=params, problem=problem)
96 |
97 | if need_make_label_encoder(mode=mode, le_path=le_path, overwrite=kwargs['overwrite']):
98 | # fit and save label encoder
99 | label_encoder = load_transformer_tokenizer(params.transformer_tokenizer_name, params.transformer_tokenizer_loading)
100 | pickle.dump(label_encoder, open(le_path, 'wb'))
101 | try:
102 | params.set_problem_info(problem=problem, info_name='num_classes', info=len(label_encoder.vocab))
103 | except AttributeError:
104 | # models like xlnet's vocab size can only be retrieved from config instead of tokenizer
105 | params.set_problem_info(problem=problem, info_name='num_classes', info=params.bert_config.vocab_size)
106 | else:
107 | # models like xlnet's vocab size can only be retrieved from config instead of tokenizer
108 | params.set_problem_info(problem=problem, info_name='num_classes', info=params.bert_config.vocab_size)
109 | label_encoder = pickle.load(open(le_path, 'rb'))
110 |
111 | return label_encoder
112 |
113 | # Cell
114 | def masklm_label_handling_fn(target, label_encoder=None, tokenizer=None, decoding_length=None, *args, **kwargs):
115 | # masklm is a special case since it modifies inputs
116 | # for more standard implementation of masklm, please see premask_mlm
117 | return None, None
118 |
119 |
--------------------------------------------------------------------------------
/docs/1_problem_type_cls.html:
--------------------------------------------------------------------------------
1 | ---
2 |
3 | title: Classification(cls)
4 |
5 |
6 | keywords: fastai
7 | sidebar: home_sidebar
8 |
9 |
10 |
11 | nb_path: "source_nbs/12_1_problem_type_cls.ipynb"
12 | ---
13 |
22 |
23 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 |
33 |
34 |
Classification. By default this problem will use [CLS] token embedding.
35 |
Example: m3tl.predefined_problems.get_weibo_fake_cls_fn.
36 |
37 |
38 |
39 |
40 |
41 |
42 |
Imports and utils
43 |
44 |
45 |
46 | {% raw %}
47 |
48 |
49 |
50 |
51 | {% endraw %}
52 |
53 |
59 | {% raw %}
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
Classification(*args , **kwargs ) :: Layer
71 |
72 |
Classification Top Layer
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 | {% endraw %}
83 |
84 | {% raw %}
85 |
86 |
87 |
88 |
89 | {% endraw %}
90 |
91 |
92 |
93 |
Get or make label encoder function
94 |
95 |
96 |
97 | {% raw %}
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
cls_get_or_make_label_encoder_fn(paramsBaseParamsproblemstr, modestr, label_listList[str], *args , **kwargs )
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 | {% endraw %}
120 |
121 | {% raw %}
122 |
123 |
124 |
125 |
126 | {% endraw %}
127 |
128 |
129 |
130 |
Label handing function
131 |
132 |
133 |
134 | {% raw %}
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
cls_label_handling_fn[source] cls_label_handling_fn(targetlabel_encoderNonetokenizerNonedecoding_lengthNone*args , **kwargs )
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 | {% endraw %}
157 |
158 | {% raw %}
159 |
160 |
161 |
162 |
163 | {% endraw %}
164 |
165 |
24 |
25 | {% raw %}
26 |
27 |
28 |
29 |
30 | {% endraw %}
31 |
32 | {% raw %}
33 |
34 |
56 | {% endraw %}
57 |
58 |
59 |
60 |
Imports and utils
61 |
62 |
63 |
64 | {% raw %}
65 |
66 |
67 |
68 |
69 | {% endraw %}
70 |
71 |
77 | {% raw %}
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
MaskLM(*args , **kwargs ) :: Model
89 |
90 |
Multimodal MLM top layer.
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 | {% endraw %}
101 |
102 | {% raw %}
103 |
104 |
105 |
106 |
107 | {% endraw %}
108 |
109 |
110 |
111 |
Get or make label encoder function
112 |
113 |
114 |
115 | {% raw %}
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
masklm_get_or_make_label_encoder_fn(paramsBaseParamsproblemstr, modestr, label_listList[str], *args , **kwargs )
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 | {% endraw %}
138 |
139 | {% raw %}
140 |
141 |
142 |
143 |
144 | {% endraw %}
145 |
146 |
147 |
148 |
Label handing function
149 |
150 |
151 |
152 | {% raw %}
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
masklm_label_handling_fn[source] masklm_label_handling_fn(targetlabel_encoderNonetokenizerNonedecoding_lengthNone*args , **kwargs )
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 | {% endraw %}
175 |
176 | {% raw %}
177 |
178 |
179 |
180 |
181 | {% endraw %}
182 |
183 |