├── .github ├── ISSUE_TEMPLATE │ └── 01_bugs.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── cibuildwheel.yml │ ├── explosionbot.yml │ ├── issue-manager.yml │ ├── publish_pypi.yml │ └── tests.yml ├── .gitignore ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── README.md ├── bin └── push-tag.sh ├── build-constraints.txt ├── examples ├── 00_intro_to_thinc.ipynb ├── 01_intro_model_definition_methods.ipynb ├── 02_transformers_tagger_bert.ipynb ├── 03_pos_tagger_basic_cnn.ipynb ├── 03_textcat_basic_neural_bow.ipynb ├── 04_configure_gpu_memory.ipynb ├── 05_benchmarking_layers.ipynb ├── 05_visualizing_models.ipynb ├── 06_predicting_like_terms.ipynb ├── benchmarks │ ├── lstm_tagger.py │ └── mappers.py ├── bloom_embeddings.ipynb ├── mnist.py ├── transformers_tagger.py └── type_checking.py ├── netlify.toml ├── pyproject.toml ├── requirements.txt ├── setup.cfg ├── setup.py ├── thinc ├── __init__.pxd ├── __init__.py ├── about.py ├── api.py ├── backends │ ├── __init__.pxd │ ├── __init__.py │ ├── _accelerate.pxd │ ├── _accelerate.pyx │ ├── _cupy_allocators.py │ ├── _custom_kernels.cu │ ├── _custom_kernels.py │ ├── _murmur3.cu │ ├── _param_server.py │ ├── apple_ops.pyx │ ├── cblas.pxd │ ├── cblas.pyx │ ├── cpu_kernels.hh │ ├── cupy_ops.py │ ├── mps_ops.py │ ├── numpy_ops.pxd │ ├── numpy_ops.pyx │ └── ops.py ├── compat.py ├── config.py ├── initializers.py ├── layers │ ├── __init__.py │ ├── add.py │ ├── array_getitem.py │ ├── bidirectional.py │ ├── cauchysimilarity.py │ ├── chain.py │ ├── clipped_linear.py │ ├── clone.py │ ├── concatenate.py │ ├── dish.py │ ├── dropout.py │ ├── embed.py │ ├── expand_window.py │ ├── gelu.py │ ├── hard_swish.py │ ├── hard_swish_mobilenet.py │ ├── hashembed.py │ ├── layernorm.py │ ├── linear.py │ ├── list2array.py │ ├── list2padded.py │ ├── list2ragged.py │ ├── logistic.py │ ├── lstm.py │ ├── map_list.py │ ├── maxout.py │ ├── mish.py │ ├── multisoftmax.py │ ├── mxnetwrapper.py │ ├── noop.py │ ├── padded2list.py │ ├── parametricattention.py │ ├── parametricattention_v2.py │ ├── premap_ids.pyx │ ├── pytorchwrapper.py │ ├── ragged2list.py │ ├── reduce_first.py │ ├── reduce_last.py │ ├── reduce_max.py │ ├── reduce_mean.py │ ├── reduce_sum.py │ ├── relu.py │ ├── remap_ids.py │ ├── residual.py │ ├── resizable.py │ ├── siamese.py │ ├── sigmoid.py │ ├── sigmoid_activation.py │ ├── softmax.py │ ├── softmax_activation.py │ ├── sparselinear.pyx │ ├── strings2arrays.py │ ├── swish.py │ ├── tensorflowwrapper.py │ ├── torchscriptwrapper.py │ ├── tuplify.py │ ├── uniqued.py │ ├── with_array.py │ ├── with_array2d.py │ ├── with_cpu.py │ ├── with_debug.py │ ├── with_flatten.py │ ├── with_flatten_v2.py │ ├── with_getitem.py │ ├── with_list.py │ ├── with_nvtx_range.py │ ├── with_padded.py │ ├── with_ragged.py │ ├── with_reshape.py │ └── with_signpost_interval.py ├── loss.py ├── model.py ├── mypy.py ├── optimizers.py ├── py.typed ├── schedules.py ├── shims │ ├── __init__.py │ ├── mxnet.py │ ├── pytorch.py │ ├── pytorch_grad_scaler.py │ ├── shim.py │ ├── tensorflow.py │ └── torchscript.py ├── tests │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── _apple_blas │ │ │ ├── __init__.py │ │ │ └── test_gemm.py │ │ ├── test_mem.py │ │ ├── test_mps_ops.py │ │ └── test_ops.py │ ├── conftest.py │ ├── enable_mxnet.py │ ├── enable_tensorflow.py │ ├── layers │ │ ├── __init__.py │ │ ├── test_basic_tagger.py │ │ ├── test_combinators.py │ │ ├── test_feed_forward.py │ │ ├── test_hash_embed.py │ │ ├── test_layers_api.py │ │ ├── test_linear.py │ │ ├── test_lstm.py │ │ ├── test_mappers.py │ │ ├── test_mnist.py │ │ ├── test_mxnet_wrapper.py │ │ ├── test_parametric_attention_v2.py │ │ ├── test_pytorch_wrapper.py │ │ ├── test_reduce.py │ │ ├── test_resizable.py │ │ ├── test_shim.py │ │ ├── test_softmax.py │ │ ├── test_sparse_linear.py │ │ ├── test_tensorflow_wrapper.py │ │ ├── test_torchscriptwrapper.py │ │ ├── test_transforms.py │ │ ├── test_uniqued.py │ │ ├── test_with_debug.py │ │ ├── test_with_flatten.py │ │ └── test_with_transforms.py │ ├── model │ │ ├── __init__.py │ │ ├── test_model.py │ │ └── test_validation.py │ ├── mypy │ │ ├── __init__.py │ │ ├── configs │ │ │ ├── mypy-default.ini │ │ │ └── mypy-plugin.ini │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── fail_no_plugin.py │ │ │ ├── fail_plugin.py │ │ │ ├── success_no_plugin.py │ │ │ └── success_plugin.py │ │ ├── outputs │ │ │ ├── fail-no-plugin.txt │ │ │ ├── fail-plugin.txt │ │ │ ├── success-no-plugin.txt │ │ │ └── success-plugin.txt │ │ └── test_mypy.py │ ├── regression │ │ ├── __init__.py │ │ ├── issue519 │ │ │ ├── __init__.py │ │ │ ├── program.py │ │ │ └── test_issue519.py │ │ ├── test_issue208.py │ │ └── test_issue564.py │ ├── shims │ │ ├── __init__.py │ │ └── test_pytorch_grad_scaler.py │ ├── strategies.py │ ├── test_config.py │ ├── test_examples.py │ ├── test_import__all__.py │ ├── test_indexing.py │ ├── test_initializers.py │ ├── test_loss.py │ ├── test_optimizers.py │ ├── test_schedules.py │ ├── test_serialize.py │ ├── test_types.py │ ├── test_util.py │ └── util.py ├── types.py └── util.py └── website ├── .dockerignore ├── .eslintrc ├── .gitignore ├── .nvmrc ├── .prettierrc ├── Dockerfile ├── README.md ├── docs ├── _quickstart.json ├── _tutorials.json ├── _type_links.json ├── api-backends.md ├── api-config.md ├── api-initializers.md ├── api-layers.md ├── api-loss.md ├── api-model.md ├── api-optimizers.md ├── api-schedules.md ├── api-types.md ├── api-util.md ├── backprop101.md ├── concept.md ├── images │ ├── layer-traversal.graffle │ ├── layer-traversal.svg │ ├── schedules_compounding.svg │ ├── schedules_constant.svg │ ├── schedules_constant_then.svg │ ├── schedules_custom1.svg │ ├── schedules_custom2.svg │ ├── schedules_cyclic_triangular.svg │ ├── schedules_decaying.svg │ ├── schedules_slanted_triangular.svg │ ├── schedules_warmup_linear.svg │ ├── type_checking.jpg │ ├── type_checking2.jpg │ ├── wrapper_pytorch.svg │ └── wrapper_tensorflow.svg ├── index.md ├── install.md ├── usage-config.md ├── usage-frameworks.md ├── usage-models.md ├── usage-sequences.md ├── usage-training.md └── usage-type-checking.md ├── gatsby-browser.js ├── gatsby-config.js ├── gatsby-node.js ├── meta.json ├── package-lock.json ├── package.json ├── plugins ├── gatsby-remark-code-blocks │ ├── index.js │ └── package.json ├── gatsby-remark-custom-attrs │ ├── index.js │ └── package.json └── gatsby-remark-unwrap │ ├── index.js │ └── package.json └── src ├── components ├── box.js ├── code.js ├── dropdown.js ├── footer.js ├── grid.js ├── icon.js ├── landing.js ├── layout.js ├── link.js ├── quickstart.js ├── seo.js ├── sidebar.js ├── table.js ├── tabs.js ├── tutorials.js └── typography.js ├── images ├── colab-badge.svg ├── divider.png ├── docs_bottom-left.png ├── icon.png ├── icons │ ├── arrow-right.svg │ ├── cube.svg │ ├── file.svg │ ├── github.svg │ ├── no.svg │ ├── twitter.svg │ └── yes.svg ├── landing_top-left.png ├── landing_top-right.png ├── logo.svg ├── logos │ ├── explosion.svg │ ├── mxnet.svg │ ├── pytorch.svg │ ├── spacy.svg │ └── tensorflow.svg └── social.jpg ├── markdown.js ├── pages ├── 404.js └── index.js ├── styles ├── base.sass ├── box.module.sass ├── code.module.sass ├── docs.module.sass ├── dropdown.module.sass ├── fonts.sass ├── footer.module.sass ├── grid.module.sass ├── icon.module.sass ├── landing.module.sass ├── link.module.sass ├── quickstart.module.sass ├── sidebar.module.sass ├── table.module.sass ├── tabs.module.sass ├── tutorials.module.sass └── typography.module.sass ├── templates └── docs.js └── util.js /.github/ISSUE_TEMPLATE/01_bugs.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F6A8 Submit a Bug Report" 3 | about: Use this template if you came across a bug or unexpected behaviour differing from the docs. 4 | --- 5 | 6 | ## How to reproduce the behaviour 7 | 8 | 9 | 10 | ## Your Environment 11 | 12 | 13 | 14 | - Operating System: 15 | - Python Version Used: 16 | - Thinc Version Used: 17 | - Environment Information: 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 10 | 11 | ### Types of change 12 | 13 | 15 | 16 | ## Checklist 17 | 18 | 20 | 21 | - [ ] I confirm that I have the right to submit this contribution under the project's MIT license. 22 | - [ ] I ran the tests, and all new and existing tests passed. 23 | - [ ] My changes don't require a change to the documentation, or if they do, I've added all required information. 24 | -------------------------------------------------------------------------------- /.github/workflows/explosionbot.yml: -------------------------------------------------------------------------------- 1 | name: Explosion Bot 2 | 3 | on: 4 | issue_comment: 5 | types: 6 | - created 7 | - edited 8 | 9 | jobs: 10 | explosion-bot: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Dump GitHub context 14 | env: 15 | GITHUB_CONTEXT: ${{ toJson(github) }} 16 | run: echo "$GITHUB_CONTEXT" 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-python@v5 19 | - name: Install and run explosion-bot 20 | run: | 21 | pip install git+https://${{ secrets.EXPLOSIONBOT_TOKEN }}@github.com/explosion/explosion-bot 22 | python -m explosionbot 23 | env: 24 | INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }} 25 | INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }} 26 | ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu" 27 | ALLOWED_TEAMS: "spacy-maintainers" 28 | -------------------------------------------------------------------------------- /.github/workflows/issue-manager.yml: -------------------------------------------------------------------------------- 1 | name: Issue Manager 2 | 3 | on: 4 | schedule: 5 | - cron: "0 0 * * *" 6 | issue_comment: 7 | types: 8 | - created 9 | - edited 10 | issues: 11 | types: 12 | - labeled 13 | 14 | jobs: 15 | issue-manager: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: tiangolo/issue-manager@0.4.0 19 | with: 20 | token: ${{ secrets.GITHUB_TOKEN }} 21 | config: > 22 | { 23 | "resolved": { 24 | "delay": "P7D", 25 | "message": "This issue has been automatically closed because it was answered and there was no follow-up discussion.", 26 | "remove_label_on_comment": true, 27 | "remove_label_on_close": true 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/publish_pypi.yml: -------------------------------------------------------------------------------- 1 | # The cibuildwheel action triggers on creation of a release, this 2 | # triggers on publication. 3 | # The expected workflow is to create a draft release and let the wheels 4 | # upload, and then hit 'publish', which uploads to PyPi. 5 | 6 | on: 7 | release: 8 | types: 9 | - published 10 | 11 | jobs: 12 | upload_pypi: 13 | runs-on: ubuntu-latest 14 | environment: 15 | name: pypi 16 | url: https://pypi.org/p/thinc 17 | permissions: 18 | id-token: write 19 | contents: read 20 | if: github.event_name == 'release' && github.event.action == 'published' 21 | # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) 22 | # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') 23 | steps: 24 | - uses: robinraju/release-downloader@v1 25 | with: 26 | tag: ${{ github.event.release.tag_name }} 27 | fileName: '*' 28 | out-file-path: 'dist' 29 | - uses: pypa/gh-action-pypi-publish@release/v1 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # File extensions 2 | *.pdf 3 | *.aux 4 | *.orig 5 | *.pyo 6 | *.pickle 7 | *.dvi 8 | *.o 9 | *.sqlite 10 | .*.s* 11 | *.dat 12 | 13 | # Varia 14 | _paths.py 15 | .mypy_cache 16 | .hypothesis/ 17 | version.cc 18 | version.pl 19 | 20 | # Tests 21 | tests/model/* 22 | tests/model/ 23 | 24 | # Website 25 | website/.cache/ 26 | website/public/ 27 | website/node_modules 28 | website/.npm 29 | website/logs 30 | *.log 31 | npm-debug.log* 32 | website/www/ 33 | website/_deploy.sh 34 | *.html 35 | 36 | # Cython / C extensions 37 | cythonize.json 38 | *.cpp 39 | *.so 40 | *.so.1 41 | 42 | # Vim / VSCode / editors 43 | *.swp 44 | *.swo 45 | *.sw* 46 | Profile.prof 47 | .vscode 48 | .sass-cache 49 | 50 | # Python 51 | .Python 52 | .python-version 53 | __pycache__/ 54 | .pytest_cache 55 | *.py[cod] 56 | .env/ 57 | .env* 58 | .~env/ 59 | .venv 60 | env3.*/ 61 | venv/ 62 | .dev 63 | .denv 64 | .pypyenv 65 | .pytest_cache/ 66 | 67 | # Distribution / packaging 68 | env/ 69 | build/ 70 | develop-eggs/ 71 | dist/ 72 | eggs/ 73 | lib/ 74 | lib64/ 75 | parts/ 76 | sdist/ 77 | var/ 78 | *.egg-info/ 79 | pip-wheel-metadata/ 80 | Pipfile.lock 81 | .installed.cfg 82 | *.egg 83 | .eggs 84 | MANIFEST 85 | 86 | # Temporary files 87 | *.~* 88 | tmp/ 89 | predict 90 | train 91 | 92 | # Installer logs 93 | pip-log.txt 94 | pip-delete-this-directory.txt 95 | 96 | # Unit test / coverage reports 97 | htmlcov/ 98 | .tox/ 99 | .coverage 100 | .cache 101 | nosetests.xml 102 | coverage.xml 103 | 104 | # Translations 105 | *.mo 106 | 107 | # Mr Developer 108 | .mr.developer.cfg 109 | .project 110 | .pydevproject 111 | 112 | # Rope 113 | .ropeproject 114 | 115 | # Django stuff: 116 | *.log 117 | *.pot 118 | 119 | # Windows 120 | *.bat 121 | Thumbs.db 122 | Desktop.ini 123 | 124 | # Mac OS X 125 | *.DS_Store 126 | 127 | # Komodo project files 128 | *.komodoproject 129 | 130 | # Other 131 | *.tgz 132 | 133 | # Pycharm project files 134 | *.idea 135 | 136 | # IPython 137 | .ipynb_checkpoints/ 138 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: If you use Thinc in research, please use this as a citation. 3 | title: Thinc 4 | abstract: "🔮 A refreshing functional take on deep learning, compatible with your favorite libraries" 5 | authors: 6 | - family-names: "Honnibal" 7 | given-names: "Matthew" 8 | - family-names: "Montani" 9 | given-names: "Ines" 10 | - family-names: "Van Landeghem" 11 | given-names: "Sofie" 12 | - family-names: "Boyd" 13 | given-names: "Adriane" 14 | - family-names: "DuJardin" 15 | given-names: "Justin" 16 | version: 8.0.0 17 | date-released: "2021-01-21" 18 | license: MIT 19 | repository-code: "https://github.com/explosion/thinc" 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (C) 2016 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include thinc *.cu *.pyx *.pxd *.hh 2 | include LICENSE 3 | include README.md 4 | prune tmp/ 5 | include thinc/tests/mypy/configs/*.ini 6 | include thinc/tests/mypy/outputs/*.txt 7 | include thinc/py.typed 8 | recursive-exclude thinc *.cpp 9 | -------------------------------------------------------------------------------- /bin/push-tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # Insist repository is clean 6 | git diff-index --quiet HEAD 7 | 8 | git checkout $1 9 | git pull origin $1 10 | git push origin $1 11 | 12 | version=$(grep "__version__ = " thinc/about.py) 13 | version=${version/__version__ = } 14 | version=${version/\'/} 15 | version=${version/\'/} 16 | version=${version/\"/} 17 | version=${version/\"/} 18 | git tag "v$version" 19 | git push origin "v$version" 20 | -------------------------------------------------------------------------------- /build-constraints.txt: -------------------------------------------------------------------------------- 1 | # build version constraints for use with wheelwright + multibuild 2 | numpy>=2.0.0,<3.0.0; python_version>='3.9' 3 | -------------------------------------------------------------------------------- /examples/mnist.py: -------------------------------------------------------------------------------- 1 | """ 2 | PyTorch version: https://github.com/pytorch/examples/blob/master/mnist/main.py 3 | TensorFlow version: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist.py 4 | """ 5 | # pip install thinc ml_datasets typer 6 | from thinc.api import Model, chain, Relu, Softmax, Adam 7 | import ml_datasets 8 | from wasabi import msg 9 | from tqdm import tqdm 10 | import typer 11 | 12 | 13 | def main( 14 | n_hidden: int = 256, dropout: float = 0.2, n_iter: int = 10, batch_size: int = 128 15 | ): 16 | # Define the model 17 | model: Model = chain( 18 | Relu(nO=n_hidden, dropout=dropout), 19 | Relu(nO=n_hidden, dropout=dropout), 20 | Softmax(), 21 | ) 22 | # Load the data 23 | (train_X, train_Y), (dev_X, dev_Y) = ml_datasets.mnist() 24 | # Set any missing shapes for the model. 25 | model.initialize(X=train_X[:5], Y=train_Y[:5]) 26 | train_data = model.ops.multibatch(batch_size, train_X, train_Y, shuffle=True) 27 | dev_data = model.ops.multibatch(batch_size, dev_X, dev_Y) 28 | # Create the optimizer. 29 | optimizer = Adam(0.001) 30 | for i in range(n_iter): 31 | for X, Y in tqdm(train_data, leave=False): 32 | Yh, backprop = model.begin_update(X) 33 | backprop(Yh - Y) 34 | model.finish_update(optimizer) 35 | # Evaluate and print progress 36 | correct = 0 37 | total = 0 38 | for X, Y in dev_data: 39 | Yh = model.predict(X) 40 | correct += (Yh.argmax(axis=1) == Y.argmax(axis=1)).sum() 41 | total += Yh.shape[0] 42 | score = correct / total 43 | msg.row((i, f"{score:.3f}"), widths=(3, 5)) 44 | 45 | 46 | if __name__ == "__main__": 47 | typer.run(main) 48 | -------------------------------------------------------------------------------- /examples/type_checking.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from thinc.layers import Relu, Softmax, chain, reduce_max, concatenate 4 | from thinc.model import Model 5 | 6 | # Define Custom X/Y types 7 | MyModelX = List[List[float]] 8 | MyModelY = List[List[float]] 9 | model: Model[MyModelX, MyModelY] = chain( 10 | Relu(12), Relu(12, dropout=0.2), Softmax(), 11 | ) 12 | # ERROR: incompatible type "bool", expected "List[List[float]]" 13 | model(False) 14 | # ERROR: List item 0 has incompatible type "str"; expected "float" 15 | model.begin_update([["0"]]) 16 | # ERROR: incompatible type "bool", expected "List[List[float]]" 17 | model.predict(True) 18 | 19 | 20 | # This example should be run with mypy. This is an example of type-level checking 21 | # for network validity. 22 | # 23 | # We first define an invalid network. 24 | # It's invalid because reduce_max expects Array3d as input, while Relu produces 25 | # Array2d as output. chain has type-logic to verify input and output types 26 | # line up. 27 | # 28 | # You should see the error an error, 29 | # examples/howto/type_chain.py:10: error: Cannot infer type argument 2 of "chain" 30 | bad_model = chain(Relu(10), reduce_max(), Softmax()) 31 | 32 | concate_model = concatenate(Relu(10), reduce_max(), Relu(10), Relu(10)), reduce_max() 33 | 34 | concate_chain_model = chain( 35 | concatenate(Relu(10), reduce_max(), Relu(10), Relu(10)), reduce_max() 36 | ) 37 | 38 | # Now let's try it with a network that does work, just to be sure. 39 | good_model = chain(Relu(10), Relu(10), Softmax()) 40 | 41 | # Finally we can reveal_type on the good model, to see what it thinks. 42 | reveal_type(good_model) 43 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | base = "website" 3 | publish = "public" 4 | command = "npm run deploy" 5 | ignore = "false" 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools", 4 | "cython>=0.25,<3.0", 5 | "murmurhash>=1.0.2,<1.1.0", 6 | "cymem>=2.0.2,<2.1.0", 7 | "preshed>=3.0.2,<3.1.0", 8 | "blis>=1.0.0,<1.1.0", 9 | "numpy>=2.0.0,<3.0.0; python_version < '3.9'", 10 | "numpy>=2.0.0,<3.0.0; python_version >= '3.9'", 11 | ] 12 | build-backend = "setuptools.build_meta" 13 | 14 | [tool.cibuildwheel] 15 | build = "*" 16 | skip = "pp* cp36* cp37* cp38*" 17 | test-skip = "" 18 | free-threaded-support = false 19 | 20 | archs = ["native"] 21 | 22 | build-frontend = "default" 23 | config-settings = {} 24 | dependency-versions = "pinned" 25 | environment = {} 26 | environment-pass = [] 27 | build-verbosity = 0 28 | 29 | before-all = "" 30 | before-build = "" 31 | repair-wheel-command = "" 32 | 33 | test-command = "" 34 | before-test = "" 35 | test-requires = [] 36 | test-extras = [] 37 | 38 | container-engine = "docker" 39 | 40 | manylinux-x86_64-image = "manylinux2014" 41 | manylinux-i686-image = "manylinux2014" 42 | manylinux-aarch64-image = "manylinux2014" 43 | manylinux-ppc64le-image = "manylinux2014" 44 | manylinux-s390x-image = "manylinux2014" 45 | manylinux-pypy_x86_64-image = "manylinux2014" 46 | manylinux-pypy_i686-image = "manylinux2014" 47 | manylinux-pypy_aarch64-image = "manylinux2014" 48 | 49 | musllinux-x86_64-image = "musllinux_1_2" 50 | musllinux-i686-image = "musllinux_1_2" 51 | musllinux-aarch64-image = "musllinux_1_2" 52 | musllinux-ppc64le-image = "musllinux_1_2" 53 | musllinux-s390x-image = "musllinux_1_2" 54 | 55 | [tool.cibuildwheel.linux] 56 | repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" 57 | 58 | [tool.cibuildwheel.macos] 59 | repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" 60 | 61 | [tool.cibuildwheel.windows] 62 | 63 | [tool.cibuildwheel.pyodide] 64 | 65 | 66 | [tool.isort] 67 | profile = "black" 68 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Explosion-provided dependencies 2 | murmurhash>=1.0.2,<1.1.0 3 | cymem>=2.0.2,<2.1.0 4 | preshed>=3.0.2,<3.1.0 5 | blis>=1.0.0,<1.1.0 6 | srsly>=2.4.0,<3.0.0 7 | wasabi>=0.8.1,<1.2.0 8 | catalogue>=2.0.4,<2.1.0 9 | confection>=0.0.1,<1.0.0 10 | ml_datasets>=0.2.0,<0.3.0; python_version < "3.11" 11 | # Third-party dependencies 12 | pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0 13 | numpy>=2.0.0,<3.0.0 14 | packaging>=20.0 15 | # Development dependencies 16 | cython>=0.25.0,<3.0 17 | hypothesis>=3.27.0,<6.72.2 18 | pytest>=8.2.0 19 | pytest-cov>=2.7.0,<5.0.0 20 | coverage>=5.0.0,<8.0.0 21 | mock>=2.0.0,<3.0.0 22 | flake8>=3.5.0,<3.6.0 23 | mypy>=1.5.0,<1.6.0; platform_machine != "aarch64" 24 | types-mock>=0.1.1 25 | # Executing notebook tests 26 | ipykernel>=5.1.4,<5.2.0 27 | nbconvert>=5.6.1,<6.2.0 28 | nbformat>=5.0.4,<5.2.0 29 | # Test to_disk/from_disk against pathlib.Path subclasses 30 | pathy>=0.3.5 31 | black>=22.0,<23.0 32 | isort>=5.0,<6.0 33 | -------------------------------------------------------------------------------- /thinc/__init__.pxd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/__init__.pxd -------------------------------------------------------------------------------- /thinc/__init__.py: -------------------------------------------------------------------------------- 1 | # Necessary for some side-effects in Cython. Not sure I understand. 2 | import numpy 3 | 4 | from .about import __version__ 5 | from .config import registry 6 | 7 | # fmt: off 8 | __all__ = [ 9 | "registry", 10 | "__version__", 11 | ] 12 | # fmt: on 13 | -------------------------------------------------------------------------------- /thinc/about.py: -------------------------------------------------------------------------------- 1 | __version__ = "9.1.1" 2 | __release__ = True 3 | -------------------------------------------------------------------------------- /thinc/backends/__init__.pxd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/backends/__init__.pxd -------------------------------------------------------------------------------- /thinc/backends/_accelerate.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "Accelerate/Accelerate.h": 2 | enum CBLAS_ORDER: CblasRowMajor, CblasColMajor 3 | enum CBLAS_TRANSPOSE: CblasNoTrans, CblasTrans, CblasConjTrans 4 | enum CBLAS_UPLO: CblasUpper, CblasLower 5 | enum CBLAS_DIAG: CblasNonUnit, CblasUnit 6 | enum CBLAS_SIDE: CblasLeft, CblasRight 7 | 8 | # BLAS level 1 routines 9 | 10 | void cblas_sswap(int M, float *x, int incX, float *y, int incY) nogil 11 | void cblas_sscal(int N, float alpha, float *x, int incX) nogil 12 | void cblas_scopy(int N, float *x, int incX, float *y, int incY) nogil 13 | void cblas_saxpy(int N, float alpha, float *x, int incX, float *y, int incY ) nogil 14 | float cblas_sdot(int N, float *x, int incX, float *y, int incY ) nogil 15 | float cblas_snrm2(int N, float *x, int incX) nogil 16 | float cblas_sasum(int N, float *x, int incX) nogil 17 | int cblas_isamax(int N, float *x, int incX) nogil 18 | 19 | # BLAS level 2 routines 20 | void cblas_sgemv(CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA, int M, int N, 21 | float alpha, float *A, int lda, float *x, int incX, 22 | float beta, float *y, int incY) nogil 23 | 24 | void cblas_sger(CBLAS_ORDER Order, int M, int N, float alpha, float *x, 25 | int incX, float *y, int incY, float *A, int lda) nogil 26 | 27 | # BLAS level 3 routines 28 | void cblas_sgemm(CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA, 29 | CBLAS_TRANSPOSE TransB, int M, int N, int K, 30 | float alpha, float *A, int lda, float *B, int ldb, 31 | float beta, float *C, int ldc) nogil 32 | 33 | 34 | cdef void sgemm(bint TransA, bint TransB, int M, int N, int K, 35 | float alpha, const float* A, int lda, const float *B, 36 | int ldb, float beta, float* C, int ldc) nogil 37 | 38 | 39 | cdef void saxpy(int N, float alpha, const float* X, int incX, 40 | float *Y, int incY) nogil 41 | -------------------------------------------------------------------------------- /thinc/backends/_accelerate.pyx: -------------------------------------------------------------------------------- 1 | cimport numpy as np 2 | from libc.stdint cimport uintptr_t 3 | 4 | import numpy 5 | 6 | 7 | cpdef np.ndarray gemm(float[:, ::1] A, float[:, ::1] B, 8 | bint trans1=False, bint trans2=False, 9 | np.ndarray out=None): 10 | cdef int nM = A.shape[0] if not trans1 else A.shape[1] 11 | cdef int nK = A.shape[1] if not trans1 else A.shape[0] 12 | cdef int nK_b = B.shape[0] if not trans2 else B.shape[1] 13 | cdef int nN = B.shape[1] if not trans2 else B.shape[0] 14 | 15 | cdef float[:, ::1] C = out 16 | 17 | if out is None: 18 | out = numpy.empty((nM, nN), dtype="f") 19 | C = out 20 | else: 21 | if C.shape[0] != nM or C.shape[1] != nN: 22 | msg = "Shape mismatch for output matrix, was: (%d, %d), expected (%d, %d)" 23 | raise ValueError(msg % (C.shape[0], C.shape[1], nM, nN)) 24 | 25 | 26 | if nK != nK_b: 27 | msg = "Shape mismatch for gemm: (%d, %d), (%d, %d)" 28 | raise ValueError(msg % (nM, nK, nK_b, nN)) 29 | 30 | if nM == 0 or nK == 0 or nN == 0: 31 | return out 32 | 33 | cblas_sgemm( 34 | CblasRowMajor, 35 | CblasTrans if trans1 else CblasNoTrans, 36 | CblasTrans if trans2 else CblasNoTrans, 37 | nM, 38 | nN, 39 | nK, 40 | 1.0, 41 | &A[0, 0], 42 | A.shape[1], 43 | &B[0, 0], 44 | B.shape[1], 45 | 0.0, 46 | &C[0, 0], 47 | C.shape[1] 48 | ) 49 | return out 50 | 51 | 52 | cdef void sgemm(bint TransA, bint TransB, int M, int N, int K, 53 | float alpha, const float* A, int lda, const float *B, 54 | int ldb, float beta, float* C, int ldc) nogil: 55 | cblas_sgemm( 56 | CblasRowMajor, 57 | CblasTrans if TransA else CblasNoTrans, 58 | CblasTrans if TransB else CblasNoTrans, 59 | M, 60 | N, 61 | K, 62 | alpha, 63 | A, 64 | lda, 65 | B, 66 | ldb, 67 | beta, 68 | C, 69 | ldc 70 | ) 71 | 72 | 73 | cdef void saxpy(int N, float alpha, const float* X, int incX, 74 | float *Y, int incY) nogil: 75 | cblas_saxpy(N, alpha, X, incX, Y, incY) 76 | -------------------------------------------------------------------------------- /thinc/backends/_cupy_allocators.py: -------------------------------------------------------------------------------- 1 | from typing import cast 2 | 3 | from ..compat import cupy, tensorflow, torch 4 | from ..types import ArrayXd 5 | from ..util import get_torch_default_device, tensorflow2xp 6 | 7 | 8 | def cupy_tensorflow_allocator(size_in_bytes: int): 9 | """Function that can be passed into cupy.cuda.set_allocator, to have cupy 10 | allocate memory via TensorFlow. This is important when using the two libraries 11 | together, as otherwise OOM errors can occur when there's available memory 12 | sitting in the other library's pool. 13 | """ 14 | size_in_bytes = max(1024, size_in_bytes) 15 | tensor = tensorflow.zeros((size_in_bytes // 4,), dtype=tensorflow.dtypes.float32) # type: ignore 16 | # We convert to cupy via dlpack, so that we can get a memory pointer. 17 | cupy_array = cast(ArrayXd, tensorflow2xp(tensor)) 18 | address = int(cupy_array.data) 19 | # cupy has a neat class to help us here. Otherwise it will try to free. 20 | memory = cupy.cuda.memory.UnownedMemory(address, size_in_bytes, cupy_array) 21 | # Now return a new memory pointer. 22 | return cupy.cuda.memory.MemoryPointer(memory, 0) 23 | 24 | 25 | def cupy_pytorch_allocator(size_in_bytes: int): 26 | device = get_torch_default_device() 27 | """Function that can be passed into cupy.cuda.set_allocator, to have cupy 28 | allocate memory via PyTorch. This is important when using the two libraries 29 | together, as otherwise OOM errors can occur when there's available memory 30 | sitting in the other library's pool. 31 | """ 32 | # Cupy was having trouble with very small allocations? 33 | size_in_bytes = max(1024, size_in_bytes) 34 | # We use pytorch's underlying FloatStorage type to avoid overhead from 35 | # creating a whole Tensor. 36 | # This turns out to be way faster than making FloatStorage? Maybe 37 | # a Python vs C++ thing I guess? 38 | torch_tensor = torch.zeros( 39 | (size_in_bytes // 4,), requires_grad=False, device=device 40 | ) 41 | # cupy has a neat class to help us here. Otherwise it will try to free. 42 | # I think this is a private API? It's not in the types. 43 | address = torch_tensor.data_ptr() # type: ignore 44 | memory = cupy.cuda.memory.UnownedMemory(address, size_in_bytes, torch_tensor) 45 | # Now return a new memory pointer. 46 | return cupy.cuda.memory.MemoryPointer(memory, 0) 47 | -------------------------------------------------------------------------------- /thinc/backends/apple_ops.pyx: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import numpy 4 | 5 | from ._accelerate import gemm 6 | 7 | from ._accelerate cimport saxpy, sgemm 8 | from .cblas cimport CBlas, set_saxpy, set_sgemm 9 | 10 | from .. import registry 11 | from ..types import Floats2d 12 | from .numpy_ops import NumpyOps 13 | 14 | 15 | @registry.ops("AppleOps") 16 | class AppleOps(NumpyOps): 17 | """Thinc Ops class that calls into Apple's native libraries for some 18 | operations. Other operations fall back to numpy.""" 19 | name = "apple" 20 | xp = numpy 21 | 22 | def cblas(self) -> CBlas: 23 | cdef CBlas cblas = CBlas() 24 | set_saxpy(cblas, saxpy) 25 | set_sgemm(cblas, sgemm) 26 | return cblas 27 | 28 | def gemm( 29 | self, 30 | x: Floats2d, 31 | y: Floats2d, 32 | out: Optional[Floats2d] = None, 33 | trans1: bool = False, 34 | trans2: bool = False, 35 | ) -> Floats2d: 36 | """Perform General Matrix Multiplication (GeMM) and optionally store 37 | the result in the specified output variable. 38 | """ 39 | return gemm(x, y, out=out, trans1=trans1, trans2=trans2) 40 | -------------------------------------------------------------------------------- /thinc/backends/cblas.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.memory cimport shared_ptr 2 | 3 | ctypedef void (*sgemm_ptr)(bint transA, bint transB, int M, int N, int K, 4 | float alpha, const float* A, int lda, const float* B, 5 | int ldb, float beta, float* C, int ldc) nogil 6 | ctypedef void (*dgemm_ptr)(bint transA, bint transB, int M, int N, int K, 7 | double alpha, const double* A, int lda, const double* B, 8 | int ldb, double beta, double* C, int ldc) nogil 9 | 10 | 11 | ctypedef void (*saxpy_ptr)(int N, float alpha, const float* X, int incX, 12 | float *Y, int incY) nogil 13 | 14 | 15 | ctypedef void (*daxpy_ptr)(int N, double alpha, const double* X, int incX, 16 | double *Y, int incY) nogil 17 | 18 | ctypedef void (*sscal_ptr)(int N, float alpha, float* X, int incX) nogil 19 | ctypedef void (*dscal_ptr)(int N, double alpha, double* X, int incX) nogil 20 | 21 | # Forward-declaration of the BlasFuncs struct. This struct must be opaque, so 22 | # that consumers of the CBlas class cannot become dependent on its size or 23 | # ordering. 24 | cdef struct BlasFuncs 25 | 26 | 27 | cdef class CBlas: 28 | cdef shared_ptr[BlasFuncs] ptr 29 | 30 | 31 | # Note: the following functions are intentionally standalone. If we make them 32 | # methods of CBlas, Cython will generate and use a vtable. This makes it 33 | # impossible to add new BLAS functions later without breaking the ABI. 34 | # 35 | # See https://github.com/explosion/thinc/pull/700 for more information. 36 | 37 | cdef daxpy_ptr daxpy(CBlas cblas) nogil 38 | cdef saxpy_ptr saxpy(CBlas cblas) nogil 39 | cdef sgemm_ptr sgemm(CBlas cblas) nogil 40 | cdef dgemm_ptr dgemm(CBlas cblas) nogil 41 | cdef sscal_ptr sscal(CBlas cblas) nogil 42 | cdef dscal_ptr dscal(CBlas cblas) nogil 43 | cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) nogil 44 | cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) nogil 45 | cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) nogil 46 | cdef void set_dgemm(CBlas cblas, dgemm_ptr dgemm) nogil 47 | cdef void set_sscal(CBlas cblas, sscal_ptr sscal) nogil 48 | cdef void set_dscal(CBlas cblas, dscal_ptr dscal) nogil 49 | -------------------------------------------------------------------------------- /thinc/backends/cblas.pyx: -------------------------------------------------------------------------------- 1 | # cython: profile=False 2 | cimport blis.cy 3 | from cython.operator cimport dereference as deref 4 | from libcpp.memory cimport make_shared 5 | 6 | 7 | # Single- and double-precision wrappers for `blis.cy.scalv` 8 | cdef void blis_sscal(int N, float alpha, float* X, int incX) nogil: 9 | blis.cy.scalv(blis.cy.NO_CONJUGATE, N, alpha, X, incX) 10 | 11 | cdef void blis_dscal(int N, double alpha, double* X, int incX) nogil: 12 | blis.cy.scalv(blis.cy.NO_CONJUGATE, N, alpha, X, incX) 13 | 14 | 15 | cdef struct BlasFuncs: 16 | daxpy_ptr daxpy 17 | saxpy_ptr saxpy 18 | sgemm_ptr sgemm 19 | dgemm_ptr dgemm 20 | sscal_ptr sscal 21 | dscal_ptr dscal 22 | 23 | 24 | cdef class CBlas: 25 | __slots__ = [] 26 | 27 | def __init__(self): 28 | """Construct a CBlas instance set to use BLIS implementations of the 29 | supported BLAS functions.""" 30 | cdef BlasFuncs funcs 31 | funcs.daxpy = blis.cy.daxpy 32 | funcs.saxpy = blis.cy.saxpy 33 | funcs.sgemm = blis.cy.sgemm 34 | funcs.dgemm = blis.cy.dgemm 35 | funcs.sscal = blis_sscal 36 | funcs.dscal = blis_dscal 37 | self.ptr = make_shared[BlasFuncs](funcs) 38 | 39 | cdef daxpy_ptr daxpy(CBlas cblas) nogil: 40 | return deref(cblas.ptr).daxpy 41 | 42 | cdef saxpy_ptr saxpy(CBlas cblas) nogil: 43 | return deref(cblas.ptr).saxpy 44 | 45 | cdef sgemm_ptr sgemm(CBlas cblas) nogil: 46 | return deref(cblas.ptr).sgemm 47 | 48 | cdef dgemm_ptr dgemm(CBlas cblas) nogil: 49 | return deref(cblas.ptr).dgemm 50 | 51 | cdef sscal_ptr sscal(CBlas cblas) nogil: 52 | return deref(cblas.ptr).sscal 53 | 54 | cdef dscal_ptr dscal(CBlas cblas) nogil: 55 | return deref(cblas.ptr).dscal 56 | 57 | cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) nogil: 58 | deref(cblas.ptr).daxpy = daxpy 59 | 60 | cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) nogil: 61 | deref(cblas.ptr).saxpy = saxpy 62 | 63 | cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) nogil: 64 | deref(cblas.ptr).sgemm = sgemm 65 | 66 | cdef void set_dgemm(CBlas cblas, dgemm_ptr dgemm) nogil: 67 | deref(cblas.ptr).dgemm = dgemm 68 | 69 | cdef void set_sscal(CBlas cblas, sscal_ptr sscal) nogil: 70 | deref(cblas.ptr).sscal = sscal 71 | 72 | cdef void set_dscal(CBlas cblas, dscal_ptr dscal) nogil: 73 | deref(cblas.ptr).dscal = dscal 74 | -------------------------------------------------------------------------------- /thinc/backends/mps_ops.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | import numpy 4 | 5 | from .. import registry 6 | from ..compat import has_apple_ops 7 | from .numpy_ops import NumpyOps 8 | from .ops import Ops 9 | 10 | if TYPE_CHECKING: 11 | # Type checking does not work with dynamic base classes, since MyPy cannot 12 | # determine against which base class to check. So, always derive from Ops 13 | # during type checking. 14 | _Ops = Ops 15 | else: 16 | if has_apple_ops: 17 | from .apple_ops import AppleOps 18 | 19 | _Ops = AppleOps 20 | else: 21 | _Ops = NumpyOps 22 | 23 | 24 | @registry.ops("MPSOps") 25 | class MPSOps(_Ops): 26 | """Ops class for Metal Performance shaders.""" 27 | 28 | name = "mps" 29 | xp = numpy 30 | -------------------------------------------------------------------------------- /thinc/backends/numpy_ops.pxd: -------------------------------------------------------------------------------- 1 | from .cblas cimport saxpy_ptr 2 | 3 | ctypedef double[:, ::1] double2d_t 4 | ctypedef double[:, :, ::1] double3d_t 5 | ctypedef float[:, ::1] float2d_t 6 | ctypedef float[:, :, ::1] float3d_t 7 | ctypedef int[:, ::1] int2d_t 8 | ctypedef unsigned int[:, ::1] uint2d_t 9 | 10 | cdef fused ints2d_ft: 11 | int2d_t 12 | uint2d_t 13 | 14 | cdef fused reals2d_ft: 15 | float2d_t 16 | double2d_t 17 | 18 | cdef fused reals3d_ft: 19 | float3d_t 20 | double3d_t 21 | 22 | 23 | cdef extern from "cpu_kernels.hh": 24 | cdef cppclass axpy[T]: 25 | ctypedef void (*ptr)(int N, T alpha, const T* X, int incX, T *Y, int incY); 26 | 27 | void cpu_maxout[A, L](A* best__bo, L* which__bo, const A* cands_bop, 28 | L B, L O, L P) 29 | void cpu_backprop_maxout[A, L](A* dX__bop, const A* dX__bo, const L* which__bo, 30 | L B, L O, L P) except + 31 | void cpu_reduce_max[A, L](A* maxes__bo, L* which_bo, const A* X__to, 32 | const L* lengths__b, L B, L T, L O) except + 33 | 34 | void cpu_backprop_reduce_max[A, L](A* dX__to, const A* d_maxes__bo, const L* which__bo, 35 | const L* lengths__b, L B, L T, L O) except + 36 | void cpu_reduce_mean[A, L](A* means__bo, const A* X__to, const L* lengths__b, 37 | L B, L T, L O) except + 38 | void cpu_backprop_reduce_mean[A, L](A* dX__to, const A* d_means__bo, const L* lengths__b, 39 | L B, L T, L O) 40 | void cpu_mish[A, L](A* Y, L N, A threshold) 41 | void cpu_backprop_mish[A, L](A* dX, const A* X, L N, A threshold) 42 | void cpu_reduce_sum[A, L](A* sums__bo, const A* X__to, const L* lengths__b, 43 | L B, L T, L O) except + 44 | void cpu_backprop_reduce_sum[A, L](A* dX__to, const A* d_sums__bo, const L* lengths__b, 45 | L B, L T, L O) 46 | void cpu_relu[A, L](A* X, L N) 47 | void backprop_seq2col[A, L](A* d_seqs, const A* d_cols, const L* lengths, L B, L I, L nW, L nL) 48 | void seq2col[A, L](A* output, const A* X, const L* lengths, L nW, L B, L I, L nL) 49 | void cpu_gather_add[F, I, L](axpy[F].ptr axpy, F* out_bo, const F* table_to, const I* indices_bk, 50 | L T, L O, L B, L K) except + 51 | -------------------------------------------------------------------------------- /thinc/config.py: -------------------------------------------------------------------------------- 1 | import catalogue 2 | import confection 3 | from confection import VARIABLE_RE, Config, ConfigValidationError, Promise 4 | 5 | from .types import Decorator 6 | 7 | 8 | class registry(confection.registry): 9 | # fmt: off 10 | optimizers: Decorator = catalogue.create("thinc", "optimizers", entry_points=True) 11 | schedules: Decorator = catalogue.create("thinc", "schedules", entry_points=True) 12 | layers: Decorator = catalogue.create("thinc", "layers", entry_points=True) 13 | losses: Decorator = catalogue.create("thinc", "losses", entry_points=True) 14 | initializers: Decorator = catalogue.create("thinc", "initializers", entry_points=True) 15 | datasets: Decorator = catalogue.create("thinc", "datasets", entry_points=True) 16 | ops: Decorator = catalogue.create("thinc", "ops", entry_points=True) 17 | # fmt: on 18 | 19 | @classmethod 20 | def create(cls, registry_name: str, entry_points: bool = False) -> None: 21 | """Create a new custom registry.""" 22 | if hasattr(cls, registry_name): 23 | raise ValueError(f"Registry '{registry_name}' already exists") 24 | reg: Decorator = catalogue.create( 25 | "thinc", registry_name, entry_points=entry_points 26 | ) 27 | setattr(cls, registry_name, reg) 28 | 29 | 30 | __all__ = ["Config", "registry", "ConfigValidationError", "Promise", "VARIABLE_RE"] 31 | -------------------------------------------------------------------------------- /thinc/layers/add.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Dict, Optional, Tuple, TypeVar 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import ArrayXd, XY_XY_OutT 6 | from ..util import get_width 7 | 8 | InT = TypeVar("InT", bound=Any) 9 | OutT = TypeVar("OutT", bound=ArrayXd) 10 | 11 | 12 | @registry.layers("add.v1") 13 | def add( 14 | layer1: Model[InT, OutT], layer2: Model[InT, OutT], *layers: Model 15 | ) -> Model[InT, XY_XY_OutT]: 16 | """Compose two or more models `f`, `g`, etc, such that their outputs are 17 | added, i.e. `add(f, g)(x)` computes `f(x) + g(x)`. 18 | """ 19 | layers = (layer1, layer2) + layers 20 | if layers[0].name == "add": 21 | layers[0].layers.extend(layers[1:]) 22 | return layers[0] 23 | 24 | # only add an nI dimension if each sub-layer has one 25 | dims: Dict[str, Optional[int]] = {"nO": None} 26 | if all(node.has_dim("nI") in [True, None] for node in layers): 27 | dims = {"nO": None, "nI": None} 28 | 29 | return Model("add", forward, init=init, dims=dims, layers=layers) 30 | 31 | 32 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 33 | if not model.layers: 34 | return X, lambda dY: dY 35 | Y, first_callback = model.layers[0](X, is_train=is_train) 36 | callbacks = [] 37 | for layer in model.layers[1:]: 38 | layer_Y, layer_callback = layer(X, is_train=is_train) 39 | Y += layer_Y 40 | callbacks.append(layer_callback) 41 | 42 | def backprop(dY: InT) -> OutT: 43 | dX = first_callback(dY) 44 | for callback in callbacks: 45 | dX += callback(dY) 46 | return dX 47 | 48 | return Y, backprop 49 | 50 | 51 | def init( 52 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 53 | ) -> None: 54 | if X is not None: 55 | if model.has_dim("nI") is not False: 56 | model.set_dim("nI", get_width(X)) 57 | for layer in model.layers: 58 | if layer.has_dim("nI") is not False: 59 | layer.set_dim("nI", get_width(X)) 60 | if Y is not None: 61 | if model.has_dim("nO") is not False: 62 | model.set_dim("nO", get_width(Y)) 63 | for layer in model.layers: 64 | if layer.has_dim("nO") is not False: 65 | layer.set_dim("nO", get_width(Y)) 66 | for layer in model.layers: 67 | layer.initialize(X=X, Y=Y) 68 | model.set_dim("nO", model.layers[0].get_dim("nO")) 69 | -------------------------------------------------------------------------------- /thinc/layers/array_getitem.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence, Tuple, TypeVar, Union 2 | 3 | from ..model import Model 4 | from ..types import ArrayXd, FloatsXd, IntsXd 5 | 6 | AxisIndex = Union[int, slice, Sequence[int]] 7 | Index = Union[AxisIndex, Tuple[AxisIndex, ...]] 8 | ArrayTXd = TypeVar("ArrayTXd", bound=ArrayXd) 9 | 10 | 11 | def array_getitem(index: Index) -> Model[ArrayTXd, ArrayTXd]: 12 | """Index into input arrays, and return the subarrays. 13 | 14 | index: 15 | A valid numpy-style index. Multi-dimensional indexing can be performed 16 | by passing in a tuple, and slicing can be performed using the slice object. 17 | For instance, X[:, :-1] would be (slice(None, None), slice(None, -1)). 18 | """ 19 | return Model("array-getitem", forward, attrs={"index": index}) 20 | 21 | 22 | def floats_getitem(index: Index) -> Model[FloatsXd, FloatsXd]: 23 | """Index into input arrays, and return the subarrays. 24 | 25 | This delegates to `array_getitem`, but allows type declarations. 26 | """ 27 | return Model("floats-getitem", forward, attrs={"index": index}) 28 | 29 | 30 | def ints_getitem(index: Index) -> Model[IntsXd, IntsXd]: 31 | """Index into input arrays, and return the subarrays. 32 | 33 | This delegates to `array_getitem`, but allows type declarations. 34 | """ 35 | return Model("ints-getitem", forward, attrs={"index": index}) 36 | 37 | 38 | def forward(model, X, is_train): 39 | index = model.attrs["index"] 40 | shape = X.shape 41 | dtype = X.dtype 42 | 43 | def backprop_get_column(dY): 44 | dX = model.ops.alloc(shape, dtype=dtype) 45 | dX[index] = dY 46 | return dX 47 | 48 | if len(X) == 0: 49 | return X, backprop_get_column 50 | Y = X[index] 51 | return Y, backprop_get_column 52 | -------------------------------------------------------------------------------- /thinc/layers/bidirectional.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..backends import Ops 4 | from ..config import registry 5 | from ..model import Model 6 | from ..types import Padded 7 | 8 | InT = Padded 9 | OutT = Padded 10 | 11 | 12 | @registry.layers("bidirectional.v1") 13 | def bidirectional( 14 | l2r: Model[InT, OutT], r2l: Optional[Model[InT, OutT]] = None 15 | ) -> Model[InT, OutT]: 16 | """Stitch two RNN models into a bidirectional layer. Expects squared sequences.""" 17 | if r2l is None: 18 | r2l = l2r.copy() 19 | return Model(f"bi{l2r.name}", forward, layers=[l2r, r2l], init=init) 20 | 21 | 22 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 23 | l2r, r2l = model.layers 24 | X_rev = _reverse(model.ops, X) 25 | l2r_Z, bp_l2r_Z = l2r(X, is_train) 26 | r2l_Z, bp_r2l_Z = r2l(X_rev, is_train) 27 | Z = _concatenate(model.ops, l2r_Z, r2l_Z) 28 | 29 | def backprop(dZ: OutT) -> InT: 30 | d_l2r_Z, d_r2l_Z = _split(model.ops, dZ) 31 | dX_l2r = bp_l2r_Z(d_l2r_Z) 32 | dX_r2l = bp_r2l_Z(d_r2l_Z) 33 | return _sum(dX_l2r, dX_r2l) 34 | 35 | return Z, backprop 36 | 37 | 38 | def init( 39 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 40 | ) -> None: 41 | (Y1, Y2) = _split(model.ops, Y) if Y is not None else (None, None) 42 | model.layers[0].initialize(X=X, Y=Y1) 43 | model.layers[1].initialize(X=X, Y=Y2) 44 | 45 | 46 | def _reverse(ops: Ops, Xp: Padded) -> Padded: 47 | return Padded(Xp.data[::1], Xp.size_at_t, Xp.lengths, Xp.indices) 48 | 49 | 50 | def _concatenate(ops: Ops, l2r: Padded, r2l: Padded) -> Padded: 51 | return Padded( 52 | ops.xp.concatenate((l2r.data, r2l.data), axis=-1), 53 | l2r.size_at_t, 54 | l2r.lengths, 55 | l2r.indices, 56 | ) 57 | 58 | 59 | def _split(ops: Ops, Xp: Padded) -> Tuple[Padded, Padded]: 60 | half = Xp.data.shape[-1] // 2 61 | # I don't know how to write these ellipsis in the overloads :( 62 | X_l2r = Xp.data[cast(Tuple[slice, slice], (..., slice(None, half)))] 63 | X_r2l = Xp.data[cast(Tuple[slice, slice], (..., slice(half)))] 64 | return ( 65 | Padded(X_l2r, Xp.size_at_t, Xp.lengths, Xp.indices), 66 | Padded(X_r2l, Xp.size_at_t, Xp.lengths, Xp.indices), 67 | ) 68 | 69 | 70 | def _sum(Xp: Padded, Yp: Padded) -> Padded: 71 | return Padded(Xp.data + Yp.data, Xp.size_at_t, Xp.lengths, Xp.indices) 72 | -------------------------------------------------------------------------------- /thinc/layers/cauchysimilarity.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats1d, Floats2d 6 | from ..util import get_width 7 | 8 | InT = Tuple[Floats2d, Floats2d] 9 | OutT = Floats1d 10 | 11 | 12 | @registry.layers("CauchySimilarity.v1") 13 | def CauchySimilarity(nI: Optional[int] = None) -> Model[InT, OutT]: 14 | """Compare input vectors according to the Cauchy similarity function proposed by 15 | Chen (2013). Primarily used within Siamese neural networks. 16 | """ 17 | return Model( 18 | "cauchy_similarity", 19 | forward, 20 | init=init, 21 | dims={"nI": nI, "nO": 1}, 22 | params={"W": None}, 23 | ) 24 | 25 | 26 | def forward( 27 | model: Model[InT, OutT], X1_X2: InT, is_train: bool 28 | ) -> Tuple[OutT, Callable]: 29 | X1, X2 = X1_X2 30 | W = cast(Floats2d, model.get_param("W")) 31 | diff = X1 - X2 32 | square_diff = diff**2 33 | total = (W * square_diff).sum(axis=1) 34 | sim, bp_sim = inverse(total) 35 | 36 | def backprop(d_sim: OutT) -> InT: 37 | d_total = bp_sim(d_sim) 38 | d_total = model.ops.reshape2f(d_total, -1, 1) 39 | model.inc_grad("W", (d_total * square_diff).sum(axis=0)) 40 | d_square_diff = W * d_total 41 | d_diff = 2 * d_square_diff * diff 42 | return (d_diff, -d_diff) 43 | 44 | return sim, backprop 45 | 46 | 47 | def init( 48 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 49 | ) -> None: 50 | if X is not None: 51 | model.set_dim("nI", get_width(X[0])) 52 | # Initialize weights to 1 53 | W = model.ops.alloc1f(model.get_dim("nI")) 54 | W += 1 55 | model.set_param("W", W) 56 | 57 | 58 | def inverse(total: OutT) -> Tuple[OutT, Callable]: 59 | inv = 1.0 / (1 + total) 60 | 61 | def backward(d_inverse: OutT) -> OutT: 62 | return d_inverse * (-1 / (total + 1) ** 2) 63 | 64 | return inv, backward 65 | -------------------------------------------------------------------------------- /thinc/layers/clone.py: -------------------------------------------------------------------------------- 1 | from typing import List, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from .chain import chain 6 | from .noop import noop 7 | 8 | InT = TypeVar("InT") 9 | OutT = TypeVar("OutT") 10 | 11 | 12 | @registry.layers("clone.v1") 13 | def clone(orig: Model[InT, OutT], n: int) -> Model[InT, OutT]: 14 | """Construct `n` copies of a layer, with distinct weights. i.e. 15 | `clone(f, 3)(x)` computes f(f'(f''(x))). 16 | """ 17 | if n == 0: 18 | return cast(Model[InT, OutT], noop()) 19 | elif n == 1: 20 | return orig 21 | layers: List[Model] = [orig] 22 | for i in range(n - 1): 23 | layers.append(orig.copy()) 24 | return cast(Model[InT, OutT], chain(*layers)) 25 | -------------------------------------------------------------------------------- /thinc/layers/dish.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import he_normal_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | 13 | @registry.layers("Dish.v1") 14 | def Dish( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | dropout: Optional[float] = None, 21 | normalize: bool = False, 22 | ) -> Model[Floats2d, Floats2d]: 23 | if init_W is None: 24 | init_W = he_normal_init 25 | if init_b is None: 26 | init_b = zero_init 27 | model: Model[Floats2d, Floats2d] = Model( 28 | "dish", 29 | forward, 30 | init=partial(init, init_W, init_b), 31 | dims={"nO": nO, "nI": nI}, 32 | params={"W": None, "b": None}, 33 | ) 34 | if normalize: 35 | model = chain(model, LayerNorm(nI=nO)) 36 | if dropout is not None: 37 | model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) 38 | return model 39 | 40 | 41 | def forward( 42 | model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool 43 | ) -> Tuple[Floats2d, Callable]: 44 | W = cast(Floats2d, model.get_param("W")) 45 | b = cast(Floats1d, model.get_param("b")) 46 | Y_preact = model.ops.affine(X, W, b) 47 | Y = model.ops.dish(Y_preact) 48 | 49 | def backprop(dY: Floats2d) -> Floats2d: 50 | dY = model.ops.backprop_dish(dY, X, inplace=False) 51 | model.inc_grad("b", dY.sum(axis=0)) 52 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 53 | return model.ops.gemm(dY, W) 54 | 55 | return Y, backprop 56 | 57 | 58 | def init( 59 | init_W: Callable, 60 | init_b: Callable, 61 | model: Model[Floats2d, Floats2d], 62 | X: Optional[Floats2d] = None, 63 | Y: Optional[Floats2d] = None, 64 | ) -> None: 65 | if X is not None: 66 | model.set_dim("nI", get_width(X)) 67 | if Y is not None: 68 | model.set_dim("nO", get_width(Y)) 69 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 70 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 71 | -------------------------------------------------------------------------------- /thinc/layers/expand_window.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, TypeVar, Union, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d, Ragged 6 | 7 | InT = TypeVar("InT", Floats2d, Ragged) 8 | 9 | 10 | @registry.layers("expand_window.v1") 11 | def expand_window(window_size: int = 1) -> Model[InT, InT]: 12 | """For each vector in an input, construct an output vector that contains the 13 | input and a window of surrounding vectors. This is one step in a convolution. 14 | """ 15 | return Model("expand_window", forward, attrs={"window_size": window_size}) 16 | 17 | 18 | def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: 19 | if isinstance(X, Ragged): 20 | return _expand_window_ragged(model, X) 21 | else: 22 | return _expand_window_floats(model, X) 23 | 24 | 25 | def _expand_window_floats( 26 | model: Model[InT, InT], X: Floats2d 27 | ) -> Tuple[Floats2d, Callable]: 28 | nW = model.attrs["window_size"] 29 | if len(X) > 0: 30 | Y = model.ops.seq2col(X, nW) 31 | else: 32 | assert len(X) == 0 33 | Y = model.ops.tile(X, (nW * 2) + 1) 34 | 35 | def backprop(dY: Floats2d) -> Floats2d: 36 | return model.ops.backprop_seq2col(dY, nW) 37 | 38 | return Y, backprop 39 | 40 | 41 | def _expand_window_ragged( 42 | model: Model[InT, InT], Xr: Ragged 43 | ) -> Tuple[Ragged, Callable]: 44 | nW = model.attrs["window_size"] 45 | Y = Ragged( 46 | model.ops.seq2col(cast(Floats2d, Xr.data), nW, lengths=Xr.lengths), Xr.lengths 47 | ) 48 | 49 | def backprop(dYr: Ragged) -> Ragged: 50 | return Ragged( 51 | model.ops.backprop_seq2col( 52 | cast(Floats2d, dYr.data), nW, lengths=Xr.lengths 53 | ), 54 | Xr.lengths, 55 | ) 56 | 57 | return Y, backprop 58 | -------------------------------------------------------------------------------- /thinc/layers/gelu.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import he_normal_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | 13 | @registry.layers("Gelu.v1") 14 | def Gelu( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | dropout: Optional[float] = None, 21 | normalize: bool = False, 22 | ) -> Model[Floats2d, Floats2d]: 23 | if init_W is None: 24 | init_W = he_normal_init 25 | if init_b is None: 26 | init_b = zero_init 27 | model: Model[Floats2d, Floats2d] = Model( 28 | "gelu", 29 | forward, 30 | init=partial(init, init_W, init_b), 31 | dims={"nO": nO, "nI": nI}, 32 | params={"W": None, "b": None}, 33 | ) 34 | if normalize: 35 | model = chain(model, LayerNorm(nI=nO)) 36 | if dropout is not None: 37 | model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) 38 | return model 39 | 40 | 41 | def forward( 42 | model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool 43 | ) -> Tuple[Floats2d, Callable]: 44 | W = cast(Floats2d, model.get_param("W")) 45 | b = cast(Floats1d, model.get_param("b")) 46 | Y_preact = model.ops.affine(X, W, b) 47 | Y = model.ops.gelu(Y_preact) 48 | 49 | def backprop(dY: Floats2d) -> Floats2d: 50 | dY = model.ops.backprop_gelu(dY, Y_preact, inplace=False) 51 | model.inc_grad("b", dY.sum(axis=0)) 52 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 53 | return model.ops.gemm(dY, W) 54 | 55 | return Y, backprop 56 | 57 | 58 | def init( 59 | init_W: Callable, 60 | init_b: Callable, 61 | model: Model[Floats2d, Floats2d], 62 | X: Optional[Floats2d] = None, 63 | Y: Optional[Floats2d] = None, 64 | ) -> None: 65 | if X is not None: 66 | model.set_dim("nI", get_width(X)) 67 | if Y is not None: 68 | model.set_dim("nO", get_width(Y)) 69 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 70 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 71 | -------------------------------------------------------------------------------- /thinc/layers/hard_swish.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import he_normal_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | 13 | @registry.layers("HardSwish.v1") 14 | def HardSwish( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | dropout: Optional[float] = None, 21 | normalize: bool = False, 22 | ) -> Model[Floats2d, Floats2d]: 23 | if init_W is None: 24 | init_W = he_normal_init 25 | if init_b is None: 26 | init_b = zero_init 27 | model: Model[Floats2d, Floats2d] = Model( 28 | "hardswish", 29 | forward, 30 | init=partial(init, init_W, init_b), 31 | dims={"nO": nO, "nI": nI}, 32 | params={"W": None, "b": None}, 33 | ) 34 | if normalize: 35 | model = chain(model, LayerNorm(nI=nO)) 36 | if dropout is not None: 37 | model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) 38 | return model 39 | 40 | 41 | def forward( 42 | model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool 43 | ) -> Tuple[Floats2d, Callable]: 44 | W = cast(Floats2d, model.get_param("W")) 45 | b = cast(Floats1d, model.get_param("b")) 46 | Y_preact = model.ops.affine(X, W, b) 47 | Y = model.ops.hard_swish(Y_preact) 48 | 49 | def backprop(dY: Floats2d) -> Floats2d: 50 | dY = model.ops.backprop_hard_swish(dY, Y_preact, inplace=False) 51 | model.inc_grad("b", dY.sum(axis=0)) 52 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 53 | return model.ops.gemm(dY, W) 54 | 55 | return Y, backprop 56 | 57 | 58 | def init( 59 | init_W: Callable, 60 | init_b: Callable, 61 | model: Model[Floats2d, Floats2d], 62 | X: Optional[Floats2d] = None, 63 | Y: Optional[Floats2d] = None, 64 | ) -> None: 65 | if X is not None: 66 | model.set_dim("nI", get_width(X)) 67 | if Y is not None: 68 | model.set_dim("nO", get_width(Y)) 69 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 70 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 71 | -------------------------------------------------------------------------------- /thinc/layers/hard_swish_mobilenet.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import he_normal_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | 13 | @registry.layers("HardSwishMobilenet.v1") 14 | def HardSwishMobilenet( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | dropout: Optional[float] = None, 21 | normalize: bool = False, 22 | ) -> Model[Floats2d, Floats2d]: 23 | if init_W is None: 24 | init_W = he_normal_init 25 | if init_b is None: 26 | init_b = zero_init 27 | model: Model[Floats2d, Floats2d] = Model( 28 | "hardswishmobilenet", 29 | forward, 30 | init=partial(init, init_W, init_b), 31 | dims={"nO": nO, "nI": nI}, 32 | params={"W": None, "b": None}, 33 | ) 34 | if normalize: 35 | model = chain(model, LayerNorm(nI=nO)) 36 | if dropout is not None: 37 | model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) 38 | return model 39 | 40 | 41 | def forward( 42 | model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool 43 | ) -> Tuple[Floats2d, Callable]: 44 | W = cast(Floats2d, model.get_param("W")) 45 | b = cast(Floats1d, model.get_param("b")) 46 | Y_preact = model.ops.affine(X, W, b) 47 | Y = model.ops.hard_swish_mobilenet(Y_preact) 48 | 49 | def backprop(dY: Floats2d) -> Floats2d: 50 | dY = model.ops.backprop_hard_swish_mobilenet(dY, Y_preact, inplace=False) 51 | model.inc_grad("b", dY.sum(axis=0)) 52 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 53 | return model.ops.gemm(dY, W) 54 | 55 | return Y, backprop 56 | 57 | 58 | def init( 59 | init_W: Callable, 60 | init_b: Callable, 61 | model: Model[Floats2d, Floats2d], 62 | X: Optional[Floats2d] = None, 63 | Y: Optional[Floats2d] = None, 64 | ) -> None: 65 | if X is not None: 66 | model.set_dim("nI", get_width(X)) 67 | if Y is not None: 68 | model.set_dim("nO", get_width(Y)) 69 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 70 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 71 | -------------------------------------------------------------------------------- /thinc/layers/linear.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import glorot_uniform_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | 9 | InT = Floats2d 10 | OutT = Floats2d 11 | 12 | 13 | @registry.layers("Linear.v1") 14 | def Linear( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | ) -> Model[InT, OutT]: 21 | """Multiply inputs by a weights matrix and adds a bias vector.""" 22 | if init_W is None: 23 | init_W = glorot_uniform_init 24 | if init_b is None: 25 | init_b = zero_init 26 | return Model( 27 | "linear", 28 | forward, 29 | init=partial(init, init_W, init_b), 30 | dims={"nO": nO, "nI": nI}, 31 | params={"W": None, "b": None}, 32 | ) 33 | 34 | 35 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 36 | W = cast(Floats2d, model.get_param("W")) 37 | b = cast(Floats1d, model.get_param("b")) 38 | Y = model.ops.gemm(X, W, trans2=True) 39 | Y += b 40 | 41 | def backprop(dY: OutT) -> InT: 42 | model.inc_grad("b", dY.sum(axis=0)) 43 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 44 | return model.ops.gemm(dY, W) 45 | 46 | return Y, backprop 47 | 48 | 49 | def init( 50 | init_W: Callable, 51 | init_b: Callable, 52 | model: Model[InT, OutT], 53 | X: Optional[InT] = None, 54 | Y: Optional[OutT] = None, 55 | ) -> None: 56 | if X is not None: 57 | model.set_dim("nI", get_width(X)) 58 | if Y is not None: 59 | model.set_dim("nO", get_width(Y)) 60 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 61 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 62 | -------------------------------------------------------------------------------- /thinc/layers/list2array.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Tuple, TypeVar 2 | 3 | from ..backends import NumpyOps 4 | from ..config import registry 5 | from ..model import Model 6 | from ..types import Array2d 7 | 8 | NUMPY_OPS = NumpyOps() 9 | 10 | 11 | OutT = TypeVar("OutT", bound=Array2d) 12 | InT = List[OutT] 13 | 14 | 15 | @registry.layers("list2array.v1") 16 | def list2array() -> Model[InT, OutT]: 17 | """Transform sequences to ragged arrays if necessary and return the data 18 | from the ragged array. If sequences are already ragged, do nothing. A 19 | ragged array is a tuple (data, lengths), where data is the concatenated data. 20 | """ 21 | return Model("list2array", forward) 22 | 23 | 24 | def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: 25 | lengths = NUMPY_OPS.asarray1i([len(x) for x in Xs]) 26 | 27 | def backprop(dY: OutT) -> InT: 28 | return model.ops.unflatten(dY, lengths) 29 | 30 | return model.ops.flatten(Xs), backprop 31 | -------------------------------------------------------------------------------- /thinc/layers/list2padded.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import List2d, Padded 6 | 7 | InT = TypeVar("InT", bound=List2d) 8 | OutT = Padded 9 | 10 | 11 | @registry.layers("list2padded.v1") 12 | def list2padded() -> Model[InT, OutT]: 13 | """Create a layer to convert a list of array inputs into Padded.""" 14 | return Model(f"list2padded", forward) 15 | 16 | 17 | def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: 18 | Yp = model.ops.list2padded(Xs) 19 | 20 | def backprop(dYp: OutT) -> InT: 21 | return cast(InT, model.ops.padded2list(dYp)) 22 | 23 | return Yp, backprop 24 | -------------------------------------------------------------------------------- /thinc/layers/list2ragged.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import ArrayXd, ListXd, Ragged 6 | 7 | InT = TypeVar("InT", bound=ListXd) 8 | OutT = Ragged 9 | 10 | 11 | @registry.layers("list2ragged.v1") 12 | def list2ragged() -> Model[InT, OutT]: 13 | """Transform sequences to ragged arrays if necessary and return the ragged 14 | array. If sequences are already ragged, do nothing. A ragged array is a 15 | tuple (data, lengths), where data is the concatenated data. 16 | """ 17 | return Model("list2ragged", forward) 18 | 19 | 20 | def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: 21 | def backprop(dYr: OutT) -> InT: 22 | return cast(InT, model.ops.unflatten(dYr.data, dYr.lengths)) 23 | 24 | lengths = model.ops.asarray1i([len(x) for x in Xs]) 25 | return Ragged(model.ops.flatten(Xs), lengths), backprop 26 | -------------------------------------------------------------------------------- /thinc/layers/logistic.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d 6 | 7 | InT = Floats2d 8 | OutT = Floats2d 9 | 10 | 11 | @registry.layers("Logistic.v1") 12 | def Logistic() -> Model[InT, OutT]: 13 | """Deprecated in favor of `sigmoid_activation` layer, for more consistent 14 | naming. 15 | """ 16 | return Model("logistic", forward) 17 | 18 | 19 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 20 | Y = model.ops.sigmoid(X, inplace=False) 21 | 22 | def backprop(dY: OutT) -> InT: 23 | return dY * model.ops.dsigmoid(Y, inplace=False) 24 | 25 | return Y, backprop 26 | -------------------------------------------------------------------------------- /thinc/layers/map_list.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Optional, Tuple, TypeVar 2 | 3 | from ..model import Model 4 | 5 | InT = TypeVar("InT") 6 | OutT = TypeVar("OutT") 7 | 8 | 9 | def map_list(layer: Model[InT, OutT]) -> Model[List[InT], List[OutT]]: 10 | """Create a model that maps a child layer across list inputs.""" 11 | return Model("map_list", forward, layers=[layer], init=init) 12 | 13 | 14 | def forward( 15 | model: Model[List[InT], List[OutT]], Xs: List[InT], is_train: bool 16 | ) -> Tuple[List[OutT], Callable[[List[OutT]], List[InT]]]: 17 | layer = model.layers[0] 18 | Ys = [] 19 | callbacks = [] 20 | for X in Xs: 21 | Y, get_dX = layer(X, is_train) 22 | Ys.append(Y) 23 | callbacks.append(get_dX) 24 | 25 | def backprop_map_list(dYs: List[OutT]) -> List[InT]: 26 | return [callback(dY) for callback, dY in zip(callbacks, dYs)] 27 | 28 | return Ys, backprop_map_list 29 | 30 | 31 | def init( 32 | model: Model[List[InT], List[OutT]], 33 | X: Optional[List[InT]] = None, 34 | Y: Optional[List[OutT]] = None, 35 | ) -> None: 36 | model.layers[0].initialize(X=X[0] if X else None, Y=Y[0] if Y else None) 37 | -------------------------------------------------------------------------------- /thinc/layers/mish.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import glorot_uniform_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | InT = Floats2d 13 | OutT = Floats2d 14 | 15 | 16 | @registry.layers("Mish.v1") 17 | def Mish( 18 | nO: Optional[int] = None, 19 | nI: Optional[int] = None, 20 | *, 21 | init_W: Optional[Callable] = None, 22 | init_b: Optional[Callable] = None, 23 | dropout: Optional[float] = None, 24 | normalize: bool = False, 25 | ) -> Model[InT, OutT]: 26 | """Dense layer with mish activation. 27 | https://arxiv.org/pdf/1908.08681.pdf 28 | """ 29 | if init_W is None: 30 | init_W = glorot_uniform_init 31 | if init_b is None: 32 | init_b = zero_init 33 | model: Model[InT, OutT] = Model( 34 | "mish", 35 | forward, 36 | init=partial(init, init_W, init_b), 37 | dims={"nO": nO, "nI": nI}, 38 | params={"W": None, "b": None}, 39 | ) 40 | if normalize: 41 | model = chain(model, cast(Model[InT, OutT], LayerNorm(nI=nO))) 42 | if dropout is not None: 43 | model = chain(model, cast(Model[InT, OutT], Dropout(dropout))) 44 | return model 45 | 46 | 47 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 48 | W = cast(Floats2d, model.get_param("W")) 49 | b = cast(Floats1d, model.get_param("b")) 50 | Y_pre_mish = model.ops.gemm(X, W, trans2=True) 51 | Y_pre_mish += b 52 | Y = model.ops.mish(Y_pre_mish) 53 | 54 | def backprop(dY: OutT) -> InT: 55 | dY_pre_mish = model.ops.backprop_mish(dY, Y_pre_mish) 56 | model.inc_grad("W", model.ops.gemm(dY_pre_mish, X, trans1=True)) 57 | model.inc_grad("b", dY_pre_mish.sum(axis=0)) 58 | dX = model.ops.gemm(dY_pre_mish, W) 59 | return dX 60 | 61 | return Y, backprop 62 | 63 | 64 | def init( 65 | init_W: Callable, 66 | init_b: Callable, 67 | model: Model[InT, OutT], 68 | X: Optional[InT] = None, 69 | Y: Optional[OutT] = None, 70 | ) -> None: 71 | if X is not None: 72 | model.set_dim("nI", get_width(X)) 73 | if Y is not None: 74 | model.set_dim("nO", get_width(Y)) 75 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 76 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 77 | -------------------------------------------------------------------------------- /thinc/layers/multisoftmax.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats1d, Floats2d 6 | from ..util import get_width 7 | 8 | InT = Floats2d 9 | OutT = Floats2d 10 | 11 | 12 | @registry.layers("MultiSoftmax.v1") 13 | def MultiSoftmax(nOs: Tuple[int, ...], nI: Optional[int] = None) -> Model[InT, OutT]: 14 | """Neural network layer that predicts several multi-class attributes at once. 15 | For instance, we might predict one class with 6 variables, and another with 5. 16 | We predict the 11 neurons required for this, and then softmax them such 17 | that columns 0-6 make a probability distribution and columns 6-11 make another. 18 | """ 19 | return Model( 20 | "multisoftmax", 21 | forward, 22 | init=init, 23 | dims={"nO": sum(nOs), "nI": nI}, 24 | attrs={"nOs": nOs}, 25 | params={"W": None, "b": None}, 26 | ) 27 | 28 | 29 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 30 | nOs = model.attrs["nOs"] 31 | W = cast(Floats2d, model.get_param("W")) 32 | b = cast(Floats1d, model.get_param("b")) 33 | 34 | def backprop(dY: OutT) -> InT: 35 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 36 | model.inc_grad("b", dY.sum(axis=0)) 37 | return model.ops.gemm(dY, W) 38 | 39 | Y = model.ops.gemm(X, W, trans2=True) 40 | Y += b 41 | i = 0 42 | for out_size in nOs: 43 | model.ops.softmax(Y[:, i : i + out_size], inplace=True) 44 | i += out_size 45 | return Y, backprop 46 | 47 | 48 | def init( 49 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 50 | ) -> None: 51 | if X is not None: 52 | model.set_dim("nI", get_width(X)) 53 | nO = model.get_dim("nO") 54 | nI = model.get_dim("nI") 55 | model.set_param("W", model.ops.alloc2f(nO, nI)) 56 | model.set_param("b", model.ops.alloc1f(nO)) 57 | -------------------------------------------------------------------------------- /thinc/layers/noop.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, TypeVar 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | 6 | InOutT = TypeVar("InOutT") 7 | 8 | 9 | @registry.layers("noop.v1") 10 | def noop(*layers: Model) -> Model[InOutT, InOutT]: 11 | """Transform a sequences of layers into a null operation.""" 12 | return Model("noop", forward, layers=layers) 13 | 14 | 15 | def forward( 16 | model: Model[InOutT, InOutT], X: InOutT, is_train: bool 17 | ) -> Tuple[InOutT, Callable]: 18 | def backprop(dY: InOutT) -> InOutT: 19 | return dY 20 | 21 | return X, backprop 22 | -------------------------------------------------------------------------------- /thinc/layers/padded2list.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import List2d, Padded 6 | 7 | InT = Padded 8 | OutT = TypeVar("OutT", bound=List2d) 9 | 10 | 11 | @registry.layers("padded2list.v1") 12 | def padded2list() -> Model[InT, OutT]: 13 | """Create a layer to convert a Padded input into a list of arrays.""" 14 | return Model(f"padded2list", forward) 15 | 16 | 17 | def forward( 18 | model: Model[InT, OutT], Xp: InT, is_train: bool 19 | ) -> Tuple[OutT, Callable[[OutT], InT]]: 20 | Ys = cast(OutT, model.ops.padded2list(Xp)) 21 | 22 | def backprop(dYs: OutT) -> InT: 23 | dYp = model.ops.list2padded(dYs) 24 | assert isinstance(dYp, Padded) 25 | return dYp 26 | 27 | return Ys, backprop 28 | -------------------------------------------------------------------------------- /thinc/layers/parametricattention.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Ragged 6 | from ..util import get_width 7 | 8 | InT = Ragged 9 | OutT = Ragged 10 | 11 | 12 | @registry.layers("ParametricAttention.v1") 13 | def ParametricAttention(nO: Optional[int] = None) -> Model[InT, OutT]: 14 | """Weight inputs by similarity to a learned vector""" 15 | return Model("para-attn", forward, init=init, params={"Q": None}, dims={"nO": nO}) 16 | 17 | 18 | def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: 19 | Q = model.get_param("Q") 20 | attention, bp_attention = _get_attention(model.ops, Q, Xr.dataXd, Xr.lengths) 21 | output, bp_output = _apply_attention(model.ops, attention, Xr.dataXd, Xr.lengths) 22 | 23 | def backprop(dYr: OutT) -> InT: 24 | dX, d_attention = bp_output(dYr.dataXd) 25 | dQ, dX2 = bp_attention(d_attention) 26 | model.inc_grad("Q", dQ.ravel()) 27 | dX += dX2 28 | return Ragged(dX, dYr.lengths) 29 | 30 | return Ragged(output, Xr.lengths), backprop 31 | 32 | 33 | def init( 34 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 35 | ) -> None: 36 | if X is not None: 37 | model.set_dim("nO", get_width(X)) 38 | # Randomly initialize the parameter, as though it were an embedding. 39 | Q = model.ops.alloc1f(model.get_dim("nO")) 40 | Q += model.ops.xp.random.uniform(-0.1, 0.1, Q.shape) 41 | model.set_param("Q", Q) 42 | 43 | 44 | def _get_attention(ops, Q, X, lengths): 45 | attention = ops.gemm(X, ops.reshape2f(Q, -1, 1)) 46 | attention = ops.softmax_sequences(attention, lengths) 47 | 48 | def get_attention_bwd(d_attention): 49 | d_attention = ops.backprop_softmax_sequences(d_attention, attention, lengths) 50 | dQ = ops.gemm(X, d_attention, trans1=True) 51 | dX = ops.xp.outer(d_attention, Q) 52 | return dQ, dX 53 | 54 | return attention, get_attention_bwd 55 | 56 | 57 | def _apply_attention(ops, attention, X, lengths): 58 | output = X * attention 59 | 60 | def apply_attention_bwd(d_output): 61 | d_attention = (X * d_output).sum(axis=1, keepdims=True) 62 | dX = d_output * attention 63 | return dX, d_attention 64 | 65 | return output, apply_attention_bwd 66 | -------------------------------------------------------------------------------- /thinc/layers/ragged2list.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import ListXd, Ragged 6 | 7 | InT = Ragged 8 | OutT = TypeVar("OutT", bound=ListXd) 9 | 10 | 11 | @registry.layers("ragged2list.v1") 12 | def ragged2list() -> Model[InT, OutT]: 13 | """Transform sequences from a ragged format into lists.""" 14 | return Model("ragged2list", forward) 15 | 16 | 17 | def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: 18 | lengths = Xr.lengths 19 | 20 | def backprop(dXs: OutT) -> InT: 21 | return Ragged(model.ops.flatten(dXs, pad=0), lengths) # type:ignore[arg-type] 22 | # type ignore necessary for older versions of Mypy/Pydantic 23 | 24 | data = cast(OutT, model.ops.unflatten(Xr.dataXd, Xr.lengths)) 25 | return data, backprop 26 | -------------------------------------------------------------------------------- /thinc/layers/reduce_first.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d, Ragged 6 | from ..util import ArrayInfo 7 | 8 | InT = Ragged 9 | OutT = Floats2d 10 | 11 | 12 | @registry.layers("reduce_first.v1") 13 | def reduce_first() -> Model[InT, OutT]: 14 | """Reduce ragged-formatted sequences to their first element.""" 15 | return Model("reduce_first", forward) 16 | 17 | 18 | def forward( 19 | model: Model[InT, OutT], Xr: InT, is_train: bool 20 | ) -> Tuple[OutT, Callable[[OutT], InT]]: 21 | Y, starts_ends = model.ops.reduce_first(cast(Floats2d, Xr.data), Xr.lengths) 22 | 23 | array_info = ArrayInfo.from_array(Y) 24 | 25 | def backprop(dY: OutT) -> InT: 26 | array_info.check_consistency(dY) 27 | dX = model.ops.backprop_reduce_first(dY, starts_ends) 28 | return Ragged(dX, Xr.lengths) 29 | 30 | return Y, backprop 31 | -------------------------------------------------------------------------------- /thinc/layers/reduce_last.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d, Ragged 6 | from ..util import ArrayInfo 7 | 8 | InT = Ragged 9 | OutT = Floats2d 10 | 11 | 12 | @registry.layers("reduce_last.v1") 13 | def reduce_last() -> Model[InT, OutT]: 14 | """Reduce ragged-formatted sequences to their last element.""" 15 | return Model("reduce_last", forward) 16 | 17 | 18 | def forward( 19 | model: Model[InT, OutT], Xr: InT, is_train: bool 20 | ) -> Tuple[OutT, Callable[[OutT], InT]]: 21 | Y, lasts = model.ops.reduce_last(cast(Floats2d, Xr.data), Xr.lengths) 22 | array_info = ArrayInfo.from_array(Y) 23 | 24 | def backprop(dY: OutT) -> InT: 25 | array_info.check_consistency(dY) 26 | dX = model.ops.backprop_reduce_last(dY, lasts) 27 | return Ragged(dX, Xr.lengths) 28 | 29 | return Y, backprop 30 | -------------------------------------------------------------------------------- /thinc/layers/reduce_max.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d, Ragged 6 | from ..util import ArrayInfo 7 | 8 | InT = Ragged 9 | OutT = Floats2d 10 | 11 | 12 | @registry.layers("reduce_max.v1") 13 | def reduce_max() -> Model[InT, OutT]: 14 | return Model("reduce_max", forward) 15 | 16 | 17 | def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: 18 | Y, which = model.ops.reduce_max(cast(Floats2d, Xr.data), Xr.lengths) 19 | lengths = Xr.lengths 20 | array_info = ArrayInfo.from_array(Y) 21 | 22 | def backprop(dY: OutT) -> InT: 23 | array_info.check_consistency(dY) 24 | return Ragged(model.ops.backprop_reduce_max(dY, which, lengths), lengths) 25 | 26 | return Y, backprop 27 | -------------------------------------------------------------------------------- /thinc/layers/reduce_mean.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d, Ragged 6 | from ..util import ArrayInfo 7 | 8 | InT = Ragged 9 | OutT = Floats2d 10 | 11 | 12 | @registry.layers("reduce_mean.v1") 13 | def reduce_mean() -> Model[InT, OutT]: 14 | return Model("reduce_mean", forward) 15 | 16 | 17 | def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: 18 | Y = model.ops.reduce_mean(cast(Floats2d, Xr.data), Xr.lengths) 19 | lengths = Xr.lengths 20 | 21 | array_info = ArrayInfo.from_array(Y) 22 | 23 | def backprop(dY: OutT) -> InT: 24 | array_info.check_consistency(dY) 25 | return Ragged(model.ops.backprop_reduce_mean(dY, lengths), lengths) 26 | 27 | return Y, backprop 28 | -------------------------------------------------------------------------------- /thinc/layers/reduce_sum.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d, Ragged 6 | from ..util import ArrayInfo 7 | 8 | InT = Ragged 9 | OutT = Floats2d 10 | 11 | 12 | @registry.layers("reduce_sum.v1") 13 | def reduce_sum() -> Model[InT, OutT]: 14 | return Model("reduce_sum", forward) 15 | 16 | 17 | def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: 18 | Y = model.ops.reduce_sum(cast(Floats2d, Xr.data), Xr.lengths) 19 | lengths = Xr.lengths 20 | array_info = ArrayInfo.from_array(Y) 21 | 22 | def backprop(dY: OutT) -> InT: 23 | array_info.check_consistency(dY) 24 | return Ragged(model.ops.backprop_reduce_sum(dY, lengths), lengths) 25 | 26 | return Y, backprop 27 | -------------------------------------------------------------------------------- /thinc/layers/relu.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import glorot_uniform_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | InT = Floats2d 13 | OutT = Floats2d 14 | 15 | 16 | @registry.layers("Relu.v1") 17 | def Relu( 18 | nO: Optional[int] = None, 19 | nI: Optional[int] = None, 20 | *, 21 | init_W: Optional[Callable] = None, 22 | init_b: Optional[Callable] = None, 23 | dropout: Optional[float] = None, 24 | normalize: bool = False, 25 | ) -> Model[InT, OutT]: 26 | if init_W is None: 27 | init_W = glorot_uniform_init 28 | if init_b is None: 29 | init_b = zero_init 30 | model: Model[InT, OutT] = Model( 31 | "relu", 32 | forward, 33 | init=partial(init, init_W, init_b), 34 | dims={"nO": nO, "nI": nI}, 35 | params={"W": None, "b": None}, 36 | ) 37 | if normalize: 38 | model = chain(model, LayerNorm(nI=nO)) 39 | if dropout is not None: 40 | model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) 41 | return model 42 | 43 | 44 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 45 | W = cast(Floats2d, model.get_param("W")) 46 | b = cast(Floats1d, model.get_param("b")) 47 | Y = model.ops.affine(X, W, b) 48 | Y = model.ops.relu(Y) 49 | 50 | def backprop(dY: OutT) -> InT: 51 | dY = model.ops.backprop_relu(dY, Y) 52 | model.inc_grad("b", dY.sum(axis=0)) 53 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 54 | return model.ops.gemm(dY, W) 55 | 56 | return Y, backprop 57 | 58 | 59 | def init( 60 | init_W: Callable, 61 | init_b: Callable, 62 | model: Model[InT, OutT], 63 | X: Optional[InT] = None, 64 | Y: Optional[OutT] = None, 65 | ) -> None: 66 | if X is not None: 67 | model.set_dim("nI", get_width(X)) 68 | if Y is not None: 69 | model.set_dim("nO", get_width(Y)) 70 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 71 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 72 | -------------------------------------------------------------------------------- /thinc/layers/residual.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Optional, Tuple, TypeVar 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats1d, Floats2d, Floats3d, Floats4d, FloatsXd, Padded, Ragged 6 | 7 | # fmt: off 8 | InT = TypeVar( 9 | "InT", List[Floats1d], List[Floats2d], List[Floats3d], List[Floats4d], 10 | Ragged, Padded, FloatsXd, Floats1d, Floats2d, Floats3d, Floats4d) 11 | # fmt: on 12 | 13 | 14 | @registry.layers("residual.v1") 15 | def residual(layer: Model[InT, InT]) -> Model[InT, InT]: 16 | return Model( 17 | f"residual({layer.name})", 18 | forward, 19 | init=init, 20 | layers=[layer], 21 | dims={ 22 | "nO": layer.get_dim("nO") if layer.has_dim("nO") else None, 23 | "nI": layer.get_dim("nI") if layer.has_dim("nI") else None, 24 | }, 25 | ) 26 | 27 | 28 | def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: 29 | def backprop(d_output: InT) -> InT: 30 | dX = backprop_layer(d_output) 31 | if isinstance(d_output, list): 32 | return [d_output[i] + dX[i] for i in range(len(d_output))] 33 | elif isinstance(d_output, Ragged): 34 | return Ragged(d_output.data + dX.data, dX.lengths) 35 | elif isinstance(X, Padded): 36 | dX.data += d_output.data 37 | return dX 38 | else: 39 | return d_output + dX 40 | 41 | Y, backprop_layer = model.layers[0](X, is_train) 42 | if isinstance(X, list): 43 | return [X[i] + Y[i] for i in range(len(X))], backprop 44 | elif isinstance(X, Ragged): 45 | return Ragged(X.data + Y.data, X.lengths), backprop 46 | elif isinstance(X, Padded): 47 | Y.data += X.data 48 | return Y, backprop 49 | else: 50 | return X + Y, backprop 51 | 52 | 53 | def init( 54 | model: Model[InT, InT], X: Optional[InT] = None, Y: Optional[InT] = None 55 | ) -> None: 56 | first_layer = model.layers[0] 57 | if first_layer.has_dim("nO") is None: 58 | first_layer.initialize(X=X, Y=Y) 59 | else: 60 | first_layer.initialize(X=X) 61 | if first_layer.has_dim("nO"): 62 | model.set_dim("nO", first_layer.get_dim("nO")) 63 | if first_layer.has_dim("nI"): 64 | model.set_dim("nI", first_layer.get_dim("nI")) 65 | -------------------------------------------------------------------------------- /thinc/layers/siamese.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, TypeVar 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import ArrayXd 6 | from ..util import get_width 7 | 8 | LayerT = TypeVar("LayerT") 9 | SimT = TypeVar("SimT") 10 | InT = Tuple[LayerT, LayerT] 11 | OutT = TypeVar("OutT", bound=ArrayXd) 12 | 13 | 14 | @registry.layers("siamese.v1") 15 | def siamese( 16 | layer: Model[LayerT, SimT], similarity: Model[Tuple[SimT, SimT], OutT] 17 | ) -> Model[InT, OutT]: 18 | return Model( 19 | f"siamese({layer.name}, {similarity.name})", 20 | forward, 21 | init=init, 22 | layers=[layer, similarity], 23 | dims={"nI": layer.get_dim("nI"), "nO": similarity.get_dim("nO")}, 24 | ) 25 | 26 | 27 | def forward( 28 | model: Model[InT, OutT], X1_X2: InT, is_train: bool 29 | ) -> Tuple[OutT, Callable]: 30 | X1, X2 = X1_X2 31 | vec1, bp_vec1 = model.layers[0](X1, is_train) 32 | vec2, bp_vec2 = model.layers[0](X2, is_train) 33 | output, bp_output = model.layers[1]((vec1, vec2), is_train) 34 | 35 | def finish_update(d_output: OutT) -> InT: 36 | d_vec1, d_vec2 = bp_output(d_output) 37 | d_input1 = bp_vec1(d_vec1) 38 | d_input2 = bp_vec2(d_vec2) 39 | return (d_input1, d_input2) 40 | 41 | return output, finish_update 42 | 43 | 44 | def init( 45 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 46 | ) -> None: 47 | if X is not None: 48 | model.layers[0].set_dim("nI", get_width(X[1])) 49 | model.layers[0].initialize(X=X[0]) 50 | X = (model.layers[0].predict(X[0]), model.layers[0].predict(X[1])) 51 | model.layers[1].initialize(X=X, Y=Y) 52 | model.set_dim("nI", model.layers[0].get_dim("nI")) 53 | model.set_dim("nO", model.layers[1].get_dim("nO")) 54 | -------------------------------------------------------------------------------- /thinc/layers/sigmoid.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | 9 | InT = Floats2d 10 | OutT = Floats2d 11 | 12 | 13 | @registry.layers("Sigmoid.v1") 14 | def Sigmoid( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | ) -> Model[InT, OutT]: 21 | """A dense layer, followed by a sigmoid (logistic) activation function. This 22 | is usually used instead of the Softmax layer as an output for multi-label 23 | classification. 24 | """ 25 | if init_W is None: 26 | init_W = zero_init 27 | if init_b is None: 28 | init_b = zero_init 29 | return Model( 30 | "sigmoid", 31 | forward, 32 | init=partial(init, init_W, init_b), 33 | dims={"nO": nO, "nI": nI}, 34 | params={"W": None, "b": None}, 35 | ) 36 | 37 | 38 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 39 | W = cast(Floats2d, model.get_param("W")) 40 | b = cast(Floats1d, model.get_param("b")) 41 | Y = model.ops.affine(X, W, b) 42 | Y = model.ops.sigmoid(Y) 43 | 44 | def backprop(dY: InT) -> OutT: 45 | dY = model.ops.backprop_sigmoid(dY, Y, inplace=False) 46 | model.inc_grad("b", dY.sum(axis=0)) 47 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 48 | return model.ops.gemm(dY, W) 49 | 50 | return Y, backprop 51 | 52 | 53 | def init( 54 | init_W: Callable, 55 | init_b: Callable, 56 | model: Model[InT, OutT], 57 | X: Optional[InT] = None, 58 | Y: Optional[OutT] = None, 59 | ) -> None: 60 | if X is not None and model.has_dim("nI") is None: 61 | model.set_dim("nI", get_width(X)) 62 | if Y is not None and model.has_dim("nO") is None: 63 | model.set_dim("nO", get_width(Y)) 64 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 65 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 66 | -------------------------------------------------------------------------------- /thinc/layers/sigmoid_activation.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import FloatsXdT 6 | 7 | 8 | @registry.layers("sigmoid_activation.v1") 9 | def sigmoid_activation() -> Model[FloatsXdT, FloatsXdT]: 10 | return Model("sigmoid_activation", forward) 11 | 12 | 13 | def forward( 14 | model: Model[FloatsXdT, FloatsXdT], X: FloatsXdT, is_train: bool 15 | ) -> Tuple[FloatsXdT, Callable]: 16 | Y = model.ops.sigmoid(X, inplace=False) 17 | 18 | def backprop(dY: FloatsXdT) -> FloatsXdT: 19 | return cast( 20 | FloatsXdT, 21 | dY * model.ops.dsigmoid(Y, inplace=False), # type:ignore[operator] 22 | ) 23 | 24 | return Y, backprop 25 | -------------------------------------------------------------------------------- /thinc/layers/softmax_activation.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Floats2d 6 | 7 | InT = Floats2d 8 | OutT = Floats2d 9 | 10 | 11 | @registry.layers("softmax_activation.v1") 12 | def softmax_activation() -> Model[InT, OutT]: 13 | return Model("softmax_activation", forward) 14 | 15 | 16 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 17 | Y = model.ops.softmax(X, inplace=False) 18 | 19 | def backprop(dY: OutT) -> InT: 20 | return model.ops.backprop_softmax(Y, dY, axis=-1) 21 | 22 | return Y, backprop 23 | -------------------------------------------------------------------------------- /thinc/layers/strings2arrays.py: -------------------------------------------------------------------------------- 1 | from ctypes import c_uint64 2 | from typing import Callable, List, Sequence, Tuple 3 | 4 | from murmurhash import hash_unicode 5 | 6 | from ..config import registry 7 | from ..model import Model 8 | from ..types import Ints2d 9 | 10 | InT = Sequence[Sequence[str]] 11 | OutT = List[Ints2d] 12 | 13 | 14 | @registry.layers("strings2arrays.v1") 15 | def strings2arrays() -> Model[InT, OutT]: 16 | """Transform a sequence of string sequences to a list of arrays.""" 17 | return Model("strings2arrays", forward) 18 | 19 | 20 | def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: 21 | # Cast 32-bit (signed) integer to 64-bit unsigned, since such casting 22 | # is deprecated in NumPy. 23 | hashes = [[c_uint64(hash_unicode(word)).value for word in X] for X in Xs] 24 | hash_arrays = [model.ops.asarray1i(h, dtype="uint64") for h in hashes] 25 | arrays = [model.ops.reshape2i(array, -1, 1) for array in hash_arrays] 26 | 27 | def backprop(dX: OutT) -> InT: 28 | return [] 29 | 30 | return arrays, backprop 31 | -------------------------------------------------------------------------------- /thinc/layers/swish.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple, cast 2 | 3 | from ..config import registry 4 | from ..initializers import he_normal_init, zero_init 5 | from ..model import Model 6 | from ..types import Floats1d, Floats2d 7 | from ..util import get_width, partial 8 | from .chain import chain 9 | from .dropout import Dropout 10 | from .layernorm import LayerNorm 11 | 12 | 13 | @registry.layers("Swish.v1") 14 | def Swish( 15 | nO: Optional[int] = None, 16 | nI: Optional[int] = None, 17 | *, 18 | init_W: Optional[Callable] = None, 19 | init_b: Optional[Callable] = None, 20 | dropout: Optional[float] = None, 21 | normalize: bool = False, 22 | ) -> Model[Floats2d, Floats2d]: 23 | if init_W is None: 24 | init_W = he_normal_init 25 | if init_b is None: 26 | init_b = zero_init 27 | model: Model[Floats2d, Floats2d] = Model( 28 | "swish", 29 | forward, 30 | init=partial(init, init_W, init_b), 31 | dims={"nO": nO, "nI": nI}, 32 | params={"W": None, "b": None}, 33 | ) 34 | if normalize: 35 | model = chain(model, LayerNorm(nI=nO)) 36 | if dropout is not None: 37 | model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) 38 | return model 39 | 40 | 41 | def forward( 42 | model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool 43 | ) -> Tuple[Floats2d, Callable]: 44 | W = cast(Floats2d, model.get_param("W")) 45 | b = cast(Floats1d, model.get_param("b")) 46 | Y_preact = model.ops.affine(X, W, b) 47 | Y = model.ops.swish(Y_preact) 48 | 49 | def backprop(dY: Floats2d) -> Floats2d: 50 | dY = model.ops.backprop_swish(dY, Y_preact, Y, inplace=False) 51 | model.inc_grad("b", dY.sum(axis=0)) 52 | model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) 53 | return model.ops.gemm(dY, W) 54 | 55 | return Y, backprop 56 | 57 | 58 | def init( 59 | init_W: Callable, 60 | init_b: Callable, 61 | model: Model[Floats2d, Floats2d], 62 | X: Optional[Floats2d] = None, 63 | Y: Optional[Floats2d] = None, 64 | ) -> None: 65 | if X is not None: 66 | model.set_dim("nI", get_width(X)) 67 | if Y is not None: 68 | model.set_dim("nO", get_width(Y)) 69 | model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) 70 | model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) 71 | -------------------------------------------------------------------------------- /thinc/layers/tuplify.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, Tuple, TypeVar 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | 6 | InT = TypeVar("InT") 7 | OutT = Tuple 8 | 9 | 10 | @registry.layers("tuplify.v1") 11 | def tuplify( 12 | layer1: Model[InT, Any], layer2: Model[InT, Any], *layers 13 | ) -> Model[InT, Tuple]: 14 | """Send a separate copy of the input to each child layer, and join the 15 | outputs of the children into a tuple on the way out. 16 | 17 | Typically used to provide both modified data and the original input to a 18 | downstream layer. 19 | """ 20 | 21 | layers = (layer1, layer2) + layers 22 | names = [layer.name for layer in layers] 23 | return Model( 24 | "tuple(" + ", ".join(names) + ")", 25 | tuplify_forward, 26 | init=init, 27 | layers=layers, 28 | dims={"nI": None}, 29 | ) 30 | 31 | 32 | def tuplify_forward(model, X, is_train): 33 | Ys = [] 34 | backprops = [] 35 | for layer in model.layers: 36 | Y, backprop = layer(X, is_train) 37 | Ys.append(Y) 38 | backprops.append(backprop) 39 | 40 | def backprop_tuplify(dYs): 41 | dXs = [bp(dY) for bp, dY in zip(backprops, dYs)] 42 | dX = dXs[0] 43 | for dx in dXs[1:]: 44 | dX += dx 45 | return dX 46 | 47 | return tuple(Ys), backprop_tuplify 48 | 49 | 50 | def init( 51 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 52 | ) -> None: 53 | if X is None and Y is None: 54 | for layer in model.layers: 55 | layer.initialize() 56 | if model.layers[0].has_dim("nI"): 57 | model.set_dim("nI", model.layers[0].get_dim("nI")) 58 | 59 | # Try to set nO on each layer, where available. 60 | # All layers have the same input, and the output should map directly from the 61 | # given Y, if provided. 62 | for ii, layer in enumerate(model.layers): 63 | if Y is not None and layer.has_dim("nO") is None: 64 | layer.initialize(X=X, Y=Y[ii]) 65 | else: 66 | layer.initialize(X=X) 67 | 68 | if model.layers[0].has_dim("nI"): 69 | model.set_dim("nI", model.layers[0].get_dim("nI")) 70 | # this model can have an input dimension, but can't have an output dimension 71 | -------------------------------------------------------------------------------- /thinc/layers/uniqued.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional, Tuple 2 | 3 | import numpy 4 | 5 | from ..config import registry 6 | from ..model import Model 7 | from ..types import Floats2d, Ints2d 8 | 9 | InT = Ints2d 10 | OutT = Floats2d 11 | 12 | 13 | @registry.layers("uniqued.v1") 14 | def uniqued(layer: Model, *, column: int = 0) -> Model[InT, OutT]: 15 | """Group inputs to a layer, so that the layer only has to compute for the 16 | unique values. The data is transformed back before output, and the same 17 | transformation is applied for the gradient. Effectively, this is a cache 18 | local to each minibatch. 19 | """ 20 | return Model( 21 | f"uniqued({layer.name})", 22 | forward, 23 | init=init, 24 | layers=[layer], 25 | dims={"nO": None, "nI": None}, 26 | attrs={"column": column}, 27 | ) 28 | 29 | 30 | def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: 31 | column: int = model.attrs["column"] 32 | layer = model.layers[0] 33 | if X.size < 2: 34 | return layer(X, is_train) 35 | keys = X[:, column] 36 | if not isinstance(keys, numpy.ndarray): 37 | keys = keys.get() # pragma: no cover 38 | uniq_keys, ind, inv, counts = layer.ops.xp.unique( 39 | keys, return_index=True, return_inverse=True, return_counts=True 40 | ) 41 | counts = model.ops.reshape2i(counts, -1, 1) 42 | X_uniq = X[ind] 43 | Y_uniq, bp_Y_uniq = layer(X_uniq, is_train) 44 | Y = Y_uniq[inv].reshape((X.shape[0],) + Y_uniq.shape[1:]) 45 | uniq_shape = tuple(Y_uniq.shape) 46 | 47 | def backprop(dY: OutT) -> InT: 48 | dY_uniq = layer.ops.alloc2f(*uniq_shape) 49 | layer.ops.scatter_add(dY_uniq, layer.ops.asarray_i(inv), dY) 50 | d_uniques = bp_Y_uniq(dY_uniq) 51 | # This confusing bit of indexing "ununiques" 52 | return (d_uniques / counts)[inv] 53 | 54 | return Y, backprop 55 | 56 | 57 | def init( 58 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 59 | ) -> None: 60 | layer = model.layers[0] 61 | layer.initialize(X=X, Y=Y) 62 | if layer.has_dim("nI"): 63 | model.set_dim("nI", layer.get_dim("nI")) # pragma: no cover 64 | if layer.has_dim("nO"): 65 | model.set_dim("nO", layer.get_dim("nO")) 66 | -------------------------------------------------------------------------------- /thinc/layers/with_cpu.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Tuple 2 | 3 | import numpy 4 | 5 | from thinc.backends import Ops 6 | 7 | from ..config import registry 8 | from ..model import Model 9 | 10 | 11 | @registry.layers("with_cpu.v1") 12 | def with_cpu(layer: Model, ops: Ops) -> Model: 13 | layer.to_cpu() 14 | return Model( 15 | f"with_cpu({layer.name})", 16 | forward, 17 | layers=[layer], 18 | ops=ops, 19 | init=init, 20 | dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, 21 | ) 22 | 23 | 24 | def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: 25 | cpu_outputs, backprop = model.layers[0].begin_update(_to_cpu(X)) 26 | gpu_outputs = _to_device(model.ops, cpu_outputs) 27 | 28 | def with_cpu_backprop(d_outputs): 29 | cpu_d_outputs = _to_cpu(d_outputs) 30 | return backprop(cpu_d_outputs) 31 | 32 | return gpu_outputs, with_cpu_backprop 33 | 34 | 35 | def init(model: Model, X: Any, Y: Any) -> None: 36 | model.layers[0].initialize(X, Y) 37 | 38 | 39 | def _to_cpu(X): 40 | if isinstance(X, numpy.ndarray): 41 | return X 42 | elif isinstance(X, tuple): 43 | return tuple([_to_cpu(x) for x in X]) 44 | elif isinstance(X, list): 45 | return [_to_cpu(x) for x in X] 46 | elif hasattr(X, "get"): 47 | return X.get() 48 | else: 49 | return X 50 | 51 | 52 | def _to_device(ops, X): 53 | if isinstance(X, tuple): 54 | return tuple([_to_device(ops, x) for x in X]) 55 | elif isinstance(X, list): 56 | return [_to_device(ops, x) for x in X] 57 | else: 58 | return ops.asarray(X) 59 | -------------------------------------------------------------------------------- /thinc/layers/with_debug.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional, Tuple, TypeVar 2 | 3 | from ..model import Model 4 | 5 | _ModelT = TypeVar("_ModelT", bound=Model) 6 | 7 | do_nothing = lambda *args, **kwargs: None 8 | 9 | 10 | def with_debug( 11 | layer: _ModelT, 12 | name: Optional[str] = None, 13 | *, 14 | on_init: Callable[[Model, Any, Any], None] = do_nothing, 15 | on_forward: Callable[[Model, Any, bool], None] = do_nothing, 16 | on_backprop: Callable[[Any], None] = do_nothing, 17 | ) -> _ModelT: 18 | """Debugging layer that wraps any layer and allows executing callbacks 19 | during the forward pass, backward pass and initialization. The callbacks 20 | will receive the same arguments as the functions they're called in. 21 | """ 22 | name = layer.name if name is None else name 23 | 24 | orig_forward = layer._func 25 | orig_init = layer.init 26 | 27 | def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: 28 | on_forward(model, X, is_train) 29 | layer_Y, layer_callback = orig_forward(layer, X, is_train=is_train) 30 | 31 | def backprop(dY: Any) -> Any: 32 | on_backprop(dY) 33 | return layer_callback(dY) 34 | 35 | return layer_Y, backprop 36 | 37 | def init(model: Model, X: Any, Y: Any) -> None: 38 | on_init(model, X, Y) 39 | if orig_init is not None: 40 | orig_init(layer, X, Y) 41 | 42 | layer.replace_callbacks(forward, init=init) 43 | 44 | return layer 45 | -------------------------------------------------------------------------------- /thinc/layers/with_flatten.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, List, Optional, Sequence, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import ArrayXd, ListXd 6 | 7 | ItemT = TypeVar("ItemT") 8 | InT = Sequence[Sequence[ItemT]] 9 | OutT = ListXd 10 | InnerInT = Sequence[ItemT] 11 | InnerOutT = ArrayXd 12 | 13 | 14 | @registry.layers("with_flatten.v1") 15 | def with_flatten(layer: Model[InnerInT[ItemT], InnerOutT]) -> Model[InT[ItemT], OutT]: 16 | return Model(f"with_flatten({layer.name})", forward, layers=[layer], init=init) 17 | 18 | 19 | def forward( 20 | model: Model[InT, OutT], Xnest: InT, is_train: bool 21 | ) -> Tuple[OutT, Callable]: 22 | layer: Model[InnerInT, InnerOutT] = model.layers[0] 23 | Xflat = _flatten(Xnest) 24 | Yflat, backprop_layer = layer(Xflat, is_train) 25 | # Get the split points. We want n-1 splits for n items. 26 | arr = layer.ops.asarray1i([len(x) for x in Xnest[:-1]]) 27 | splits = arr.cumsum() 28 | Ynest = layer.ops.xp.split(Yflat, splits, axis=0) 29 | 30 | def backprop(dYnest: OutT) -> InT: 31 | dYflat = model.ops.flatten(dYnest) # type: ignore[arg-type, var-annotated] 32 | # type ignore necessary for older versions of Mypy/Pydantic 33 | dXflat = backprop_layer(dYflat) 34 | dXnest = layer.ops.xp.split(dXflat, splits, axis=-1) 35 | return dXnest 36 | 37 | return Ynest, backprop 38 | 39 | 40 | def _flatten(nested: InT) -> InnerInT: 41 | flat: List = [] 42 | for item in nested: 43 | flat.extend(item) 44 | return cast(InT, flat) 45 | 46 | 47 | def init( 48 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 49 | ) -> None: 50 | model.layers[0].initialize( 51 | _flatten(X) if X is not None else None, 52 | model.layers[0].ops.xp.hstack(Y) if Y is not None else None, 53 | ) 54 | -------------------------------------------------------------------------------- /thinc/layers/with_flatten_v2.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, List, Optional, Sequence, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | 6 | InItemT = TypeVar("InItemT") 7 | OutItemT = TypeVar("OutItemT") 8 | ItemT = TypeVar("ItemT") 9 | 10 | NestedT = List[List[ItemT]] 11 | FlatT = List[ItemT] 12 | 13 | 14 | @registry.layers("with_flatten.v2") 15 | def with_flatten_v2( 16 | layer: Model[FlatT[InItemT], FlatT[OutItemT]] 17 | ) -> Model[NestedT[InItemT], NestedT[OutItemT]]: 18 | return Model(f"with_flatten({layer.name})", forward, layers=[layer], init=init) 19 | 20 | 21 | def forward( 22 | model: Model[NestedT[InItemT], NestedT[OutItemT]], 23 | Xnest: NestedT[InItemT], 24 | is_train: bool, 25 | ) -> Tuple[NestedT[OutItemT], Callable]: 26 | layer: Model[FlatT[InItemT], FlatT[OutItemT]] = model.layers[0] 27 | Xflat, lens = _flatten(Xnest) 28 | Yflat, backprop_layer = layer(Xflat, is_train) 29 | Ynest = _unflatten(Yflat, lens) 30 | 31 | def backprop(dYnest: NestedT[InItemT]) -> NestedT[OutItemT]: 32 | dYflat, _ = _flatten(dYnest) # type: ignore[arg-type, var-annotated] 33 | # type ignore necessary for older versions of Mypy/Pydantic 34 | dXflat = backprop_layer(dYflat) 35 | dXnest = _unflatten(dXflat, lens) 36 | return dXnest 37 | 38 | return Ynest, backprop 39 | 40 | 41 | def _flatten(nested: NestedT[ItemT]) -> Tuple[FlatT[ItemT], List[int]]: 42 | flat: List = [] 43 | lens: List[int] = [] 44 | for item in nested: 45 | flat.extend(item) 46 | lens.append(len(item)) 47 | return cast(FlatT[ItemT], flat), lens 48 | 49 | 50 | def _unflatten(flat: FlatT[ItemT], lens: List[int]) -> NestedT[ItemT]: 51 | nested = [] 52 | for l in lens: 53 | nested.append(flat[:l]) 54 | flat = flat[l:] 55 | return nested 56 | 57 | 58 | def init( 59 | model: Model[NestedT[InItemT], NestedT[OutItemT]], 60 | X: Optional[NestedT[InItemT]] = None, 61 | Y: Optional[NestedT[OutItemT]] = None, 62 | ) -> None: 63 | model.layers[0].initialize( 64 | _flatten(X)[0] if X is not None else None, 65 | model.layers[0].ops.xp.hstack(Y) if Y is not None else None, 66 | ) 67 | -------------------------------------------------------------------------------- /thinc/layers/with_getitem.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional, Tuple 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | 6 | InT = Tuple[Any, ...] 7 | OutT = Tuple[Any, ...] 8 | 9 | 10 | @registry.layers("with_getitem.v1") 11 | def with_getitem(idx: int, layer: Model) -> Model[InT, OutT]: 12 | """Transform data on the way into and out of a layer, by plucking an item 13 | from a tuple. 14 | """ 15 | return Model( 16 | f"with_getitem({layer.name})", 17 | forward, 18 | init=init, 19 | layers=[layer], 20 | attrs={"idx": idx}, 21 | ) 22 | 23 | 24 | def forward( 25 | model: Model[InT, OutT], items: InT, is_train: bool 26 | ) -> Tuple[OutT, Callable]: 27 | idx = model.attrs["idx"] 28 | Y_i, backprop_item = model.layers[0](items[idx], is_train) 29 | 30 | def backprop(d_output: OutT) -> InT: 31 | dY_i = backprop_item(d_output[idx]) 32 | return d_output[:idx] + (dY_i,) + d_output[idx + 1 :] 33 | 34 | return items[:idx] + (Y_i,) + items[idx + 1 :], backprop 35 | 36 | 37 | def init( 38 | model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None 39 | ) -> None: 40 | idx = model.attrs["idx"] 41 | X_i = X[idx] if X is not None else X 42 | Y_i = Y[idx] if Y is not None else Y 43 | model.layers[0].initialize(X=X_i, Y=Y_i) 44 | -------------------------------------------------------------------------------- /thinc/layers/with_nvtx_range.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional, Tuple, TypeVar 2 | 3 | from ..model import Model 4 | from ..util import use_nvtx_range 5 | 6 | _ModelT = TypeVar("_ModelT", bound=Model) 7 | 8 | 9 | def with_nvtx_range( 10 | layer: _ModelT, 11 | name: Optional[str] = None, 12 | *, 13 | forward_color: int = -1, 14 | backprop_color: int = -1, 15 | ) -> _ModelT: 16 | """Wraps any layer and marks the forward and backprop phases as 17 | NVTX ranges for CUDA profiling. 18 | 19 | By default, the name of the layer is used as the name of the range, 20 | followed by the name of the pass. 21 | """ 22 | name = layer.name if name is None else name 23 | 24 | orig_forward = layer._func 25 | orig_init = layer.init 26 | 27 | def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: 28 | with use_nvtx_range(f"{name} forward", forward_color): 29 | layer_Y, layer_callback = orig_forward(model, X, is_train=is_train) 30 | 31 | def backprop(dY: Any) -> Any: 32 | with use_nvtx_range(f"{name} backprop", backprop_color): 33 | return layer_callback(dY) 34 | 35 | return layer_Y, backprop 36 | 37 | def init(_model: Model, X: Any, Y: Any) -> Model: 38 | if orig_init is not None: 39 | return orig_init(layer, X, Y) 40 | else: 41 | return layer 42 | 43 | layer.replace_callbacks(forward, init=init) 44 | 45 | return layer 46 | -------------------------------------------------------------------------------- /thinc/layers/with_reshape.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Optional, Tuple, TypeVar, cast 2 | 3 | from ..config import registry 4 | from ..model import Model 5 | from ..types import Array2d, Array3d 6 | 7 | InT = TypeVar("InT", bound=Array3d) 8 | OutT = TypeVar("OutT", bound=Array2d) 9 | 10 | 11 | @registry.layers("with_reshape.v1") 12 | def with_reshape(layer: Model[OutT, OutT]) -> Model[InT, InT]: 13 | """Reshape data on the way into and out from a layer.""" 14 | return Model( 15 | f"with_reshape({layer.name})", 16 | forward, 17 | init=init, 18 | layers=[layer], 19 | dims={"nO": None, "nI": None}, 20 | ) 21 | 22 | 23 | def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: 24 | layer = model.layers[0] 25 | initial_shape = X.shape 26 | final_shape = list(initial_shape[:-1]) + [layer.get_dim("nO")] 27 | nB = X.shape[0] 28 | nT = X.shape[1] 29 | X2d = model.ops.reshape(X, (-1, X.shape[2])) 30 | Y2d, Y2d_backprop = layer(X2d, is_train=is_train) 31 | Y = model.ops.reshape3(Y2d, *final_shape) 32 | 33 | def backprop(dY: InT) -> InT: 34 | reshaped = model.ops.reshape2(dY, nB * nT, -1) 35 | return Y2d_backprop(model.ops.reshape3(reshaped, *initial_shape)) 36 | 37 | return cast(InT, Y), backprop 38 | 39 | 40 | def init( 41 | model: Model[InT, InT], X: Optional[Array3d] = None, Y: Optional[Array3d] = None 42 | ) -> None: 43 | layer = model.layers[0] 44 | if X is None and Y is None: 45 | layer.initialize() 46 | X2d: Optional[Array2d] = None 47 | Y2d: Optional[Array2d] = None 48 | if X is not None: 49 | X2d = cast(Array2d, model.ops.reshape(X, (-1, X.shape[-1]))) 50 | if Y is not None: 51 | Y2d = cast(Array2d, model.ops.reshape(Y, (-1, Y.shape[-1]))) 52 | layer.initialize(X=X2d, Y=Y2d) 53 | if layer.has_dim("nI"): 54 | model.set_dim("nI", layer.get_dim("nI")) 55 | if layer.has_dim("nO"): 56 | model.set_dim("nO", layer.get_dim("nO")) 57 | -------------------------------------------------------------------------------- /thinc/layers/with_signpost_interval.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Optional, Tuple, TypeVar 2 | 3 | from ..compat import has_os_signpost, os_signpost 4 | from ..model import Model 5 | 6 | _ModelT = TypeVar("_ModelT", bound=Model) 7 | 8 | 9 | def with_signpost_interval( 10 | layer: _ModelT, 11 | signposter: "os_signpost.Signposter", 12 | name: Optional[str] = None, 13 | ) -> _ModelT: 14 | """Wraps any layer and marks the init, forward and backprop phases using 15 | signpost intervals for macOS Instruments profiling 16 | 17 | By default, the name of the layer is used as the name of the range, 18 | followed by the name of the pass. 19 | """ 20 | if not has_os_signpost: 21 | raise ValueError( 22 | "with_signpost_interval layer requires the 'os_signpost' package" 23 | ) 24 | 25 | name = layer.name if name is None else name 26 | 27 | orig_forward = layer._func 28 | orig_init = layer.init 29 | 30 | def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: 31 | with signposter.use_interval(f"{name} forward"): 32 | layer_Y, layer_callback = orig_forward(model, X, is_train=is_train) 33 | 34 | def backprop(dY: Any) -> Any: 35 | with signposter.use_interval(f"{name} backprop"): 36 | return layer_callback(dY) 37 | 38 | return layer_Y, backprop 39 | 40 | def init(_model: Model, X: Any, Y: Any) -> Model: 41 | if orig_init is not None: 42 | with signposter.use_interval(f"{name} init"): 43 | return orig_init(layer, X, Y) 44 | else: 45 | return layer 46 | 47 | layer.replace_callbacks(forward, init=init) 48 | 49 | return layer 50 | -------------------------------------------------------------------------------- /thinc/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/py.typed -------------------------------------------------------------------------------- /thinc/shims/__init__.py: -------------------------------------------------------------------------------- 1 | from .mxnet import MXNetShim 2 | from .pytorch import PyTorchShim 3 | from .pytorch_grad_scaler import PyTorchGradScaler 4 | from .shim import Shim 5 | from .tensorflow import TensorFlowShim, keras_model_fns, maybe_handshake_model 6 | from .torchscript import TorchScriptShim 7 | 8 | # fmt: off 9 | __all__ = [ 10 | "MXNetShim", 11 | "PyTorchShim", 12 | "PyTorchGradScaler", 13 | "Shim", 14 | "TensorFlowShim", 15 | "TorchScriptShim", 16 | "maybe_handshake_model", 17 | "keras_model_fns", 18 | ] 19 | # fmt: on 20 | -------------------------------------------------------------------------------- /thinc/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/__init__.py -------------------------------------------------------------------------------- /thinc/tests/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/backends/__init__.py -------------------------------------------------------------------------------- /thinc/tests/backends/_apple_blas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/backends/_apple_blas/__init__.py -------------------------------------------------------------------------------- /thinc/tests/backends/test_mem.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | from thinc.backends._param_server import ParamServer 4 | 5 | 6 | def test_param_server_init(): 7 | array = numpy.zeros((5,), dtype="f") 8 | params = {("a", 1): array, ("b", 2): array} 9 | grads = {("a", 1): array, ("c", 3): array} 10 | ps = ParamServer(params, grads) 11 | assert ps.param_keys == (("a", 1), ("b", 2)) 12 | assert ps.grad_keys == (("a", 1),) 13 | -------------------------------------------------------------------------------- /thinc/tests/backends/test_mps_ops.py: -------------------------------------------------------------------------------- 1 | from thinc.api import NumpyOps, get_ops 2 | from thinc.compat import has_apple_ops 3 | 4 | 5 | def test_mps_ops_inherits_apple_ops(): 6 | ops = get_ops("mps") 7 | assert isinstance(ops, NumpyOps) 8 | if has_apple_ops: 9 | # We can't import AppleOps directly, because its' not 10 | # available on non-Darwin systems. 11 | assert "AppleOps" in [base.__name__ for base in type(ops).__bases__] 12 | -------------------------------------------------------------------------------- /thinc/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from hypothesis import settings 3 | 4 | # Functionally disable deadline settings for tests 5 | # to prevent spurious test failures in CI builds. 6 | settings.register_profile("no_deadlines", deadline=2 * 60 * 1000) # in ms 7 | settings.load_profile("no_deadlines") 8 | 9 | 10 | def pytest_sessionstart(session): 11 | # If Tensorflow is installed, attempt to enable memory growth 12 | # to prevent it from allocating all of the GPU's free memory 13 | # to its internal memory pool(s). 14 | try: 15 | import tensorflow as tf 16 | 17 | physical_devices = tf.config.list_physical_devices("GPU") 18 | for device in physical_devices: 19 | try: 20 | tf.config.experimental.set_memory_growth(device, True) 21 | except: 22 | # Invalid device or cannot modify virtual devices once initialized. 23 | print(f"failed to enable Tensorflow memory growth on {device}") 24 | except ImportError: 25 | pass 26 | 27 | 28 | def pytest_addoption(parser): 29 | try: 30 | parser.addoption("--slow", action="store_true", help="include slow tests") 31 | # Options are already added, e.g. if conftest is copied in a build pipeline 32 | # and runs twice 33 | except ValueError: 34 | pass 35 | 36 | 37 | def pytest_runtest_setup(item): 38 | def getopt(opt): 39 | # When using 'pytest --pyargs thinc' to test an installed copy of 40 | # thinc, pytest skips running our pytest_addoption() hook. Later, when 41 | # we call getoption(), pytest raises an error, because it doesn't 42 | # recognize the option we're asking about. To avoid this, we need to 43 | # pass a default value. We default to False, i.e., we act like all the 44 | # options weren't given. 45 | return item.config.getoption(f"--{opt}", False) 46 | 47 | for opt in ["slow"]: 48 | if opt in item.keywords and not getopt(opt): 49 | pytest.skip(f"need --{opt} option to run") 50 | 51 | 52 | @pytest.fixture() 53 | def pathy_fixture(): 54 | pytest.importorskip("pathy") 55 | import shutil 56 | import tempfile 57 | 58 | from pathy import Pathy, use_fs 59 | 60 | temp_folder = tempfile.mkdtemp(prefix="thinc-pathy") 61 | use_fs(temp_folder) 62 | 63 | root = Pathy("gs://test-bucket") 64 | root.mkdir(exist_ok=True) 65 | 66 | yield root 67 | use_fs(False) 68 | shutil.rmtree(temp_folder) 69 | -------------------------------------------------------------------------------- /thinc/tests/enable_mxnet.py: -------------------------------------------------------------------------------- 1 | from thinc.compat import enable_mxnet 2 | 3 | try: 4 | enable_mxnet() 5 | except ImportError: 6 | pass 7 | -------------------------------------------------------------------------------- /thinc/tests/enable_tensorflow.py: -------------------------------------------------------------------------------- 1 | from thinc.compat import enable_tensorflow 2 | 3 | try: 4 | enable_tensorflow() 5 | except ImportError: 6 | pass 7 | -------------------------------------------------------------------------------- /thinc/tests/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/layers/__init__.py -------------------------------------------------------------------------------- /thinc/tests/layers/test_hash_embed.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | from thinc.api import HashEmbed 4 | 5 | 6 | def test_init(): 7 | model = HashEmbed(64, 1000).initialize() 8 | assert model.get_dim("nV") == 1000 9 | assert model.get_dim("nO") == 64 10 | assert model.get_param("E").shape == (1000, 64) 11 | 12 | 13 | def test_seed_changes_bucket(): 14 | model1 = HashEmbed(64, 1000, seed=2).initialize() 15 | model2 = HashEmbed(64, 1000, seed=1).initialize() 16 | arr = numpy.ones((1,), dtype="uint64") 17 | vector1 = model1.predict(arr) 18 | vector2 = model2.predict(arr) 19 | assert vector1.sum() != vector2.sum() 20 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_mappers.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | 4 | from thinc.layers import premap_ids, remap_ids, remap_ids_v2 5 | 6 | 7 | @pytest.fixture 8 | def keys(): 9 | return numpy.array([4, 2, 6, 1, 8, 7, 9, 3, 30]) 10 | 11 | 12 | @pytest.fixture 13 | def mapper(keys): 14 | return {int(k): int(v) for v, k in enumerate(keys)} 15 | 16 | 17 | def test_premap(keys, mapper): 18 | premap = premap_ids(mapper, default=99) 19 | values, _ = premap(keys, False) 20 | numpy.testing.assert_equal(values.squeeze(), numpy.asarray(range(len(keys)))) 21 | 22 | 23 | def test_remap(keys, mapper): 24 | remap = remap_ids(mapper, default=99) 25 | values, _ = remap(keys, False) 26 | numpy.testing.assert_equal(values.squeeze(), numpy.asarray(range(len(keys)))) 27 | 28 | 29 | def test_remap_v2(keys, mapper): 30 | remap = remap_ids_v2(mapper, default=99) 31 | values, _ = remap(keys, False) 32 | numpy.testing.assert_equal(values.squeeze(), numpy.asarray(range(len(keys)))) 33 | 34 | 35 | def test_remap_premap_eq(keys, mapper): 36 | remap = remap_ids(mapper, default=99) 37 | remap_v2 = remap_ids_v2(mapper, default=99) 38 | premap = premap_ids(mapper, default=99) 39 | values1, _ = remap(keys, False) 40 | values2, _ = remap_v2(keys, False) 41 | values3, _ = premap(keys, False) 42 | numpy.testing.assert_equal(values1, values2) 43 | numpy.testing.assert_equal(values2, values3) 44 | 45 | 46 | def test_column(keys, mapper): 47 | idx = numpy.zeros((len(keys), 4), dtype="int") 48 | idx[:, 3] = keys 49 | remap_v2 = remap_ids_v2(mapper, column=3) 50 | premap = premap_ids(mapper, column=3) 51 | numpy.testing.assert_equal( 52 | remap_v2(idx, False)[0].squeeze(), numpy.asarray(range(len(keys))) 53 | ) 54 | numpy.testing.assert_equal( 55 | premap(idx, False)[0].squeeze(), numpy.asarray(range(len(keys))) 56 | ) 57 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_parametric_attention_v2.py: -------------------------------------------------------------------------------- 1 | from thinc.layers.gelu import Gelu 2 | from thinc.layers.parametricattention_v2 import ( 3 | KEY_TRANSFORM_REF, 4 | ParametricAttention_v2, 5 | ) 6 | 7 | 8 | def test_key_transform_used(): 9 | attn = ParametricAttention_v2(key_transform=Gelu()) 10 | assert attn.get_ref(KEY_TRANSFORM_REF).name == "gelu" 11 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_resizable.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import pytest 4 | 5 | from thinc.api import Linear, resizable 6 | from thinc.layers.resizable import resize_linear_weighted, resize_model 7 | 8 | 9 | @pytest.fixture 10 | def model(): 11 | output_layer = Linear(nO=None, nI=None) 12 | fill_defaults = {"b": 0, "W": 0} 13 | model = resizable( 14 | output_layer, 15 | resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults), 16 | ) 17 | return model 18 | 19 | 20 | def test_resizable_linear_default_name(model): 21 | assert model.name == "resizable(linear)" 22 | 23 | 24 | def test_resize_model(model): 25 | """Test that resizing the model doesn't cause an exception.""" 26 | resize_model(model, new_nO=10) 27 | resize_model(model, new_nO=11) 28 | 29 | model.set_dim("nO", 0, force=True) 30 | resize_model(model, new_nO=10) 31 | 32 | model.set_dim("nI", 10, force=True) 33 | model.set_dim("nO", 0, force=True) 34 | resize_model(model, new_nO=10) 35 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_shim.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from thinc.shims.shim import Shim 4 | 5 | from ..util import make_tempdir 6 | 7 | 8 | class MockShim(Shim): 9 | def __init__(self, data: List[int]): 10 | super().__init__(None, config=None, optimizer=None) 11 | self.data = data 12 | 13 | def to_bytes(self): 14 | return bytes(self.data) 15 | 16 | def from_bytes(self, data: bytes) -> "MockShim": 17 | return MockShim(data=list(data)) 18 | 19 | 20 | def test_shim_can_roundtrip_with_path(): 21 | 22 | with make_tempdir() as path: 23 | shim_path = path / "cool_shim.data" 24 | shim = MockShim([1, 2, 3]) 25 | shim.to_disk(shim_path) 26 | copy_shim = shim.from_disk(shim_path) 27 | assert copy_shim.to_bytes() == shim.to_bytes() 28 | 29 | 30 | def test_shim_can_roundtrip_with_path_subclass(pathy_fixture): 31 | shim_path = pathy_fixture / "cool_shim.data" 32 | shim = MockShim([1, 2, 3]) 33 | shim.to_disk(shim_path) 34 | copy_shim = shim.from_disk(shim_path) 35 | assert copy_shim.to_bytes() == shim.to_bytes() 36 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_torchscriptwrapper.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | 4 | from thinc.api import ( 5 | PyTorchWrapper_v2, 6 | TorchScriptWrapper_v1, 7 | pytorch_to_torchscript_wrapper, 8 | ) 9 | from thinc.compat import has_torch, torch 10 | 11 | 12 | @pytest.mark.skipif(not has_torch, reason="needs PyTorch") 13 | @pytest.mark.parametrize("nN,nI,nO", [(2, 3, 4)]) 14 | def test_pytorch_script(nN, nI, nO): 15 | 16 | model = PyTorchWrapper_v2(torch.nn.Linear(nI, nO)).initialize() 17 | script_model = pytorch_to_torchscript_wrapper(model) 18 | 19 | X = numpy.random.randn(nN, nI).astype("f") 20 | Y = model.predict(X) 21 | Y_script = script_model.predict(X) 22 | numpy.testing.assert_allclose(Y, Y_script) 23 | 24 | serialized = script_model.to_bytes() 25 | script_model2 = TorchScriptWrapper_v1() 26 | script_model2.from_bytes(serialized) 27 | 28 | numpy.testing.assert_allclose(Y, script_model2.predict(X)) 29 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_transforms.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | 4 | from thinc.api import NumpyOps, Ragged, registry, strings2arrays 5 | 6 | from ..util import get_data_checker 7 | 8 | 9 | @pytest.fixture(params=[[], [(10, 2)], [(5, 3), (1, 3)], [(2, 3), (0, 3), (1, 3)]]) 10 | def shapes(request): 11 | return request.param 12 | 13 | 14 | @pytest.fixture 15 | def ops(): 16 | return NumpyOps() 17 | 18 | 19 | @pytest.fixture 20 | def list_data(shapes): 21 | return [numpy.zeros(shape, dtype="f") for shape in shapes] 22 | 23 | 24 | @pytest.fixture 25 | def ragged_data(ops, list_data): 26 | lengths = numpy.array([len(x) for x in list_data], dtype="i") 27 | if not list_data: 28 | return Ragged(ops.alloc2f(0, 0), lengths) 29 | else: 30 | return Ragged(ops.flatten(list_data), lengths) 31 | 32 | 33 | @pytest.fixture 34 | def padded_data(ops, list_data): 35 | return ops.list2padded(list_data) 36 | 37 | 38 | @pytest.fixture 39 | def array_data(ragged_data): 40 | return ragged_data.data 41 | 42 | 43 | def check_transform(transform, in_data, out_data): 44 | model = registry.resolve({"config": {"@layers": transform}})["config"] 45 | input_checker = get_data_checker(in_data) 46 | output_checker = get_data_checker(out_data) 47 | model.initialize(in_data, out_data) 48 | Y, backprop = model(in_data, is_train=True) 49 | output_checker(Y, out_data) 50 | dX = backprop(Y) 51 | input_checker(dX, in_data) 52 | 53 | 54 | def test_list2array(list_data, array_data): 55 | check_transform("list2array.v1", list_data, array_data) 56 | 57 | 58 | def test_list2ragged(list_data, ragged_data): 59 | check_transform("list2ragged.v1", list_data, ragged_data) 60 | 61 | 62 | def test_list2padded(list_data, padded_data): 63 | check_transform("list2padded.v1", list_data, padded_data) 64 | 65 | 66 | def test_ragged2list(ragged_data, list_data): 67 | check_transform("ragged2list.v1", ragged_data, list_data) 68 | 69 | 70 | def test_padded2list(padded_data, list_data): 71 | check_transform("padded2list.v1", padded_data, list_data) 72 | 73 | 74 | def test_strings2arrays(): 75 | strings = ["hello", "world"] 76 | model = strings2arrays() 77 | Y, backprop = model.begin_update(strings) 78 | assert len(Y) == len(strings) 79 | assert backprop([]) == [] 80 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_uniqued.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | from hypothesis import given, settings 4 | from hypothesis.strategies import composite, integers, lists 5 | from numpy.testing import assert_allclose 6 | 7 | from thinc.layers import Embed 8 | from thinc.layers.uniqued import uniqued 9 | 10 | ROWS = 10 11 | 12 | # This test uses a newer hypothesis feature than the skanky flatmap-style 13 | # I used previously. This is much nicer, although it still takes some getting 14 | # used to. The key feature is this composite decorator. It injects a function, 15 | # 'draw'. 16 | 17 | 18 | @composite 19 | def lists_of_integers(draw, columns=2, lo=0, hi=ROWS - 1): 20 | # We call draw to get example values, which we can manipulate. 21 | # Here we get a list of integers, where each member of the list 22 | # should be between a min and max value. 23 | int_list = draw(lists(integers(min_value=lo, max_value=hi))) 24 | # Now we can use this int list to make an array, and it'll be the arrays 25 | # that our functions receive. 26 | # We trim the list, so we're of length divisible by columns. 27 | int_list = int_list[len(int_list) % columns :] 28 | # And make the array and reshape it. 29 | array = numpy.array(int_list, dtype="uint64") 30 | return array.reshape((-1, columns)) 31 | 32 | 33 | @pytest.fixture(scope="module") 34 | def model(nO=128): 35 | return Embed(nO, ROWS, column=0).initialize() 36 | 37 | 38 | def test_uniqued_calls_init(): 39 | calls = [] 40 | embed = Embed(5, 5, column=0) 41 | embed.init = lambda *args, **kwargs: calls.append(True) 42 | embed.initialize() 43 | assert calls == [True] 44 | uembed = uniqued(embed) 45 | uembed.initialize() 46 | assert calls == [True, True] 47 | 48 | 49 | @given(X=lists_of_integers(lo=0, hi=ROWS - 1)) 50 | @settings(deadline=None) 51 | def test_uniqued_doesnt_change_result(model, X): 52 | umodel = uniqued(model, column=model.attrs["column"]).initialize() 53 | Y, bp_Y = model(X, is_train=True) 54 | Yu, bp_Yu = umodel(X, is_train=True) 55 | assert_allclose(Y, Yu) 56 | dX = bp_Y(Y) 57 | dXu = bp_Yu(Yu) 58 | assert_allclose(dX, dXu) 59 | if X.size: 60 | pass 61 | # TODO: This test is a problem, because we exceed the embedding table. 62 | # Fix it with a better cap. 63 | # Check that different inputs do give different results 64 | # Z, bp_Z = model(X + 1, is_train=True) 65 | # with pytest.raises(AssertionError): 66 | # assert_allclose(Y, Z) 67 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_with_debug.py: -------------------------------------------------------------------------------- 1 | from mock import MagicMock 2 | 3 | from thinc.api import Linear, with_debug 4 | 5 | 6 | def test_with_debug(): 7 | on_init = MagicMock() 8 | on_forward = MagicMock() 9 | on_backprop = MagicMock() 10 | model = with_debug( 11 | Linear(), on_init=on_init, on_forward=on_forward, on_backprop=on_backprop 12 | ) 13 | on_init.assert_not_called() 14 | on_forward.assert_not_called() 15 | on_backprop.assert_not_called() 16 | X = model.ops.alloc2f(1, 1) 17 | Y = model.ops.alloc2f(1, 1) 18 | model.initialize(X=X, Y=Y) 19 | on_init.assert_called_once_with(model, X, Y) 20 | on_forward.assert_not_called() 21 | on_backprop.assert_not_called() 22 | Yh, backprop = model(X, is_train=True) 23 | on_forward.assert_called_once_with(model, X, True) 24 | on_backprop.assert_not_called() 25 | backprop(Y) 26 | on_backprop.assert_called_once_with(Y) 27 | -------------------------------------------------------------------------------- /thinc/tests/layers/test_with_flatten.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from thinc.api import Model, with_flatten_v2 4 | 5 | INPUT = [[1, 2, 3], [4, 5], [], [6, 7, 8]] 6 | INPUT_FLAT = [1, 2, 3, 4, 5, 6, 7, 8] 7 | OUTPUT = [[2, 3, 4], [5, 6], [], [7, 8, 9]] 8 | BACKPROP_OUTPUT = [[3, 4, 5], [6, 7], [], [8, 9, 10]] 9 | 10 | 11 | def _memoize_input() -> Model[List[int], List[int]]: 12 | return Model(name="memoize_input", forward=_memoize_input_forward) 13 | 14 | 15 | def _memoize_input_forward( 16 | model: Model[List[int], List[int]], X: List[int], is_train: bool 17 | ): 18 | model.attrs["last_input"] = X 19 | 20 | def backprop(dY: List[int]): 21 | return [v + 2 for v in dY] 22 | 23 | return [v + 1 for v in X], backprop 24 | 25 | 26 | def test_with_flatten(): 27 | model = with_flatten_v2(_memoize_input()) 28 | Y, backprop = model(INPUT, is_train=True) 29 | assert Y == OUTPUT 30 | assert model.layers[0].attrs["last_input"] == INPUT_FLAT 31 | assert backprop(INPUT) == BACKPROP_OUTPUT 32 | -------------------------------------------------------------------------------- /thinc/tests/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/model/__init__.py -------------------------------------------------------------------------------- /thinc/tests/model/test_validation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from thinc.api import ( 4 | ParametricAttention, 5 | Relu, 6 | Softmax, 7 | chain, 8 | list2ragged, 9 | reduce_max, 10 | reduce_sum, 11 | with_ragged, 12 | ) 13 | from thinc.util import DataValidationError, data_validation 14 | 15 | 16 | def test_validation(): 17 | model = chain(Relu(10), Relu(10), with_ragged(reduce_max()), Softmax()) 18 | with data_validation(True): 19 | with pytest.raises(DataValidationError): 20 | model.initialize(X=model.ops.alloc2f(1, 10), Y=model.ops.alloc2f(1, 10)) 21 | with pytest.raises(DataValidationError): 22 | model.initialize(X=model.ops.alloc3f(1, 10, 1), Y=model.ops.alloc2f(1, 10)) 23 | with pytest.raises(DataValidationError): 24 | model.initialize(X=[model.ops.alloc2f(1, 10)], Y=model.ops.alloc2f(1, 10)) 25 | 26 | 27 | def test_validation_complex(): 28 | good_model = chain(list2ragged(), reduce_sum(), Relu(12, dropout=0.5), Relu(1)) 29 | X = [good_model.ops.xp.zeros((4, 75), dtype="f")] 30 | Y = good_model.ops.xp.zeros((1,), dtype="f") 31 | good_model.initialize(X, Y) 32 | good_model.predict(X) 33 | 34 | bad_model = chain( 35 | list2ragged(), 36 | reduce_sum(), 37 | Relu(12, dropout=0.5), 38 | # ERROR: Why can't I attach a Relu to an attention layer? 39 | ParametricAttention(12), 40 | Relu(1), 41 | ) 42 | with data_validation(True): 43 | with pytest.raises(DataValidationError): 44 | bad_model.initialize(X, Y) 45 | -------------------------------------------------------------------------------- /thinc/tests/mypy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/mypy/__init__.py -------------------------------------------------------------------------------- /thinc/tests/mypy/configs/mypy-default.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | follow_imports = silent 3 | strict_optional = True 4 | warn_redundant_casts = True 5 | warn_unused_ignores = True 6 | # disallow_any_generics = True 7 | check_untyped_defs = True 8 | disallow_untyped_defs = True 9 | -------------------------------------------------------------------------------- /thinc/tests/mypy/configs/mypy-plugin.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | plugins = thinc.mypy 3 | 4 | follow_imports = silent 5 | strict_optional = True 6 | warn_redundant_casts = True 7 | warn_unused_ignores = True 8 | # disallow_any_generics = True 9 | check_untyped_defs = True 10 | disallow_untyped_defs = True 11 | -------------------------------------------------------------------------------- /thinc/tests/mypy/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/mypy/modules/__init__.py -------------------------------------------------------------------------------- /thinc/tests/mypy/modules/fail_no_plugin.py: -------------------------------------------------------------------------------- 1 | from thinc.api import Relu, Softmax, add, chain, reduce_max 2 | 3 | bad_model = chain(Relu(10), reduce_max(), Softmax()) 4 | 5 | bad_model2 = add(Relu(10), reduce_max(), Softmax()) 6 | -------------------------------------------------------------------------------- /thinc/tests/mypy/modules/fail_plugin.py: -------------------------------------------------------------------------------- 1 | from thinc.api import Relu, Softmax, add, chain, concatenate, reduce_max 2 | 3 | bad_model = chain(Relu(10), reduce_max(), Softmax()) 4 | 5 | bad_model2 = add(Relu(10), reduce_max(), Softmax()) 6 | 7 | bad_model_only_plugin = chain( 8 | Relu(10), Relu(10), Relu(10), Relu(10), reduce_max(), Softmax() 9 | ) 10 | 11 | bad_model_only_plugin2 = add( 12 | Relu(10), Relu(10), Relu(10), Relu(10), reduce_max(), Softmax() 13 | ) 14 | reveal_type(bad_model_only_plugin2) 15 | 16 | bad_model_only_plugin3 = concatenate( 17 | Relu(10), Relu(10), Relu(10), Relu(10), reduce_max(), Softmax() 18 | ) 19 | 20 | reveal_type(bad_model_only_plugin3) 21 | -------------------------------------------------------------------------------- /thinc/tests/mypy/modules/success_no_plugin.py: -------------------------------------------------------------------------------- 1 | from thinc.api import Relu, Softmax, add, chain, reduce_max 2 | 3 | good_model = chain(Relu(10), Relu(10), Softmax()) 4 | reveal_type(good_model) 5 | 6 | good_model2 = add(Relu(10), Relu(10), Softmax()) 7 | reveal_type(good_model2) 8 | 9 | bad_model_undetected = chain(Relu(10), Relu(10), reduce_max(), Softmax()) 10 | reveal_type(bad_model_undetected) 11 | 12 | bad_model_undetected2 = add(Relu(10), Relu(10), reduce_max(), Softmax()) 13 | reveal_type(bad_model_undetected2) 14 | -------------------------------------------------------------------------------- /thinc/tests/mypy/modules/success_plugin.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TypeVar 2 | 3 | from thinc.api import Model, Relu, Softmax, add, chain, reduce_max 4 | 5 | good_model = chain(Relu(10), Relu(10), Softmax()) 6 | reveal_type(good_model) 7 | 8 | good_model2 = add(Relu(10), Relu(10), Softmax()) 9 | reveal_type(good_model2) 10 | 11 | bad_model_undetected = chain(Relu(10), Relu(10), Relu(10), Relu(10), Softmax()) 12 | reveal_type(bad_model_undetected) 13 | 14 | bad_model_undetected2 = add(Relu(10), Relu(10), Relu(10), Relu(10), Softmax()) 15 | reveal_type(bad_model_undetected2) 16 | 17 | 18 | def forward() -> None: 19 | pass 20 | 21 | 22 | OtherType = TypeVar("OtherType") 23 | 24 | 25 | def other_function( 26 | layer1: Model, layer2: Model, *layers: Model 27 | ) -> Model[Any, OtherType]: 28 | return Model("some_model", forward) 29 | 30 | 31 | non_combinator_model = other_function( 32 | Model("x", forward), Model("y", forward), Model("z", forward) 33 | ) 34 | reveal_type(non_combinator_model) 35 | -------------------------------------------------------------------------------- /thinc/tests/mypy/outputs/fail-no-plugin.txt: -------------------------------------------------------------------------------- 1 | 3: error: Cannot infer type argument 2 of "chain" [misc] 2 | 5: error: Cannot infer type argument 1 of "add" [misc] -------------------------------------------------------------------------------- /thinc/tests/mypy/outputs/fail-plugin.txt: -------------------------------------------------------------------------------- 1 | 3: error: Cannot infer type argument 2 of "chain" [misc] 2 | 3: error: Layer outputs type (thinc.types.Floats2d) but the next layer expects (thinc.types.Ragged) as an input [layer-mismatch-output] 3 | 3: error: Layer input type (thinc.types.Ragged) is not compatible with output (thinc.types.Floats2d) from previous layer [layer-mismatch-input] 4 | 5: error: Cannot infer type argument 1 of "add" [misc] 5 | 5: error: Layer input (thinc.types.Floats2d) not compatible with next layer input (thinc.types.Ragged) [layer-mismatch-input] 6 | 5: error: Layer input (thinc.types.Ragged) not compatible with previous layer input (thinc.types.Floats2d) [layer-mismatch-input] 7 | 5: error: Layer input (thinc.types.Ragged) not compatible with next layer input (thinc.types.Floats2d) [layer-mismatch-input] 8 | 5: error: Layer input (thinc.types.Floats2d) not compatible with previous layer input (thinc.types.Ragged) [layer-mismatch-input] 9 | 8: error: Layer outputs type (thinc.types.Floats2d) but the next layer expects (thinc.types.Ragged) as an input [layer-mismatch-output] 10 | 8: error: Layer input type (thinc.types.Ragged) is not compatible with output (thinc.types.Floats2d) from previous layer [layer-mismatch-input] 11 | 12: error: Layer input (thinc.types.Floats2d) not compatible with next layer input (thinc.types.Ragged) [layer-mismatch-input] 12 | 12: error: Layer input (thinc.types.Ragged) not compatible with previous layer input (thinc.types.Floats2d) [layer-mismatch-input] 13 | 12: error: Layer input (thinc.types.Ragged) not compatible with next layer input (thinc.types.Floats2d) [layer-mismatch-input] 14 | 12: error: Layer input (thinc.types.Floats2d) not compatible with previous layer input (thinc.types.Ragged) [layer-mismatch-input] 15 | 14: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" 16 | 17: error: Layer input (thinc.types.Floats2d) not compatible with next layer input (thinc.types.Ragged) [layer-mismatch-input] 17 | 17: error: Layer input (thinc.types.Ragged) not compatible with previous layer input (thinc.types.Floats2d) [layer-mismatch-input] 18 | 17: error: Layer input (thinc.types.Ragged) not compatible with next layer input (thinc.types.Floats2d) [layer-mismatch-input] 19 | 17: error: Layer input (thinc.types.Floats2d) not compatible with previous layer input (thinc.types.Ragged) [layer-mismatch-input] 20 | 20: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" 21 | -------------------------------------------------------------------------------- /thinc/tests/mypy/outputs/success-no-plugin.txt: -------------------------------------------------------------------------------- 1 | 3: error: Need type annotation for "good_model" [var-annotated] 2 | 4: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" 3 | 6: error: Need type annotation for "good_model2" [var-annotated] 4 | 7: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" 5 | 9: error: Need type annotation for "bad_model_undetected" [var-annotated] 6 | 10: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" 7 | 12: error: Need type annotation for "bad_model_undetected2" [var-annotated] 8 | 13: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" -------------------------------------------------------------------------------- /thinc/tests/mypy/outputs/success-plugin.txt: -------------------------------------------------------------------------------- 1 | 6: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" 2 | 9: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" 3 | 12: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" 4 | 15: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" 5 | 31: error: Need type annotation for "non_combinator_model" [var-annotated] 6 | 34: note: Revealed type is "thinc.model.Model[Any, Any]" -------------------------------------------------------------------------------- /thinc/tests/regression/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/regression/__init__.py -------------------------------------------------------------------------------- /thinc/tests/regression/issue519/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/regression/issue519/__init__.py -------------------------------------------------------------------------------- /thinc/tests/regression/issue519/program.py: -------------------------------------------------------------------------------- 1 | from thinc.api import Relu, Softmax, chain, concatenate 2 | from thinc.model import Model 3 | from thinc.types import Floats2d 4 | 5 | n_hidden = 32 6 | dropout = 0.2 7 | 8 | model1: Model[Floats2d, Floats2d] = chain( 9 | Relu(nO=n_hidden, dropout=dropout), Relu(nO=n_hidden, dropout=dropout), Softmax() 10 | ) 11 | 12 | model2: Model[Floats2d, Floats2d] = chain( 13 | Relu(nO=n_hidden, dropout=dropout), Relu(nO=n_hidden, dropout=dropout), Softmax() 14 | ) 15 | 16 | model3: Model[Floats2d, Floats2d] = concatenate(*[model1, model2]) 17 | -------------------------------------------------------------------------------- /thinc/tests/regression/issue519/test_issue519.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | try: 5 | import importlib.resources as importlib_resources 6 | except ImportError: 7 | import importlib_resources # type: ignore 8 | 9 | 10 | import pytest 11 | 12 | 13 | @pytest.mark.slow 14 | def test_issue519(): 15 | """ 16 | Test ability of Thinc mypy plugin to handle variadic arguments. 17 | 18 | This test can take up to 45 seconds, and is thus marked as slow. 19 | """ 20 | # Determine the name of the parent module (which contains the test program) 21 | parent_module_name = __name__[: __name__.rfind(".")] 22 | 23 | # Load test program that calls a Thinc API with variadic arguments 24 | program_text = importlib_resources.read_text(parent_module_name, "program.py") 25 | 26 | # Ask Mypy to type-check the loaded program text 27 | subprocess.run( 28 | [sys.executable, "-m", "mypy", "--command", program_text], check=True 29 | ) 30 | -------------------------------------------------------------------------------- /thinc/tests/regression/test_issue208.py: -------------------------------------------------------------------------------- 1 | from thinc.api import Linear, chain 2 | 3 | 4 | def test_issue208(): 5 | """Test issue that was caused by trying to flatten nested chains.""" 6 | layer1 = Linear(nO=9, nI=3) 7 | layer2 = Linear(nO=12, nI=9) 8 | layer3 = Linear(nO=5, nI=12) 9 | model = chain(layer1, chain(layer2, layer3)).initialize() 10 | assert model.get_dim("nO") == 5 11 | -------------------------------------------------------------------------------- /thinc/tests/regression/test_issue564.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from thinc.api import CupyOps 4 | from thinc.compat import has_torch, has_torch_cuda_gpu 5 | 6 | 7 | @pytest.mark.skipif(not has_torch, reason="needs PyTorch") 8 | @pytest.mark.skipif(not has_torch_cuda_gpu, reason="needs a GPU") 9 | def test_issue564(): 10 | import torch 11 | 12 | if CupyOps.xp is not None: 13 | ops = CupyOps() 14 | t = torch.zeros((10, 2)).cuda() 15 | a = ops.asarray(t) 16 | 17 | assert a.shape == t.shape 18 | ops.xp.testing.assert_allclose( 19 | a, 20 | ops.alloc2f(10, 2), 21 | ) 22 | -------------------------------------------------------------------------------- /thinc/tests/shims/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/thinc/tests/shims/__init__.py -------------------------------------------------------------------------------- /thinc/tests/test_examples.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | 7 | @pytest.fixture 8 | def test_files(nb_file): 9 | pytest.importorskip("nbconvert", exc_type=ImportError) 10 | pytest.importorskip("nbformat") 11 | import nbconvert 12 | import nbformat 13 | from nbconvert.preprocessors import ExecutePreprocessor 14 | 15 | if not Path(nb_file).exists(): 16 | return 17 | kernel_name = os.environ.get("NOTEBOOK_KERNEL", "python3") 18 | with open(nb_file) as f: 19 | nb = nbformat.read(f, as_version=4) 20 | proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) 21 | proc.allow_errors = True 22 | proc.preprocess(nb, {"metadata": {"path": "/"}}) 23 | cells_with_outputs = [c for c in nb.cells if "outputs" in c] 24 | for cell in cells_with_outputs: 25 | for output in cell["outputs"]: 26 | if output.output_type == "error": 27 | for l in output.traceback: 28 | print(l) 29 | raise Exception(f"{output.ename}: {output.evalue}") 30 | 31 | 32 | @pytest.mark.parametrize( 33 | "nb_file", 34 | ( 35 | "examples/01_intro_model_definition_methods.ipynb", 36 | "examples/05_benchmarking_layers.ipynb", 37 | ), 38 | ) 39 | def test_ipython_notebooks(test_files: None): 40 | ... 41 | 42 | 43 | @pytest.mark.skip(reason="these notebooks need special software or hardware") 44 | @pytest.mark.parametrize( 45 | "nb_file", 46 | ( 47 | "examples/00_intro_to_thinc.ipynb", 48 | "examples/02_transformers_tagger_bert.ipynb", 49 | "examples/03_pos_tagger_basic_cnn.ipynb", 50 | "examples/03_textcat_basic_neural_bow.ipynb", 51 | "examples/04_configure_gpu_memory.ipynb", 52 | "examples/04_parallel_training_ray.ipynb", 53 | "examples/05_visualizing_models.ipynb", 54 | "examples/06_predicting_like_terms.ipynb", 55 | ), 56 | ) 57 | def test_ipython_notebooks_slow(test_files: None): 58 | ... 59 | -------------------------------------------------------------------------------- /thinc/tests/test_import__all__.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import importlib 3 | from collections import namedtuple 4 | from typing import List, Tuple 5 | 6 | import pytest 7 | 8 | _Import = namedtuple("_Import", ["module", "name", "alias"]) 9 | 10 | 11 | def get_imports(path: str) -> Tuple[_Import, ...]: 12 | """Parse Python file at path, retrieve import statements. 13 | Adapted from https://stackoverflow.com/a/9049549. 14 | path (str): Path to Python file. 15 | RETURNS (Tuple[_Import]): All imports found in file at path. 16 | """ 17 | with open(path) as fh: 18 | root = ast.parse(fh.read(), path) 19 | 20 | imports: List[_Import] = [] 21 | for node in ast.walk(root): 22 | if isinstance(node, ast.Import): 23 | module: List[str] = [] 24 | elif isinstance(node, ast.ImportFrom) and node.module: 25 | module = node.module.split(".") 26 | else: 27 | continue 28 | 29 | assert isinstance(node, (ast.Import, ast.ImportFrom)) 30 | imports.extend( 31 | [_Import(module, n.name.split("."), n.asname) for n in node.names] 32 | ) 33 | 34 | return tuple(imports) 35 | 36 | 37 | @pytest.mark.parametrize("module_name", ["thinc.api", "thinc.shims", "thinc.layers"]) 38 | def test_import_reexport_equivalency(module_name: str): 39 | """Tests whether a module's __all__ is equivalent to its imports. This assumes that this module is supposed to 40 | re-export all imported values. 41 | module_name (str): Module to load. 42 | """ 43 | mod = importlib.import_module(module_name) 44 | 45 | assert set(mod.__all__) == { 46 | k 47 | for k in set(n for i in get_imports(str(mod.__file__)) for n in i.name) 48 | if ( 49 | # Ignore all values prefixed with _, as we expect those not to be re-exported. 50 | # However, __version__ should be reexported in thinc/__init__.py. 51 | (not k.startswith("_") or module_name == "thinc" and k == "__version__") 52 | ) 53 | } 54 | -------------------------------------------------------------------------------- /thinc/tests/test_indexing.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | from numpy.testing import assert_allclose 4 | 5 | from thinc.types import Pairs, Ragged 6 | 7 | 8 | @pytest.fixture 9 | def ragged(): 10 | data = numpy.zeros((20, 4), dtype="f") 11 | lengths = numpy.array([4, 2, 8, 1, 4], dtype="i") 12 | data[0] = 0 13 | data[1] = 1 14 | data[2] = 2 15 | data[3] = 3 16 | data[4] = 4 17 | data[5] = 5 18 | return Ragged(data, lengths) 19 | 20 | 21 | def test_ragged_empty(): 22 | data = numpy.zeros((0, 4), dtype="f") 23 | lengths = numpy.array([], dtype="i") 24 | ragged = Ragged(data, lengths) 25 | assert_allclose(ragged[0:0].data, ragged.data) 26 | assert_allclose(ragged[0:0].lengths, ragged.lengths) 27 | assert_allclose(ragged[0:2].data, ragged.data) 28 | assert_allclose(ragged[0:2].lengths, ragged.lengths) 29 | assert_allclose(ragged[1:2].data, ragged.data) 30 | assert_allclose(ragged[1:2].lengths, ragged.lengths) 31 | 32 | 33 | def test_ragged_starts_ends(ragged): 34 | starts = ragged._get_starts() 35 | ends = ragged._get_ends() 36 | assert list(starts) == [0, 4, 6, 14, 15] 37 | assert list(ends) == [4, 6, 14, 15, 19] 38 | 39 | 40 | def test_ragged_simple_index(ragged, i=1): 41 | r = ragged[i] 42 | assert_allclose(r.data, ragged.data[4:6]) 43 | assert_allclose(r.lengths, ragged.lengths[i : i + 1]) 44 | 45 | 46 | def test_ragged_slice_index(ragged, start=0, end=2): 47 | r = ragged[start:end] 48 | size = ragged.lengths[start:end].sum() 49 | assert r.data.shape == (size, r.data.shape[1]) 50 | assert_allclose(r.lengths, ragged.lengths[start:end]) 51 | 52 | 53 | def test_ragged_array_index(ragged): 54 | arr = numpy.array([2, 1, 4], dtype="i") 55 | r = ragged[arr] 56 | assert r.data.shape[0] == ragged.lengths[arr].sum() 57 | 58 | 59 | def test_pairs_arrays(): 60 | one = numpy.zeros((128, 45), dtype="f") 61 | two = numpy.zeros((128, 12), dtype="f") 62 | pairs = Pairs(one, two) 63 | assert pairs[:2].one.shape == (2, 45) 64 | assert pairs[0].two.shape == (12,) 65 | assert pairs[-1:].one.shape == (1, 45) 66 | assert pairs[-1:].two.shape == (1, 12) 67 | -------------------------------------------------------------------------------- /thinc/tests/test_initializers.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | 4 | from thinc import registry 5 | from thinc.api import ( 6 | NumpyOps, 7 | glorot_uniform_init, 8 | normal_init, 9 | uniform_init, 10 | zero_init, 11 | ) 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "init_func", [glorot_uniform_init, zero_init, uniform_init, normal_init] 16 | ) 17 | def test_initializer_func_setup(init_func): 18 | ops = NumpyOps() 19 | data = numpy.ndarray([1, 2, 3, 4], dtype="f") 20 | result = init_func(ops, data.shape) 21 | assert not numpy.array_equal(data, result) 22 | 23 | 24 | @pytest.mark.parametrize( 25 | "name,kwargs", 26 | [ 27 | ("glorot_uniform_init.v1", {}), 28 | ("zero_init.v1", {}), 29 | ("uniform_init.v1", {"lo": -0.5, "hi": 0.5}), 30 | ("normal_init.v1", {"mean": 0.1}), 31 | ], 32 | ) 33 | def test_initializer_from_config(name, kwargs): 34 | """Test that initializers are loaded and configured correctly from registry 35 | (as partials).""" 36 | cfg = {"test": {"@initializers": name, **kwargs}} 37 | func = registry.resolve(cfg)["test"] 38 | func(NumpyOps(), (1, 2, 3, 4)) 39 | -------------------------------------------------------------------------------- /thinc/tests/test_types.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import pytest 3 | 4 | from thinc.types import ( 5 | Floats1d, 6 | Floats2d, 7 | Floats3d, 8 | Floats4d, 9 | Ints1d, 10 | Ints2d, 11 | Ints3d, 12 | Ints4d, 13 | ) 14 | 15 | try: 16 | from pydantic.v1 import ValidationError, create_model 17 | except ImportError: 18 | from pydantic import ValidationError, create_model # type: ignore 19 | 20 | 21 | from thinc.types import ( 22 | Floats1d, 23 | Floats2d, 24 | Floats3d, 25 | Floats4d, 26 | Ints1d, 27 | Ints2d, 28 | Ints3d, 29 | Ints4d, 30 | ) 31 | 32 | 33 | @pytest.mark.parametrize( 34 | "arr,arr_type", 35 | [ 36 | (numpy.zeros(0, dtype=numpy.float32), Floats1d), 37 | (numpy.zeros((0, 0), dtype=numpy.float32), Floats2d), 38 | (numpy.zeros((0, 0, 0), dtype=numpy.float32), Floats3d), 39 | (numpy.zeros((0, 0, 0, 0), dtype=numpy.float32), Floats4d), 40 | (numpy.zeros(0, dtype=numpy.int32), Ints1d), 41 | (numpy.zeros((0, 0), dtype=numpy.int32), Ints2d), 42 | (numpy.zeros((0, 0, 0), dtype=numpy.int32), Ints3d), 43 | (numpy.zeros((0, 0, 0, 0), dtype=numpy.int32), Ints4d), 44 | ], 45 | ) 46 | def test_array_validation_valid(arr, arr_type): 47 | test_model = create_model("TestModel", arr=(arr_type, ...)) 48 | result = test_model(arr=arr) 49 | assert numpy.array_equal(arr, result.arr) 50 | 51 | 52 | @pytest.mark.parametrize( 53 | "arr,arr_type", 54 | [ 55 | (numpy.zeros((0, 0), dtype=numpy.float32), Floats1d), 56 | (numpy.zeros((0, 0), dtype=numpy.float32), Ints2d), 57 | ], 58 | ) 59 | def test_array_validation_invalid(arr, arr_type): 60 | test_model = create_model("TestModel", arr=(arr_type, ...)) 61 | with pytest.raises(ValidationError): 62 | test_model(arr=arr) 63 | -------------------------------------------------------------------------------- /website/.dockerignore: -------------------------------------------------------------------------------- 1 | # Avoid uploading large Docker contexts 2 | .cache/ 3 | public/ 4 | node_modules 5 | .npm 6 | logs 7 | *.log 8 | npm-debug.log* 9 | www/ 10 | _deploy.sh 11 | *.html 12 | -------------------------------------------------------------------------------- /website/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["standard", "prettier"], 3 | "plugins": ["standard", "react", "react-hooks"], 4 | "rules": { 5 | "no-var": "error", 6 | "no-unused-vars": 1, 7 | "arrow-spacing": ["error", { "before": true, "after": true }], 8 | "indent": ["error", 4], 9 | "semi": ["error", "never"], 10 | "arrow-parens": ["error", "as-needed"], 11 | "standard/object-curly-even-spacing": ["error", "either"], 12 | "standard/array-bracket-even-spacing": ["error", "either"], 13 | "standard/computed-property-even-spacing": ["error", "even"], 14 | "standard/no-callback-literal": ["error", ["cb", "callback"]], 15 | "react/jsx-uses-react": "error", 16 | "react/jsx-uses-vars": "error", 17 | "react-hooks/rules-of-hooks": "error", 18 | "react-hooks/exhaustive-deps": "warn", 19 | "import/no-duplicates": "off", 20 | "new-cap": "off" 21 | }, 22 | "parser": "babel-eslint", 23 | "parserOptions": { 24 | "ecmaVersion": 8 25 | }, 26 | "env": { 27 | "browser": true 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .cache/ 3 | public/ 4 | node_modules/ 5 | .npm 6 | logs 7 | *.log 8 | npm-debug.log* 9 | .idea/ 10 | src/fonts/ 11 | -------------------------------------------------------------------------------- /website/.nvmrc: -------------------------------------------------------------------------------- 1 | 16 -------------------------------------------------------------------------------- /website/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "es5", 5 | "tabWidth": 4, 6 | "bracketSpacing": true, 7 | "printWidth": 100, 8 | "overrides": [ 9 | { 10 | "files": "*.sass", 11 | "options": { 12 | "printWidth": 999 13 | } 14 | }, 15 | { 16 | "files": "*.md", 17 | "options": { 18 | "tabWidth": 2, 19 | "printWidth": 80, 20 | "proseWrap": "always", 21 | "htmlWhitespaceSensitivity": "strict" 22 | } 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /website/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16 2 | 3 | USER node 4 | 5 | # This is so the installed node_modules will be up one directory 6 | # from where a user mounts files, so that they don't accidentally mount 7 | # their own node_modules from a different build 8 | # https://nodejs.org/api/modules.html#modules_loading_from_node_modules_folders 9 | WORKDIR /home/node 10 | COPY --chown=node package.json . 11 | COPY --chown=node package-lock.json . 12 | RUN npm install 13 | 14 | WORKDIR /home/node/website/ 15 | -------------------------------------------------------------------------------- /website/docs/_quickstart.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaults": { "cuda": "-" }, 3 | "options": [ 4 | { 5 | "label": "CUDA", 6 | "name": "cuda", 7 | "options": [ 8 | { "label": "none", "value": "-" }, 9 | { "label": "8.0", "value": "cuda80" }, 10 | { "label": "9.0", "value": "cuda90" }, 11 | { "label": "9.1", "value": "cuda91" }, 12 | { "label": "9.2", "value": "cuda92" }, 13 | { "label": "10.0", "value": "cuda100" }, 14 | { "label": "10.1", "value": "cuda101" }, 15 | { "label": "10.2", "value": "cuda102" }, 16 | { "label": "11.0", "value": "cuda110" }, 17 | { "label": "11.1", "value": "cuda111" }, 18 | { "label": "11.2-11.x", "value": "cuda11x" }, 19 | { "label": "12.x", "value": "cuda12x" } 20 | ] 21 | }, 22 | { 23 | "label": "Libraries", 24 | "name": "libraries", 25 | "multi": true, 26 | "options": [ 27 | { "label": "PyTorch", "value": "torch" }, 28 | { "label": "TensorFlow", "value": "tensorflow" }, 29 | { "label": "MXNet", "value": "mxnet" } 30 | ] 31 | } 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /website/docs/_type_links.json: -------------------------------------------------------------------------------- 1 | { 2 | "__default__": "/docs/api-types#types", 3 | "Model": "/docs/api-model#model", 4 | "Shim": "/docs/api-model#shim", 5 | "Ops": "/docs/api-backends#ops", 6 | "NumpyOps": "/docs/api-backends#ops", 7 | "CupyOps": "/docs/api-backends#ops", 8 | "Config": "/docs/api-config#config", 9 | "Ragged": "/docs/api-types#ragged", 10 | "Padded": "/docs/api-types#padded", 11 | "Pairs": "/docs/api-types#pairs", 12 | "ArgsKwargs": "/docs/api-types#argskwargs", 13 | "SizedGenerator": "/docs/api-types#sizedgenerator", 14 | "Array1d": true, 15 | "Array2d": true, 16 | "Array3d": true, 17 | "Array4d": true, 18 | "ArrayXd": true, 19 | "Floats1d": true, 20 | "Floats2d": true, 21 | "Floats3d": true, 22 | "Floats4d": true, 23 | "FloatsXd": true, 24 | "Ints1d": true, 25 | "Ints2d": true, 26 | "Ints3d": true, 27 | "Ints4d": true, 28 | "IntsXd": true, 29 | "List1d": true, 30 | "List2d": true, 31 | "List3d": true, 32 | "List4d": true, 33 | "ListXd": true, 34 | "Generator": true, 35 | "Shape": true, 36 | "DTypes": true, 37 | "DTypesInt": true, 38 | "DTypesFloat": true, 39 | "Xp": true, 40 | "Batchable": true, 41 | "BaseModel": "https://pydantic-docs.helpmanual.io/usage/models/", 42 | "Doc": "https://spacy.io/api/doc", 43 | "Device": "https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.cuda.Device.html" 44 | } 45 | -------------------------------------------------------------------------------- /website/docs/images/layer-traversal.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/website/docs/images/layer-traversal.graffle -------------------------------------------------------------------------------- /website/docs/images/type_checking.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/website/docs/images/type_checking.jpg -------------------------------------------------------------------------------- /website/docs/images/type_checking2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/explosion/thinc/20eec35566997d5272d083be1f1a0ec9006934ff/website/docs/images/type_checking2.jpg -------------------------------------------------------------------------------- /website/gatsby-browser.js: -------------------------------------------------------------------------------- 1 | exports.onRouteUpdate = ({ location }) => { 2 | // Fix anchor links, especially if links are opened in new tab 3 | if (location.hash) { 4 | setTimeout(() => { 5 | const el = document.querySelector(`${location.hash}`) 6 | if (el) { 7 | // Navigate to targeted element 8 | el.scrollIntoView() 9 | // Force recomputing :target pseudo class with pushState/popState 10 | window.location.hash = location.hash 11 | } 12 | }, 0) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /website/gatsby-node.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const { createFilePath } = require('gatsby-source-filesystem') 3 | 4 | const pageTemplate = path.resolve('src/templates/docs.js') 5 | 6 | function replacePath(pagePath) { 7 | return pagePath === `/` ? pagePath : pagePath.replace(/\/$/, ``) 8 | } 9 | 10 | exports.onCreatePage = ({ page, actions }) => { 11 | const { createPage, deletePage } = actions 12 | const oldPage = Object.assign({}, page) 13 | if (oldPage.path != '/dev-404-page/') { 14 | page.path = replacePath(page.path) 15 | if (page.path !== oldPage.path) { 16 | deletePage(oldPage) 17 | createPage(page) 18 | } 19 | } 20 | } 21 | 22 | exports.onCreateNode = ({ node, actions, getNode }) => { 23 | const { createNodeField } = actions 24 | if (node.internal.type === 'MarkdownRemark') { 25 | const slug = createFilePath({ node, getNode, basePath: 'docs', trailingSlash: false }) 26 | createNodeField({ name: 'slug', node, value: `/docs${slug}` }) 27 | } 28 | } 29 | 30 | exports.createPages = ({ actions, graphql }) => { 31 | const { createPage } = actions 32 | return graphql(` 33 | { 34 | allMarkdownRemark { 35 | edges { 36 | node { 37 | frontmatter { 38 | title 39 | } 40 | fields { 41 | slug 42 | } 43 | } 44 | } 45 | } 46 | } 47 | `).then(result => { 48 | if (result.errors) { 49 | return Promise.reject(result.errors) 50 | } 51 | const posts = result.data.allMarkdownRemark.edges 52 | posts.forEach(({ node }) => { 53 | createPage({ 54 | path: replacePath(node.fields.slug), 55 | component: pageTemplate, 56 | context: { 57 | slug: node.fields.slug, 58 | }, 59 | }) 60 | }) 61 | }) 62 | } 63 | 64 | exports.onCreateWebpackConfig = ({ stage, actions, getConfig }) => { 65 | const config = getConfig() 66 | const miniCssExtractPlugin = config.plugins.find( 67 | plugin => plugin.constructor.name === 'MiniCssExtractPlugin' 68 | ) 69 | 70 | if (miniCssExtractPlugin) { 71 | miniCssExtractPlugin.options.ignoreOrder = true 72 | } 73 | actions.replaceWebpackConfig(config) 74 | } 75 | -------------------------------------------------------------------------------- /website/plugins/gatsby-remark-code-blocks/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-remark-code-blocks", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC" 11 | } 12 | -------------------------------------------------------------------------------- /website/plugins/gatsby-remark-custom-attrs/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Simplified implementation of remark-attr that allows custom attributes on 3 | * nodes *inline* via the following syntax: {#some-id key="value"}. Extracting 4 | * them inline is slightly hackier (at least in this implementation), but it 5 | * makes the resulting markup valid and compatible with formatters like 6 | * Prettier, which do not allow additional content right below headlines. 7 | * Based on: https://github.com/arobase-che/remark-attr 8 | */ 9 | 10 | const visit = require('unist-util-visit') 11 | const parseAttr = require('md-attr-parser') 12 | 13 | const defaultOptions = { 14 | elements: ['heading'], 15 | } 16 | 17 | module.exports = ({ markdownAST }, userOptions = {}) => { 18 | const options = Object.assign({}, defaultOptions, userOptions) 19 | visit(markdownAST, null, node => { 20 | if (options.elements.includes(node.type)) { 21 | if ( 22 | node.children && 23 | node.children.length && 24 | node.children[node.children.length - 1].type === 'text' && 25 | node.children[node.children.length - 1].value 26 | ) { 27 | if (node.children.length > 1 && node.children.every(el => el.type === 'text')) { 28 | // If headlines contain escaped characters, e.g. 29 | // Doc.\_\_init\_\_, it will be split into several nodes 30 | const mergedText = node.children.map(el => el.value).join('') 31 | node.children[0].value = mergedText 32 | node.children = [node.children[0]] 33 | } 34 | const lastIdx = node.children.length - 1 35 | const parsed = node.children[lastIdx].value.split(/\{(.*?)\}/) 36 | if (parsed.length >= 2 && parsed[1]) { 37 | const text = parsed[0] 38 | const { prop } = parseAttr(parsed[1]) 39 | const data = node.data || (node.data = {}) 40 | const hProps = data.hProperties || (data.hProperties = {}) 41 | node.data.hProperties = Object.assign({}, hProps, prop) 42 | node.children[lastIdx].value = text 43 | } 44 | } 45 | } 46 | }) 47 | return markdownAST 48 | } 49 | -------------------------------------------------------------------------------- /website/plugins/gatsby-remark-custom-attrs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-remark-custom-attrs", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC" 11 | } 12 | -------------------------------------------------------------------------------- /website/plugins/gatsby-remark-unwrap/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Unwrap elements and remove outer

3 | * Fork of: https://github.com/xuopled/gatsby-remark-unwrap-images 4 | */ 5 | const visit = require('unist-util-visit') 6 | const remove = require('unist-util-remove') 7 | 8 | const defaultOptions = { 9 | elements: ['image', 'html'], 10 | } 11 | 12 | function isWrapped(child, elements = []) { 13 | return elements.includes(child.type) || (child.type === 'text' && child.value === '\n') 14 | } 15 | 16 | module.exports = ({ markdownAST }, userOptions = {}) => { 17 | const options = Object.assign({}, defaultOptions, userOptions) 18 | const elements = options.elements || [] 19 | visit(markdownAST, 'paragraph', (node, index, parent) => { 20 | const wrapped = node.children.every(child => isWrapped(child, elements)) 21 | if (!wrapped) return 22 | remove(node, 'text') 23 | parent.children.splice(index, 1, ...node.children) 24 | return index 25 | }) 26 | return markdownAST 27 | } 28 | -------------------------------------------------------------------------------- /website/plugins/gatsby-remark-unwrap/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gatsby-remark-unwrap", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC" 11 | } 12 | -------------------------------------------------------------------------------- /website/src/components/box.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import classNames from 'classnames' 3 | 4 | import * as classes from '../styles/box.module.sass' 5 | 6 | export const Box = ({ Component = 'section', id, className, children }) => ( 7 | 8 | {children} 9 | 10 | ) 11 | 12 | export const Infobox = ({ variant, children }) => { 13 | const infoboxClassNames = classNames(classes.infobox, { 14 | [classes.warning]: variant === 'warning', 15 | [classes.danger]: variant === 'danger', 16 | }) 17 | return ( 18 | 19 | {variant ? '!' : 'i'} 20 | {children} 21 | 22 | ) 23 | } 24 | -------------------------------------------------------------------------------- /website/src/components/dropdown.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import classNames from 'classnames' 3 | import { navigate } from 'gatsby' 4 | 5 | import * as classes from '../styles/dropdown.module.sass' 6 | 7 | const Dropdown = ({ defaultValue, className, onChange, children }) => { 8 | const defaultOnChange = ({ target }) => { 9 | const isExternal = /((http(s?)):\/\/|mailto:)/gi.test(target.value) 10 | if (isExternal) { 11 | window.location.href = target.value 12 | } else { 13 | navigate(target.value) 14 | } 15 | } 16 | return ( 17 | 24 | ) 25 | } 26 | 27 | export default Dropdown 28 | -------------------------------------------------------------------------------- /website/src/components/footer.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { graphql, StaticQuery } from 'gatsby' 3 | import classNames from 'classnames' 4 | 5 | import Link from './link' 6 | import Icon from './icon' 7 | import * as classes from '../styles/footer.module.sass' 8 | 9 | export default ({ className }) => ( 10 | { 13 | const { twitter, github, company, companyUrl, imprintUrl } = site.siteMetadata 14 | return ( 15 | 41 | ) 42 | }} 43 | /> 44 | ) 45 | 46 | const query = graphql` 47 | query { 48 | site { 49 | siteMetadata { 50 | company 51 | companyUrl 52 | imprintUrl 53 | email 54 | twitter 55 | github 56 | } 57 | } 58 | } 59 | ` 60 | -------------------------------------------------------------------------------- /website/src/components/grid.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import classNames from 'classnames' 3 | 4 | import * as classes from '../styles/grid.module.sass' 5 | 6 | export default ({ layout = 'auto', children }) => { 7 | const gridClassNames = classNames(classes.root, { 8 | [classes.feature]: layout === 'feature', 9 | }) 10 | return
{children}
11 | } 12 | -------------------------------------------------------------------------------- /website/src/components/icon.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import classNames from 'classnames' 3 | 4 | import * as classes from '../styles/icon.module.sass' 5 | 6 | import TwitterIcon from '../images/icons/twitter.svg' 7 | import GitHubIcon from '../images/icons/github.svg' 8 | import ArrowRightIcon from '../images/icons/arrow-right.svg' 9 | import YesIcon from '../images/icons/yes.svg' 10 | import NoIcon from '../images/icons/no.svg' 11 | import CubeIcon from '../images/icons/cube.svg' 12 | import FileIcon from '../images/icons/file.svg' 13 | 14 | const ICONS = { 15 | twitter: TwitterIcon, 16 | github: GitHubIcon, 17 | right: ArrowRightIcon, 18 | yes: YesIcon, 19 | no: NoIcon, 20 | cube: CubeIcon, 21 | file: FileIcon, 22 | } 23 | 24 | export default ({ name, size = 16, alt = null, className, ...props }) => { 25 | const SvgIcon = ICONS[name] 26 | if (!SvgIcon) throw Error(`Invalid icon name: '${name}'`) 27 | const style = { minWidth: size } 28 | const iconClassNames = classNames(classes.root, className, { 29 | [classes.red]: name === 'no', 30 | [classes.green]: name === 'yes', 31 | }) 32 | const altTexts = { yes: 'yes', no: 'no' } 33 | return ( 34 | 42 | ) 43 | } 44 | 45 | export const Emoji = ({ alt, children }) => { 46 | const attrs = alt ? { role: 'img', 'aria-label': alt } : { 'aria-hidden': true } 47 | return ( 48 | 49 | {children} 50 | 51 | ) 52 | } 53 | -------------------------------------------------------------------------------- /website/src/components/layout.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | 3 | import SEO from './seo' 4 | 5 | const Layout = ({ title, description, className, children }) => ( 6 | <> 7 | 8 |
{children}
9 | 10 | ) 11 | 12 | export default Layout 13 | -------------------------------------------------------------------------------- /website/src/components/link.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { Link as GatsbyLink } from 'gatsby' 3 | import classNames from 'classnames' 4 | 5 | import Icon from './icon' 6 | import * as classes from '../styles/link.module.sass' 7 | 8 | const internalRegex = /(http(s?)):\/\/(explosion.ai|prodi.gy|spacy.io|irl.spacy.io|support.prodi.gy)/gi 9 | 10 | const Link = ({ type, ...props }) => { 11 | // This is a workaround for the gatsby-remark-copy-linked-files, which 12 | // only recognizes elements, not a custom 30 | 31 |