├── .all_crates.sh ├── .change_crate_dep.sh ├── .github └── workflows │ ├── asan.yml │ ├── beta-and-nightly.yml │ ├── binaries.yml │ ├── cost_model.yml │ ├── cross-platform.yml │ ├── examples.yml │ ├── full.yml │ ├── llm.yml │ ├── metal.yml │ ├── pydoc.yml │ ├── release.yml │ ├── unix.yml │ ├── wheels.yml │ └── windows.yml ├── .gitignore ├── .travis.yml ├── .travis ├── README.md ├── android-ndk.sh ├── asan.sh ├── bundle-entrypoint.sh ├── cache_file.sh ├── cargo-deny-check.sh ├── ci-system-setup.sh ├── cli-tests.sh ├── cost_model_task_build.sh ├── cross.sh ├── debug-tests.sh ├── docker-debian-stretch │ ├── Dockerfile │ └── sources.list ├── examples.sh ├── make_bundle.sh ├── minion.sh ├── minionrc ├── native.sh ├── onnx-tests.sh ├── regular-tests.sh ├── run-bundle.sh ├── run_all.sh ├── setup-sccache.sh ├── test-harness.sh ├── test-llm.sh ├── test-published-crates.sh ├── test-rt.sh ├── tf.sh ├── tflite.sh ├── tflite │ ├── Dockerfile.tensorflow-aarch64 │ ├── Dockerfile.tensorflow-official-rpi │ ├── Dockerfile.tensorflow-rpitools │ ├── build_tflite_aarch64.sh │ ├── build_tflite_raspbian.sh │ ├── convert_all.sh │ ├── linux_makefile.inc │ └── run_all.sh └── travis.sh ├── .vim └── coc-settings.json ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── api ├── .gitignore ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── c │ ├── .gitignore │ ├── Makefile │ ├── grace_hopper_3_224_224.f32.raw │ └── mobilenet.c ├── ffi │ ├── Cargo.toml │ ├── cbindgen.toml │ └── src │ │ └── lib.rs ├── generate-tract-h.sh ├── proxy │ ├── Cargo.toml │ ├── LICENSE │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── ci.sh │ ├── src │ │ └── lib.rs │ ├── sys │ │ ├── Cargo.toml │ │ ├── build.rs │ │ ├── src │ │ │ └── lib.rs │ │ └── tract.h │ └── tests │ │ └── mobilenet.rs ├── py │ ├── .gitignore │ ├── MANIFEST.in │ ├── docs │ │ ├── fact.md │ │ ├── index.md │ │ ├── inference_model.md │ │ ├── model.md │ │ ├── nnef.md │ │ ├── onnx.md │ │ ├── runnable.md │ │ └── value.md │ ├── grace_hopper_1x3x224x244.npy │ ├── mkdocs.yml │ ├── pyproject.toml │ ├── requirements-docs.txt │ ├── requirements.txt │ ├── setup.py │ ├── tests │ │ └── mobilenet_onnx_test.py │ └── tract │ │ ├── __init__.py │ │ ├── bindings.py │ │ ├── fact.py │ │ ├── inference_model.py │ │ ├── model.py │ │ ├── nnef.py │ │ ├── onnx.py │ │ ├── runnable.py │ │ ├── state.py │ │ └── value.py ├── rs │ ├── Cargo.toml │ ├── LICENSE │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── src │ │ └── lib.rs │ └── tests │ │ └── mobilenet.rs ├── src │ ├── lib.rs │ └── macros.rs ├── tests │ ├── grace_hopper_3_224_224.f32.raw │ └── mobilenet │ │ └── mod.rs └── tract.h ├── assets └── tract-logo │ ├── PNG │ ├── PNG_body │ │ ├── tract-body-black.png │ │ ├── tract-body-blue.png │ │ └── tract-body-white.png │ ├── PNG_icon │ │ ├── tract-icon-black.png │ │ ├── tract-icon-blue.png │ │ └── tract-icon-white.png │ ├── PNG_logo │ │ ├── tract-horizontal-black.png │ │ ├── tract-horizontal-blue.png │ │ ├── tract-horizontal-white.png │ │ ├── tract-vertical-black.png │ │ ├── tract-vertical-blue.png │ │ └── tract-vertical-white.png │ ├── tract-body-black.png │ ├── tract-body-blue.png │ ├── tract-body-white.png │ ├── tract-horizontal-black.png │ ├── tract-horizontal-blue.png │ ├── tract-horizontal-white.png │ ├── tract-icon-black.png │ ├── tract-icon-blue.png │ ├── tract-icon-white.png │ ├── tract-vertical-black.png │ ├── tract-vertical-blue.png │ └── tract-vertical-white.png │ ├── SVG │ ├── @2xbody │ │ ├── tract-body-black.svg │ │ ├── tract-body-blue.svg │ │ └── tract-body-white.svg │ ├── @2xicon │ │ ├── tract-icon-black.svg │ │ ├── tract-icon-blue.svg │ │ └── tract-icon-white.svg │ ├── @2xlogo │ │ ├── tract-horizontal-black.svg │ │ ├── tract-horizontal-blue.svg │ │ ├── tract-horizontal-white.svg │ │ ├── tract-vertical-black.svg │ │ ├── tract-vertical-blue.svg │ │ └── tract-vertical-white.svg │ ├── tract-body-black.svg │ ├── tract-body-blue.svg │ ├── tract-body-white.svg │ ├── tract-horizontal-black.svg │ ├── tract-horizontal-blue.svg │ ├── tract-horizontal-white.svg │ ├── tract-icon-black.svg │ ├── tract-icon-blue.svg │ ├── tract-icon-white.svg │ ├── tract-vertical-black.svg │ ├── tract-vertical-blue.svg │ └── tract-vertical-white.svg │ └── post │ └── tract-horizontal-white copy.png ├── ci └── tract-ci-minion │ ├── Cargo.toml │ ├── minion.toml │ └── src │ └── main.rs ├── cli ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT └── src │ ├── bench.rs │ ├── compare.rs │ ├── cost.rs │ ├── dump.rs │ ├── llm.rs │ ├── macros.rs │ ├── main.rs │ ├── memory_arena.rs │ ├── model.rs │ ├── params.rs │ ├── plan_options.rs │ ├── run.rs │ ├── stream_check.rs │ ├── tensor.rs │ └── utils.rs ├── core ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── src │ ├── axes │ │ ├── mapping.rs │ │ ├── mod.rs │ │ └── model.rs │ ├── broadcast.rs │ ├── floats.rs │ ├── framework.rs │ ├── late_bind.rs │ ├── lib.rs │ ├── macros.rs │ ├── model │ │ ├── fact.rs │ │ ├── graph.rs │ │ ├── memory.rs │ │ ├── mod.rs │ │ ├── node.rs │ │ ├── order.rs │ │ ├── patch.rs │ │ ├── rewriter.rs │ │ ├── translator.rs │ │ └── typed.rs │ ├── ops │ │ ├── array │ │ │ ├── broadcast.rs │ │ │ ├── concat.rs │ │ │ ├── dyn_slice.rs │ │ │ ├── gather.rs │ │ │ ├── gather_elements.rs │ │ │ ├── gather_nd.rs │ │ │ ├── mod.rs │ │ │ ├── one_hot.rs │ │ │ ├── pad.rs │ │ │ ├── range.rs │ │ │ ├── reshape.rs │ │ │ ├── scatter_elements.rs │ │ │ ├── scatter_nd.rs │ │ │ ├── slice.rs │ │ │ ├── strided_slice.rs │ │ │ ├── tile.rs │ │ │ ├── topk.rs │ │ │ └── trilu.rs │ │ ├── binary.rs │ │ ├── cast.rs │ │ ├── change_axes.rs │ │ ├── cnn │ │ │ ├── conv │ │ │ │ ├── block_quant.rs │ │ │ │ ├── conv.rs │ │ │ │ ├── depth_wise.rs │ │ │ │ ├── im2col.rs │ │ │ │ ├── lazy_im2col.rs │ │ │ │ ├── mod.rs │ │ │ │ └── q_sum_b.rs │ │ │ ├── deconv │ │ │ │ ├── deconv.rs │ │ │ │ ├── deconv_sum.rs │ │ │ │ └── mod.rs │ │ │ ├── maxpool.rs │ │ │ ├── mod.rs │ │ │ ├── padding.rs │ │ │ ├── patch_axis.rs │ │ │ ├── patches.rs │ │ │ ├── pools.rs │ │ │ └── sumpool.rs │ │ ├── downsample │ │ │ ├── array.rs │ │ │ ├── conv.rs │ │ │ ├── mod.rs │ │ │ └── scan.rs │ │ ├── dummy.rs │ │ ├── einsum │ │ │ ├── as_blas.rs │ │ │ ├── einsum_matmul.rs │ │ │ ├── eval.rs │ │ │ ├── kernel_selection.rs │ │ │ ├── mod.rs │ │ │ ├── prefix_matmul.rs │ │ │ └── proptest.rs │ │ ├── element_wise.rs │ │ ├── fft.rs │ │ ├── identity.rs │ │ ├── konst.rs │ │ ├── logic.rs │ │ ├── logic │ │ │ ├── comparison.rs │ │ │ └── ite.rs │ │ ├── macros.rs │ │ ├── math │ │ │ ├── complex.rs │ │ │ └── mod.rs │ │ ├── matmul │ │ │ ├── de_block_quant.rs │ │ │ ├── mod.rs │ │ │ ├── optimized.rs │ │ │ ├── pack.rs │ │ │ └── quant.rs │ │ ├── memory │ │ │ ├── force_eval.rs │ │ │ ├── load.rs │ │ │ ├── mod.rs │ │ │ └── store.rs │ │ ├── mod.rs │ │ ├── nn │ │ │ ├── data_formats.rs │ │ │ ├── mod.rs │ │ │ ├── reduce.rs │ │ │ └── softmax │ │ │ │ ├── fixedpoint.rs │ │ │ │ ├── math.rs │ │ │ │ └── mod.rs │ │ ├── quant.rs │ │ ├── scan │ │ │ ├── decluttered.rs │ │ │ ├── mod.rs │ │ │ └── optimized.rs │ │ ├── source.rs │ │ ├── submodel.rs │ │ └── unimpl.rs │ ├── optim │ │ ├── change_axes.rs │ │ ├── concat_then_einsum.rs │ │ ├── mod.rs │ │ ├── op_optim.rs │ │ ├── prop_const.rs │ │ ├── push_split_down.rs │ │ └── slice.rs │ ├── plan.rs │ ├── runtime.rs │ ├── transform.rs │ └── value.rs └── test_data │ └── test_data.cfg ├── data ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── benches │ ├── stack_tensors.rs │ └── tensor_from_datum.rs └── src │ ├── blob.rs │ ├── datum.rs │ ├── dim │ ├── assertion.rs │ ├── mod.rs │ ├── parse.rs │ ├── resolve.rs │ ├── sym.rs │ └── tree.rs │ ├── lib.rs │ ├── macros.rs │ ├── opaque.rs │ ├── scatter.rs │ ├── tensor.rs │ └── tensor │ ├── litteral.rs │ └── view.rs ├── deny.toml ├── doc ├── README.md ├── cli-recipe.md ├── graph.md ├── intro.md ├── kernel-notes.md ├── nnef │ ├── tract-core.nnef │ ├── tract-onnx.nnef │ ├── tract-pulse.nnef │ └── tract-resource.nnef └── op.md ├── examples ├── .gitignore ├── face_detection_yolov8onnx_example │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── grace_hopper.jpg │ └── src │ │ └── main.rs ├── face_similarity_arcface_onnx │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── grace_hopper.jpeg │ ├── grace_hopper2.jpeg │ └── src │ │ ├── arc_face.rs │ │ ├── main.rs │ │ └── yolo_face.rs ├── keras-tract-tf2 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh.nope │ ├── example.onnx │ ├── example.py │ ├── io.npz │ ├── requirements.txt │ └── src │ │ └── main.rs ├── nnef-dump-mobilenet-v2 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── grace_hopper.jpg │ └── src │ │ └── main.rs ├── nnef-mobilenet-v2 │ ├── .gitignore │ ├── Cargo.toml │ ├── ci.sh │ ├── grace_hopper.jpg │ ├── imagenet_slim_labels.txt │ └── src │ │ └── main.rs ├── onnx-mobilenet-v2 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── grace_hopper.jpg │ ├── imagenet_slim_labels.txt │ └── src │ │ └── main.rs ├── pytorch-albert-v2 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── export.py │ └── src │ │ └── main.rs ├── pytorch-resnet │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── elephants.jpg │ ├── export.py │ ├── requirements.txt │ └── src │ │ └── main.rs ├── tensorflow-mobilenet-v2 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── grace_hopper.jpg │ ├── imagenet_slim_labels.txt │ └── src │ │ └── main.rs └── tflite-mobilenet-v3 │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── ci.sh │ ├── grace_hopper.jpg │ ├── imagenet_slim_labels.txt │ └── src │ └── main.rs ├── extra ├── Cargo.toml └── src │ ├── exp_unit_norm.rs │ └── lib.rs ├── gpu ├── Cargo.toml └── src │ ├── device.rs │ ├── fact.rs │ ├── lib.rs │ ├── memory │ ├── mod.rs │ ├── pool.rs │ └── schema.rs │ ├── rewrite_rules │ ├── mod.rs │ └── rewire_syncs.rs │ ├── session_handler.rs │ ├── sync.rs │ ├── tensor │ ├── arena_view.rs │ ├── mod.rs │ └── owned.rs │ └── utils.rs ├── harness ├── core-proptest-pulse │ ├── Cargo.toml │ └── src │ │ ├── conv_plus_conv.rs │ │ ├── deconv.rs │ │ ├── delay_plus_downsample.rs │ │ ├── delay_plus_pool.rs │ │ ├── einsum.rs │ │ ├── lib.rs │ │ └── pad_plus_conv.rs ├── lstm-proptest-onnx-vs-tf │ ├── Cargo.toml │ ├── proptest-regressions │ │ └── lib.txt │ └── src │ │ └── lib.rs ├── nnef-inceptionv3 │ ├── Cargo.toml │ ├── download.sh │ ├── grace_hopper.jpg │ └── src │ │ └── lib.rs ├── nnef-test-cases │ ├── .gitignore │ ├── conv-bias │ │ ├── expected │ │ ├── graph.nnef │ │ └── runme.sh │ ├── conv-q40 │ │ ├── conv2d │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_base_kernel1 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_base_kernel3 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_base_kernel9 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_dilation2 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_dilation4 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_dilation8 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_groups2 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_groups4 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_insize128 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_insize64 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ ├── conv_stride2 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ │ └── conv_stride3 │ │ │ ├── io.npz │ │ │ ├── model.nnef.tgz │ │ │ └── runme.sh │ ├── conv-with-batch │ │ ├── graph.nnef │ │ └── runme.sh │ ├── dyn_slice │ │ ├── graph.nnef │ │ └── runme.sh │ ├── fixed_roll │ │ ├── graph.nnef │ │ ├── io.npz │ │ └── runme.sh │ ├── load-store │ │ ├── expected │ │ ├── graph.nnef │ │ └── runme.sh │ ├── memory-arena │ │ ├── expected.json │ │ └── runme.sh │ ├── pool-padding │ │ ├── graph.nnef │ │ └── runme.sh │ ├── q40_linear_followed_slice │ │ ├── graph.nnef │ │ ├── linear__var0_q4_0.dat │ │ └── runme.sh │ ├── qmul │ │ ├── graph.nnef │ │ ├── graph.quant │ │ ├── io.npz │ │ └── runme.sh │ ├── range-slice-dyn-tile │ │ ├── model.nnef.tgz │ │ └── runme.sh │ ├── reshape │ │ ├── graph.nnef │ │ └── runme.sh │ ├── slice-over-slice-optim-loop │ │ ├── graph.nnef │ │ └── runme.sh │ ├── softmax │ │ ├── softmax-change-axis-1 │ │ │ ├── expected │ │ │ ├── graph.nnef │ │ │ └── runme.sh │ │ ├── softmax-change-axis │ │ │ ├── expected │ │ │ ├── graph.nnef │ │ │ └── runme.sh │ │ └── softmax-quant │ │ │ ├── expected │ │ │ ├── graph.nnef │ │ │ └── graph.quant │ │ │ ├── model │ │ │ ├── graph.nnef │ │ │ └── graph.quant │ │ │ └── runme.sh │ ├── submodel │ │ ├── expected │ │ ├── graph.nnef │ │ ├── graph.quant │ │ ├── nnet1.nnef.tar │ │ ├── nnet2 │ │ │ ├── graph.nnef │ │ │ ├── graph.quant │ │ │ ├── output_bias.dat │ │ │ └── output_weigths.dat │ │ └── runme.sh │ ├── tdim-cmp │ │ ├── graph.nnef │ │ └── runme.sh │ ├── test_manage_gru_states │ │ ├── io.npz │ │ ├── model.nnef.tgz │ │ └── runme.sh │ ├── test_upcast_f32_attn │ │ ├── io.npz │ │ ├── model.nnef.tgz │ │ └── runme.sh │ ├── tile-with-tdim │ │ ├── graph.nnef │ │ └── runme.sh │ └── uniform-mul │ │ ├── expected │ │ ├── graph.nnef │ │ ├── io.npz │ │ └── runme.sh ├── pre-optimized-graphes │ ├── .gitignore │ ├── hey_snips_v4_model17 │ │ ├── expected │ │ └── runme.sh │ └── mdl-en-2019-Q3-librispeech │ │ ├── expected │ │ └── runme.sh ├── tf-inceptionv3 │ ├── Cargo.toml │ ├── benches │ │ └── inceptionv3.rs │ ├── download.sh │ ├── grace_hopper.jpg │ └── src │ │ └── lib.rs ├── tf-mobilenet-v2 │ ├── Cargo.toml │ ├── download.sh │ └── src │ │ └── lib.rs ├── tf-moz-deepspeech │ ├── Cargo.toml │ ├── download.sh │ ├── smoketest-builder │ │ ├── .gitignore │ │ ├── Dockerfile │ │ ├── patch │ │ ├── run-env.sh │ │ ├── run_me_inside_docker.sh │ │ └── setup-env.sh │ └── src │ │ └── lib.rs └── tfl-mobilenet-v2-q │ ├── Cargo.toml │ ├── download.sh │ └── src │ └── lib.rs ├── hir ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT └── src │ ├── framework.rs │ ├── infer │ ├── analyser.rs │ ├── fact.rs │ ├── factoid.rs │ ├── helpers.rs │ ├── mod.rs │ ├── model.rs │ ├── ops.rs │ ├── optim.rs │ └── rules │ │ ├── cache.rs │ │ ├── expr.rs │ │ ├── mod.rs │ │ ├── path.rs │ │ ├── proxies.rs │ │ └── solver.rs │ ├── lib.rs │ ├── macros.rs │ └── ops │ ├── activations.rs │ ├── array │ ├── add_dims.rs │ ├── array_feature_extractor.rs │ ├── broadcast.rs │ ├── concat.rs │ ├── constant_like.rs │ ├── constant_of_shape.rs │ ├── crop.rs │ ├── dyn_slice.rs │ ├── flatten.rs │ ├── gather.rs │ ├── gather_elements.rs │ ├── gather_nd.rs │ ├── mod.rs │ ├── pad.rs │ ├── permute_axes.rs │ ├── range.rs │ ├── reshape.rs │ ├── rm_dims.rs │ ├── scatter_elements.rs │ ├── scatter_nd.rs │ ├── shape.rs │ ├── size.rs │ ├── slice.rs │ ├── split.rs │ ├── squeeze.rs │ ├── strided_slice.rs │ └── tile.rs │ ├── binary.rs │ ├── cast.rs │ ├── cnn │ ├── conv.rs │ ├── mod.rs │ └── pools.rs │ ├── downsample.rs │ ├── dummy.rs │ ├── element_wise.rs │ ├── expandable.rs │ ├── identity.rs │ ├── konst.rs │ ├── logic.rs │ ├── matmul.rs │ ├── mod.rs │ ├── nn │ ├── global_pools.rs │ ├── layer_max.rs │ ├── mod.rs │ ├── reduce.rs │ └── softmax.rs │ ├── quant.rs │ ├── scan.rs │ ├── source.rs │ └── unimpl.rs ├── libcli ├── Cargo.toml └── src │ ├── annotations.rs │ ├── display_params.rs │ ├── draw.rs │ ├── export.rs │ ├── lib.rs │ ├── model.rs │ ├── profile.rs │ ├── tensor.rs │ ├── terminal.rs │ └── time.rs ├── linalg ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── arm32 │ ├── armv7neon │ │ ├── armv7neon_mmm_f32_32x1_core.tmpl │ │ ├── armv7neon_mmm_f32_8x1_core.tmpl │ │ ├── armv7neon_mmm_f32_8x4_core.tmpl │ │ ├── armv7neon_mmm_f32_8x6_core.tmpl │ │ ├── armv7neon_mmm_f32_per_cols.tmpliq │ │ ├── armv7neon_mmm_f32_per_rows.tmpliq │ │ ├── armv7neon_mmm_f32_scalars.tmpliq │ │ ├── armv7neon_mmm_i32_32x1.tmpl │ │ ├── armv7neon_mmm_i32_8x4.tmpl │ │ ├── armv7neon_mmm_i32_per_cols.tmpliq │ │ ├── armv7neon_mmm_i32_per_rows.tmpliq │ │ ├── armv7neon_mmm_i32_scalars.tmpliq │ │ ├── armv7neon_mmm_i32_scale_q8_q15.tmpliq │ │ ├── armv7neon_mmm_q_per_col.tmpliq │ │ ├── armv7neon_mmm_q_per_row.tmpliq │ │ ├── armv7neon_mmm_q_scalar.tmpliq │ │ ├── armv7neon_prefetch.tmpl │ │ ├── armv7neon_sigmoid_f32_4n.tmpl │ │ ├── armv7neon_tanh_f32_4n.tmpl │ │ └── dispatcher.tmpliq │ └── armvfpv2 │ │ ├── armvfpv2_mmm_f32_4x4.tmpl │ │ └── dispatcher.tmpliq ├── arm64 │ ├── apple_amx │ │ ├── apple_amx_mmm_f16_64x1.tmpl │ │ ├── apple_amx_mmm_f16_64x32.tmpl │ │ ├── apple_amx_mmm_f32_32x1.tmpl │ │ ├── apple_amx_mmm_f32_32x32.tmpl │ │ ├── dispatcher.tmpliq │ │ └── instructions.rs │ ├── arm64fp16 │ │ ├── arm64fp16_leaky_relu_f16_8n.tmpl │ │ ├── arm64fp16_mmm_8h_per_col.tmpliq │ │ ├── arm64fp16_mmm_8h_per_row.tmpliq │ │ ├── arm64fp16_mmm_8h_scalar.tmpliq │ │ ├── arm64fp16_mmm_f16_128x1 │ │ │ ├── loop1 │ │ │ │ ├── cortex_a53.tmpli │ │ │ │ └── naive.tmpli │ │ │ └── loop2 │ │ │ │ └── cortex_a55.tmpli │ │ ├── arm64fp16_mmm_f16_128x1_core.tmpl │ │ ├── arm64fp16_mmm_f16_16x8 │ │ │ ├── loop1 │ │ │ │ └── naive.tmpli │ │ │ └── loop2 │ │ │ │ └── cortex_a55.tmpli │ │ ├── arm64fp16_mmm_f16_16x8_core.tmpl │ │ ├── arm64fp16_mmm_f16_32x4 │ │ │ ├── loop1 │ │ │ │ └── naive.tmpli │ │ │ └── loop2 │ │ │ │ └── cortex_a55.tmpli │ │ ├── arm64fp16_mmm_f16_32x4_core.tmpl │ │ ├── arm64fp16_mmm_f16_32x6.core.tmpl │ │ ├── arm64fp16_mmm_f16_64x1.core.tmpl │ │ ├── arm64fp16_mmm_f16_64x3.core.tmpl │ │ ├── arm64fp16_mmm_f16_per_cols.tmpliq │ │ ├── arm64fp16_mmm_f16_per_rows.tmpliq │ │ ├── arm64fp16_mmm_f16_scalars.tmpliq │ │ ├── arm64fp16_mmm_load_tile.tmpliq │ │ ├── arm64fp16_sigmoid_f16_8n.tmpl │ │ ├── arm64fp16_tanh_f16_8n.tmpl │ │ ├── dispatcher.tmpliq │ │ ├── dummy_fmla_no_pragma.S │ │ └── dummy_fmla_pragma.S │ └── arm64simd │ │ ├── arm64simd_mmm_4s_per_col.tmpliq │ │ ├── arm64simd_mmm_4s_per_row.tmpliq │ │ ├── arm64simd_mmm_4s_scalar.tmpliq │ │ ├── arm64simd_mmm_f32_12x8 │ │ ├── packed_packed_loop1 │ │ │ ├── ldr_w_no_preload.tmpli │ │ │ ├── ldr_w_preload.tmpli │ │ │ ├── ldr_x_preload.tmpli │ │ │ └── naive.tmpli │ │ └── packed_packed_loop2 │ │ │ └── cortex_a55.tmpli │ │ ├── arm64simd_mmm_f32_12x8_core.tmpl │ │ ├── arm64simd_mmm_f32_16x4 │ │ ├── packed_packed_loop1 │ │ │ ├── cortex_a53.tmpli │ │ │ └── naive.tmpli │ │ └── packed_packed_loop2 │ │ │ └── cortex_a55.tmpli │ │ ├── arm64simd_mmm_f32_16x4_core.tmpl │ │ ├── arm64simd_mmm_f32_24x4 │ │ ├── loop2 │ │ │ └── cortex_a55.tmpli │ │ └── packed_packed_loop1 │ │ │ ├── cortex_a53.tmpli │ │ │ ├── cortex_a55.tmpli │ │ │ └── naive.tmpli │ │ ├── arm64simd_mmm_f32_24x4_core.tmpl │ │ ├── arm64simd_mmm_f32_32x1_core.tmpl │ │ ├── arm64simd_mmm_f32_32x3_core.tmpl │ │ ├── arm64simd_mmm_f32_64x1 │ │ ├── loop1 │ │ │ ├── cortex_a53.tmpli │ │ │ └── naive.tmpli │ │ └── loop2 │ │ │ ├── cortex_a55.tmpli │ │ │ └── naive.tmpli │ │ ├── arm64simd_mmm_f32_64x1_core.tmpl │ │ ├── arm64simd_mmm_f32_8x8 │ │ ├── packed_packed_loop1 │ │ │ ├── broken_chains.tmpli │ │ │ ├── ldr_w_no_preload.tmpli │ │ │ ├── ldr_w_preload.tmpli │ │ │ ├── ldr_x_no_preload.tmpli │ │ │ ├── ldr_x_preload.tmpli │ │ │ └── naive.tmpli │ │ └── packed_packed_loop2 │ │ │ ├── broken_chains.tmpli │ │ │ └── cortex_a55.tmpli │ │ ├── arm64simd_mmm_f32_8x8_core.tmpl │ │ ├── arm64simd_mmm_f32_per_cols.tmpliq │ │ ├── arm64simd_mmm_f32_per_rows.tmpliq │ │ ├── arm64simd_mmm_f32_scalars.tmpliq │ │ ├── arm64simd_mmm_i32_64x1.tmpl │ │ ├── arm64simd_mmm_i32_8x8.tmpl │ │ ├── arm64simd_mmm_i32_per_cols.tmpliq │ │ ├── arm64simd_mmm_i32_per_rows.tmpliq │ │ ├── arm64simd_mmm_i32_scalars.tmpliq │ │ ├── arm64simd_mmm_i32_scale_q16_q31.tmpliq │ │ ├── arm64simd_mmm_load_tile.tmpliq │ │ ├── arm64simd_sigmoid_f32_4n.tmpl │ │ ├── arm64simd_tanh_f32_4n.tmpl │ │ └── dispatcher.tmpliq ├── benches │ ├── arm32neon.rs │ ├── arm64.rs │ ├── arm64simd.rs │ ├── intel.rs │ ├── leaky_relu.rs │ ├── mat_vec.rs │ ├── mm_for_asr_am.rs │ ├── mm_for_inception.rs │ ├── mm_for_wavenet_hw.rs │ ├── nano.rs │ ├── sigmoid.rs │ ├── softmax.rs │ ├── utils.rs │ ├── virtual_im2col.rs │ └── x86_64.rs ├── build.rs ├── cost_model │ ├── Cargo.toml │ ├── src │ │ └── main.rs │ └── train │ │ ├── README.md │ │ ├── requirements.txt │ │ ├── runme.sh │ │ └── train.py ├── matmul-bench │ ├── Cargo.toml │ ├── benches │ │ └── matmul.rs │ ├── build.rs │ ├── c │ │ ├── packed_tile_4x4.c │ │ ├── packed_tile_8x8.c │ │ ├── tile_1x1.c │ │ ├── tile_2x2.c │ │ ├── tile_4x4.c │ │ └── tile_8x8.c │ └── src │ │ └── lib.rs ├── src │ ├── arm32.rs │ ├── arm32 │ │ ├── armv7neon.rs │ │ ├── armvfpv2.rs │ │ ├── cortex_a7.rs │ │ ├── cortex_a7.txt │ │ ├── cortex_a9.rs │ │ └── cortex_a9.txt │ ├── arm64.rs │ ├── arm64 │ │ ├── apple_amx.rs │ │ ├── arm64fp16.rs │ │ ├── arm64fp16 │ │ │ ├── by_scalar.rs │ │ │ ├── leaky_relu.rs │ │ │ ├── max.rs │ │ │ ├── panel_extract.rs │ │ │ ├── sum.rs │ │ │ └── unicast.rs │ │ ├── arm64simd.rs │ │ ├── arm64simd │ │ │ ├── by_scalar.rs │ │ │ ├── leaky_relu.rs │ │ │ ├── max.rs │ │ │ ├── panel_extract.rs │ │ │ ├── softmax.rs │ │ │ ├── sum.rs │ │ │ └── unicast.rs │ │ ├── cortex_a53.rs │ │ ├── cortex_a55.rs │ │ ├── cortex_a72.rs │ │ └── cortex_a73.rs │ ├── frame │ │ ├── block_quant │ │ │ ├── helpers.rs │ │ │ ├── mod.rs │ │ │ ├── q4_0.rs │ │ │ └── value.rs │ │ ├── by_scalar.rs │ │ ├── element_wise.rs │ │ ├── element_wise_helper.rs │ │ ├── leaky_relu.rs │ │ ├── lut.rs │ │ ├── mmm │ │ │ ├── cost_model.rs │ │ │ ├── fuse.rs │ │ │ ├── input_store.rs │ │ │ ├── kernel.rs │ │ │ ├── macros.rs │ │ │ ├── mod.rs │ │ │ ├── panel_extract.rs │ │ │ ├── scratch.rs │ │ │ ├── storage.rs │ │ │ └── tests │ │ │ │ ├── frame.rs │ │ │ │ ├── fuse.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── packed_packed.rs │ │ │ │ ├── q_scale.rs │ │ │ │ └── store.rs │ │ ├── mod.rs │ │ ├── pack.rs │ │ ├── reduce │ │ │ ├── max.rs │ │ │ ├── mod.rs │ │ │ ├── softmax.rs │ │ │ └── sum.rs │ │ ├── sigmoid.rs │ │ ├── tanh.rs │ │ ├── unicast.rs │ │ └── weights.rs │ ├── generic.rs │ ├── generic │ │ ├── by_scalar.rs │ │ ├── erf.rs │ │ ├── leaky_relu.rs │ │ ├── lut.rs │ │ ├── mmm.rs │ │ ├── reduce.rs │ │ ├── rounding.rs │ │ ├── sigmoid.rs │ │ ├── tanh.rs │ │ └── unicast.rs │ ├── lib.rs │ ├── multithread.rs │ ├── wasm.rs │ ├── x86_64_fma.rs │ └── x86_64_fma │ │ ├── by_scalar.rs │ │ ├── intel.rs │ │ ├── max.rs │ │ ├── mmm.rs │ │ ├── panel_extract.rs │ │ └── softmax.rs ├── tests │ └── virtual_im2col.rs └── x86_64 │ ├── avx512 │ ├── 10x1 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 1x1 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512.tmpli │ │ │ ├── unroll-16.tmpli │ │ │ ├── unroll-4.tmpli │ │ │ ├── unroll-8.tmpli │ │ │ └── unroll.tmpli │ ├── 1x12 │ │ └── packed_packed_loop1 │ │ │ └── avx-512.tmpli │ ├── 2x5 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 2x6 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 3x4 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 4x3 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 5x2 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 6x1 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 6x2 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 7x1 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 8x1 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── 8x2 │ │ └── packed_packed_loop1 │ │ │ └── avx-512.tmpli │ ├── 8x8 │ │ └── packed_packed_loop1 │ │ │ ├── avx-512-unroll.tmpli │ │ │ └── avx-512.tmpli │ ├── avx512_mmm_f32_128x1.tmpl │ ├── avx512_mmm_f32_16x1.tmpl │ ├── avx512_mmm_f32_16x12.tmpl │ ├── avx512_mmm_f32_16x8.tmpl │ ├── avx512_mmm_f32_32x5.tmpl │ ├── avx512_mmm_f32_32x6.tmpl │ ├── avx512_mmm_f32_48x4.tmpl │ ├── avx512_mmm_f32_64x3.tmpl │ ├── avx512_mmm_f32_80x2.tmpl │ ├── avx512_mmm_load_tile.tmpliq │ ├── dispatcher.tmpliq │ ├── f32_per_cols.tmpliq │ ├── f32_per_rows.tmpliq │ ├── f32_scalars.tmpliq │ ├── i32_per_cols.tmpliq │ ├── i32_per_rows.tmpliq │ ├── i32_scalars.tmpliq │ ├── postamble.tmpliq │ ├── preamble.tmpliq │ ├── sigmoid_f32.tmpl │ ├── tanh_f32.tmpl │ ├── zmm_per_col.tmpliq │ ├── zmm_per_row.tmpliq │ └── zmm_scalar.tmpliq │ └── fma │ ├── 10x1 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 2x5 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 2x6 │ └── packed_packed_loop1 │ │ ├── original-unroll.tmpli │ │ └── original.tmpli │ ├── 3x4 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 4x3 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 5x2 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 6x1 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 6x2 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 7x1 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 8x1 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── 8x8 │ └── packed_packed_loop1 │ │ ├── avx-unroll.tmpli │ │ └── avx.tmpli │ ├── avx2_mmm_i32_8x8.tmpl │ ├── dispatcher.tmpliq │ ├── fma_mmm_f32_16x5.tmpl │ ├── fma_mmm_f32_16x6.tmpl │ ├── fma_mmm_f32_24x4.tmpl │ ├── fma_mmm_f32_32x1.tmpl │ ├── fma_mmm_f32_32x3.tmpl │ ├── fma_mmm_f32_40x2.tmpl │ ├── fma_mmm_f32_64x1.tmpl │ ├── fma_mmm_f32_8x8.tmpl │ ├── fma_mmm_f32_per_cols.tmpliq │ ├── fma_mmm_f32_per_rows.tmpliq │ ├── fma_mmm_f32_scalars.tmpliq │ ├── fma_mmm_i32_per_cols.tmpliq │ ├── fma_mmm_i32_per_rows.tmpliq │ ├── fma_mmm_i32_scalars.tmpliq │ ├── fma_mmm_load_tile.tmpliq │ ├── fma_mmm_ymm_per_col.tmpliq │ ├── fma_mmm_ymm_per_row.tmpliq │ ├── fma_mmm_ymm_scalar.tmpliq │ ├── fma_sigmoid_f32.tmpl │ ├── fma_tanh_f32.tmpl │ ├── postamble.tmpliq │ └── preamble.tmpliq ├── metal ├── Cargo.toml ├── README.md ├── benches │ └── metal_gemm.rs └── src │ ├── command_buffer.rs │ ├── context.rs │ ├── encoder.rs │ ├── func_constants.rs │ ├── kernels │ ├── array │ │ ├── array_ops.metal │ │ ├── broadcast.rs │ │ ├── cast.rs │ │ ├── concat.rs │ │ ├── copy.rs │ │ ├── mod.rs │ │ ├── permute_axes.rs │ │ └── rotate_half.rs │ ├── bin_ops.metal │ ├── bin_ops.rs │ ├── element_wise.metal │ ├── element_wise.rs │ ├── matmul │ │ ├── basic │ │ │ ├── basic_mat_mul.metal │ │ │ └── mod.rs │ │ ├── ggml_gemm │ │ │ ├── README.md │ │ │ ├── ggml_mm_mv.metal │ │ │ └── mod.rs │ │ ├── mfa │ │ │ ├── libMetalFlashAttention-ios.metallib │ │ │ ├── libMetalFlashAttention-macos.metallib │ │ │ └── mod.rs │ │ ├── mlx_gemm │ │ │ ├── mlx_gemm.metal │ │ │ ├── mlx_gemv.metal │ │ │ └── mod.rs │ │ └── mod.rs │ ├── mod.rs │ ├── nn │ │ ├── apply_rope.rs │ │ ├── gelu_approximate.rs │ │ ├── mod.rs │ │ ├── nn_ops.metal │ │ ├── reduce.rs │ │ ├── rms_norm.rs │ │ ├── scaled_masked_softmax.rs │ │ ├── silu.rs │ │ └── softmax.rs │ └── utils.rs │ ├── lib.rs │ ├── ops │ ├── apply_rope.rs │ ├── binary.rs │ ├── broadcast.rs │ ├── cast.rs │ ├── change_axes.rs │ ├── concat.rs │ ├── element_wise.rs │ ├── fused_axis_op.rs │ ├── gelu_approximate.rs │ ├── gemm.rs │ ├── mod.rs │ ├── reduce.rs │ ├── rms_norm.rs │ ├── rotate_half.rs │ ├── scaled_masked_softmax.rs │ ├── silu.rs │ ├── slice.rs │ └── softmax.rs │ ├── rewrite_rules │ ├── add_matmul_broadcast.rs │ ├── fuse_axis_op.rs │ ├── mod.rs │ ├── rms_norm.rs │ └── untranspose_matmul_output.rs │ ├── tests.rs │ ├── transform.rs │ └── utils.rs ├── nnef ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── cli │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── nnef-resources │ ├── Cargo.toml │ ├── src │ │ ├── json_loader.rs │ │ └── lib.rs │ └── tests │ │ ├── nnef_with_json │ │ ├── graph.nnef │ │ └── src_config.json │ │ └── test_json_resource.rs ├── src │ ├── ast.rs │ ├── ast │ │ ├── dump.rs │ │ ├── dump_doc.rs │ │ ├── parse.rs │ │ └── quant.rs │ ├── deser.rs │ ├── framework.rs │ ├── lib.rs │ ├── ops │ │ ├── core.rs │ │ ├── core │ │ │ ├── broadcast.rs │ │ │ ├── cast.rs │ │ │ ├── complex.rs │ │ │ ├── downsample.rs │ │ │ ├── dyn_slice.rs │ │ │ ├── einsum.rs │ │ │ ├── fft.rs │ │ │ ├── force_eval.rs │ │ │ ├── gather.rs │ │ │ ├── load.rs │ │ │ ├── matmul.rs │ │ │ ├── one_hot.rs │ │ │ ├── qconv.rs │ │ │ ├── qmatmul.rs │ │ │ ├── range.rs │ │ │ ├── reduce.rs │ │ │ ├── scan.rs │ │ │ ├── scatter.rs │ │ │ ├── shape_of.rs │ │ │ ├── softmax.rs │ │ │ ├── source.rs │ │ │ ├── store.rs │ │ │ ├── submodel.rs │ │ │ ├── topk.rs │ │ │ └── trilu.rs │ │ ├── mod.rs │ │ ├── nnef │ │ │ ├── deser.rs │ │ │ ├── mod.rs │ │ │ └── ser.rs │ │ └── resource.rs │ ├── registry.rs │ ├── resource.rs │ ├── ser.rs │ └── tensors.rs ├── stdlib.nnef └── tests │ ├── alexnet.nnef │ └── parse.rs ├── onnx-opl ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT └── src │ ├── is_inf.rs │ ├── is_nan.rs │ ├── lib.rs │ ├── lrn.rs │ ├── ml │ ├── category_mapper.rs │ ├── mod.rs │ ├── tree.rs │ └── tree_ensemble_classifier.rs │ ├── multinomial.rs │ ├── non_max_suppression.rs │ └── random.rs ├── onnx ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── build-proto.rs ├── protos │ └── onnx │ │ ├── onnx-operators.proto3 │ │ ├── onnx.proto │ │ └── onnx.proto3 ├── src │ ├── data_resolver.rs │ ├── lib.rs │ ├── model.rs │ ├── ops │ │ ├── array │ │ │ ├── compress.rs │ │ │ ├── mod.rs │ │ │ ├── nonzero.rs │ │ │ ├── one_hot.rs │ │ │ ├── pad.rs │ │ │ ├── shape.rs │ │ │ ├── slice.rs │ │ │ ├── split.rs │ │ │ ├── squeeze.rs │ │ │ ├── topk.rs │ │ │ ├── trilu.rs │ │ │ └── unsqueeze.rs │ │ ├── cast.rs │ │ ├── cumsum.rs │ │ ├── d2s.rs │ │ ├── einsum.rs │ │ ├── fft.rs │ │ ├── logic.rs │ │ ├── math.rs │ │ ├── math │ │ │ ├── clip.rs │ │ │ ├── gemm.rs │ │ │ ├── mat_mul_integer.rs │ │ │ ├── pow.rs │ │ │ └── rem.rs │ │ ├── ml │ │ │ ├── category_mapper.rs │ │ │ ├── mod.rs │ │ │ └── tree_ensemble_classifier.rs │ │ ├── mod.rs │ │ ├── multinomial.rs │ │ ├── nn │ │ │ ├── batch_norm.rs │ │ │ ├── conv_transpose.rs │ │ │ ├── dropout.rs │ │ │ ├── instance_norm.rs │ │ │ ├── layer_norm.rs │ │ │ ├── lrn.rs │ │ │ ├── mod.rs │ │ │ └── reduce.rs │ │ ├── non_max_suppression.rs │ │ ├── quant.rs │ │ ├── random.rs │ │ ├── rec.rs │ │ ├── rec │ │ │ ├── common.rs │ │ │ ├── gru.rs │ │ │ ├── lstm.rs │ │ │ ├── rnn.rs │ │ │ └── scan.rs │ │ ├── resize.rs │ │ └── s2d.rs │ ├── pb_helpers.rs │ ├── prost │ │ └── onnx.rs │ └── tensor.rs └── test_cases │ ├── byte_sb_bidi_lstm │ ├── README.md │ ├── generate_io.py │ ├── io.npz │ └── model.onnx │ ├── deconv_group │ ├── io.npz │ ├── model.onnx │ └── vars.sh │ ├── lgbm_classifier_tensor │ ├── generate_io.py │ ├── io.npz │ ├── model.onnx │ └── vars.sh │ ├── lgbm_regressor_tensor │ ├── generate_io.py │ ├── io.npz │ ├── model.onnx │ └── vars.sh │ ├── qlstm_3-2-3_T3_S1 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qrelu_1 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qrelu_2 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qsigmoid_1 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qsigmoid_2 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qtanh_1 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qtanh_2 │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── qtdnn_10x5_101_i32_biases │ ├── final.mdl │ ├── io.npz │ ├── model.onnx │ ├── model.raw │ └── vars.sh │ ├── run_all.sh │ ├── tinyyolov2 │ ├── io.npz │ └── vars.sh │ ├── transformer-mlm │ ├── generate_io.py │ ├── io.npz │ └── vars.sh │ ├── xgboost_classifier_tree │ ├── generate_io.py │ ├── io.npz │ ├── model.onnx │ └── vars.sh │ └── xgboost_regressor_tree │ ├── generate_io.py │ ├── io.npz │ ├── model.onnx │ └── vars.sh ├── post-release.sh ├── pulse-opl ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT └── src │ ├── concat.rs │ ├── deconv_delay.rs │ ├── delay.rs │ ├── lib.rs │ ├── mask.rs │ ├── pad.rs │ └── slice.rs ├── pulse ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT └── src │ ├── fact.rs │ ├── lib.rs │ ├── macros.rs │ ├── model.rs │ └── ops │ ├── array │ ├── broadcast.rs │ ├── concat.rs │ ├── mask.rs │ ├── mod.rs │ ├── pad.rs │ └── slice.rs │ ├── cnn │ ├── conv.rs │ ├── deconv.rs │ ├── mod.rs │ └── pools.rs │ ├── delay.rs │ ├── downsample.rs │ ├── dummy.rs │ ├── identity.rs │ ├── mask.rs │ ├── mod.rs │ ├── scan.rs │ ├── slice.rs │ └── source.rs ├── release.sh ├── rustfmt.toml ├── tensorflow ├── Cargo.toml ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── benches │ └── hey_snips_3.pb ├── build-proto.rs ├── examples │ └── plus3.rs ├── protos │ └── tensorflow │ │ └── core │ │ ├── framework │ │ ├── attr_value.proto │ │ ├── function.proto │ │ ├── graph.proto │ │ ├── node_def.proto │ │ ├── op_def.proto │ │ ├── resource_handle.proto │ │ ├── tensor.proto │ │ ├── tensor_shape.proto │ │ ├── types.proto │ │ ├── variable.proto │ │ └── versions.proto │ │ └── protobuf │ │ ├── meta_graph.proto │ │ ├── saved_model.proto │ │ ├── saved_object_graph.proto │ │ ├── saver.proto │ │ ├── struct.proto │ │ └── trackable_object_graph.proto ├── src │ ├── conform │ │ ├── mod.rs │ │ └── tf.rs │ ├── lib.rs │ ├── model.rs │ ├── ops │ │ ├── array │ │ │ ├── concatv2.rs │ │ │ ├── expand_dims.rs │ │ │ ├── fill.rs │ │ │ ├── gather_nd.rs │ │ │ ├── gather_v2.rs │ │ │ ├── mod.rs │ │ │ ├── pack.rs │ │ │ ├── pad.rs │ │ │ ├── squeeze.rs │ │ │ └── transpose.rs │ │ ├── control_flow.rs │ │ ├── logic.rs │ │ ├── math.rs │ │ ├── math │ │ │ └── reduce.rs │ │ ├── mod.rs │ │ ├── nn │ │ │ ├── conv2d.rs │ │ │ ├── dw_conv2d.rs │ │ │ ├── fused_batch_norm.rs │ │ │ ├── mod.rs │ │ │ ├── pools.rs │ │ │ └── s2b │ │ │ │ ├── mod.rs │ │ │ │ ├── raw.rs │ │ │ │ └── unary.rs │ │ ├── quant.rs │ │ ├── random │ │ │ ├── mod.rs │ │ │ ├── philox.rs │ │ │ └── random_uniform.rs │ │ ├── rec │ │ │ ├── block_lstm.rs │ │ │ └── mod.rs │ │ └── vars.rs │ ├── prost │ │ ├── google.protobuf.rs │ │ └── tensorflow.rs │ ├── tensor.rs │ └── tfpb.rs └── tests │ ├── models │ └── plus3.pb │ ├── ops_array_pack.rs │ ├── ops_array_strided_slice.proptest-regressions │ ├── ops_array_strided_slice.rs │ ├── ops_fake_quant_with_min_max_vars.rs │ ├── ops_nn_conv2d.proptest-regressions │ ├── ops_nn_conv2d.rs │ ├── ops_nn_dwconv2d.proptest-regressions │ ├── ops_nn_dwconv2d.rs │ ├── ops_nn_pools.proptest-regressions │ ├── ops_nn_pools.rs │ ├── ops_nn_space_to_batch.proptest-regressions │ ├── ops_nn_space_to_batch.rs │ ├── ops_random_uniform.rs │ └── utils │ └── mod.rs ├── test-rt ├── infra │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── suite-onnx │ ├── Cargo.toml │ ├── node.txt │ ├── pytorch-converted.txt │ ├── pytorch-operator.txt │ ├── simple.txt │ └── src │ │ └── lib.rs ├── suite-unit │ ├── Cargo.toml │ └── src │ │ ├── apply_rope.rs │ │ ├── bin_einsum.rs │ │ ├── conv_f32.rs │ │ ├── conv_q.rs │ │ ├── deconv.rs │ │ ├── downsample.rs │ │ ├── gelu_approximate.rs │ │ ├── lib.rs │ │ ├── matmul_q40.rs │ │ ├── q_binary.rs │ │ ├── q_elmwise.rs │ │ ├── q_flavours.rs │ │ ├── q_helpers.rs │ │ ├── rms_norm.rs │ │ ├── scaled_masked_softmax.rs │ │ ├── silu.rs │ │ └── slice.rs ├── test-blas │ ├── Cargo.toml │ ├── build.rs │ ├── src │ │ └── lib.rs │ └── suite.rs ├── test-f16 │ ├── Cargo.toml │ ├── build.rs │ ├── src │ │ └── lib.rs │ └── suite.rs ├── test-metal │ ├── Cargo.toml │ ├── build.rs │ ├── ggml_suite.rs │ ├── src │ │ └── lib.rs │ └── suite.rs ├── test-nnef-cycle │ ├── Cargo.toml │ ├── build.rs │ ├── src │ │ └── lib.rs │ └── suite.rs ├── test-onnx-core │ ├── Cargo.toml │ ├── build.rs │ ├── debug-utils │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── save_all.py │ │ └── src │ │ │ └── main.rs │ ├── include-passing-ignored.sh │ └── src │ │ ├── bin │ │ └── reset-test-list.rs │ │ └── lib.rs ├── test-tflite │ ├── Cargo.toml │ ├── build.rs │ ├── src │ │ ├── lib.rs │ │ └── tflite_runtime.rs │ └── suite.rs └── test-unit-core │ ├── Cargo.toml │ ├── build.rs │ └── src │ ├── lib.rs │ └── main.rs ├── tflite ├── Cargo.toml ├── Readme.md ├── schema │ └── tflite.fbs └── src │ ├── lib.rs │ ├── model.rs │ ├── ops │ ├── array.rs │ ├── cnn.rs │ ├── element_wise.rs │ ├── math.rs │ ├── mod.rs │ └── nn.rs │ ├── registry.rs │ ├── rewriter.rs │ ├── ser.rs │ ├── tensors.rs │ └── tflite_generated.rs ├── transformers ├── Cargo.toml └── src │ ├── lib.rs │ ├── ops │ ├── apply_rope.rs │ ├── gelu_approximate.rs │ ├── mod.rs │ ├── rms_norm.rs │ ├── scaled_masked_softmax.rs │ └── silu.rs │ └── rewriter.rs └── yank.sh /.all_crates.sh: -------------------------------------------------------------------------------- 1 | 2 | ALL_CRATES_PATH="data linalg core nnef nnef/nnef-resources pulse-opl pulse extra transformers hir tflite tensorflow onnx-opl onnx gpu metal libcli api api/rs api/ffi api/proxy/sys api/proxy cli" 3 | -------------------------------------------------------------------------------- /.change_crate_dep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | crate=$1 4 | version=$2 5 | 6 | perl -pi -e "s/^($crate = {.*version *= *)\"([^\"]*)\"(.*)$/\$1\"=$version\"\$3/" \ 7 | `find . -name Cargo.toml \! -path "./target/*" \! -path "./issue*"` 8 | -------------------------------------------------------------------------------- /.github/workflows/asan.yml: -------------------------------------------------------------------------------- 1 | name: Sanitized build tests 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '0 5 * * MON' 7 | 8 | env: 9 | CARGO_INCREMENTAL: false 10 | FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true 11 | 12 | jobs: 13 | sanitizer-address: 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | os: [ ubuntu-latest, macOS-latest ] 18 | 19 | runs-on: ${{matrix.os}} 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Rustup update 24 | run: rustup update 25 | - name: Run sanitized tests 26 | run: .travis/asan.sh 27 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - 'v*' 5 | 6 | name: Create release 7 | 8 | env: 9 | CARGO_INCREMENTAL: false 10 | FORCE_JAVASCRIPT_ACTIONS_TO_NODE20: true 11 | 12 | jobs: 13 | release: 14 | name: Create release 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Extract version tag 18 | id: version 19 | run: echo value=$(echo ${{ github.ref }} | cut -f 3 -d / | tr -d 'v' ) >> $GITHUB_OUTPUT 20 | 21 | - name: Create Release 22 | id: create_release 23 | uses: actions/create-release@latest 24 | env: 25 | GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN_RELEASE }} 26 | with: 27 | tag_name: ${{ steps.version.outputs.value }} 28 | release_name: tract ${{ steps.version.outputs.value }} 29 | 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | **/*.rs.bk 3 | *.rustfmt 4 | *.back 5 | Cargo.lock 6 | examples/data 7 | .idea 8 | .cached/** 9 | flamegraph.svg 10 | perf.data* 11 | readings.* 12 | metrics 13 | tract.out 14 | .gdb_history 15 | /issue-* 16 | /.dinghy.toml 17 | .cargo 18 | proptest-regressions 19 | /tmp 20 | wheelhouse 21 | target-bisector* 22 | -------------------------------------------------------------------------------- /.travis/asan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | # RUSTFLAGS=-Zsanitizer=address cargo +nightly test -Zbuild-std --target $(rustc -vV | sed -n 's|host: ||p') 6 | 7 | TARGET=$(rustc -vV | sed -n 's|host: ||p') 8 | 9 | rustup toolchain add nightly 10 | rustup component add rust-src --toolchain nightly-$TARGET 11 | 12 | export RUSTFLAGS=-Zsanitizer=address 13 | export RUSTDOCFLAGS=$RUSTFLAGS 14 | export RUSTUP_TOOLCHAIN=nightly 15 | export RUST_VERSION=nightly 16 | export CARGO_EXTRA="--target $TARGET" 17 | 18 | cargo -q test -q -p tract-core --features paranoid_assertions $CARGO_EXTRA 19 | 20 | ./.travis/regular-tests.sh 21 | if [ -n "$CI" ] 22 | then 23 | cargo clean 24 | fi 25 | ./.travis/onnx-tests.sh 26 | if [ -n "$CI" ] 27 | then 28 | cargo clean 29 | fi 30 | ./.travis/cli-tests.sh 31 | 32 | -------------------------------------------------------------------------------- /.travis/cache_file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | if [ -z "$CACHEDIR" ] 6 | then 7 | CACHEDIR=`dirname $0`/../.cached 8 | fi 9 | 10 | mkdir -p $CACHEDIR 11 | cd $CACHEDIR 12 | for file in $@ 13 | do 14 | mkdir -p $(dirname $file) 15 | if [ ! -e $file ] 16 | then 17 | wget --no-verbose https://s3.amazonaws.com/tract-ci-builds/tests/$file -O $file.tmp \ 18 | || aws s3 cp s3://tract-ci-builds/tests/$file $file.tmp 19 | mv $file.tmp $file 20 | fi 21 | done 22 | 23 | exit 0 24 | -------------------------------------------------------------------------------- /.travis/cargo-deny-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -e cargo-deny ] 4 | then 5 | CARGO_DENY=`pwd`/cargo-deny 6 | else 7 | CARGO_DENY="cargo deny" 8 | fi 9 | 10 | for crate in onnx pulse tensorflow 11 | do 12 | echo $crate: 13 | (cd $crate ; $CARGO_DENY check) 14 | done 15 | -------------------------------------------------------------------------------- /.travis/debug-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | if [ -z "$CACHEDIR" ] 6 | then 7 | CACHEDIR=`dirname $0`/../.cached 8 | fi 9 | 10 | # useful as debug_asserts will come into play 11 | cargo test -p tract-core 12 | cargo test -p test-onnx-core -p test-nnef-cycle -p test-unit-core 13 | -------------------------------------------------------------------------------- /.travis/docker-debian-stretch/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stretch 2 | COPY sources.list /etc/apt/sources.list 3 | -------------------------------------------------------------------------------- /.travis/docker-debian-stretch/sources.list: -------------------------------------------------------------------------------- 1 | deb http://archive.debian.org/debian/ stretch contrib main non-free 2 | deb http://archive.debian.org/debian stretch-backports main 3 | deb http://archive.debian.org/debian-security stretch/updates main 4 | -------------------------------------------------------------------------------- /.travis/examples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | WHITE='\033[1;37m' 4 | NC='\033[0m' # No Color 5 | 6 | set -e 7 | 8 | which rustup || curl https://sh.rustup.rs -sSf | sh -s -- -y 9 | rustup update 10 | 11 | PATH=$PATH:$HOME/.cargo/bin 12 | 13 | : "${RUST_VERSION:=stable}" 14 | rustup toolchain add $RUST_VERSION 15 | rustup default $RUST_VERSION 16 | 17 | for t in `find examples -name ci.sh` 18 | do 19 | df -h 20 | ex=$(dirname $t) 21 | echo ::group:: $ex 22 | echo $WHITE $ex $NC 23 | ( cd $ex ; sh ./ci.sh ) 24 | if [ -n "$CI" ] 25 | then 26 | cargo clean 27 | fi 28 | echo ::endgroup:: 29 | done 30 | 31 | -------------------------------------------------------------------------------- /.travis/minionrc: -------------------------------------------------------------------------------- 1 | MINION_ID= 2 | LOCKFILE=/tmp/minion-lock 3 | PLATFORM=raspbian 4 | GRAPHITE_HOST=graphite-proxy.snips.net 5 | GRAPHITE_PORT=2003 6 | GRAPHITE_PREFIX=tract 7 | 8 | S3PATH_TASKS=tract-ci-builds/tasks 9 | S3PATH_LOGS=tract-ci-builds/logs 10 | S3PATH_RESULTS=tract-ci-builds/logs 11 | WORKDIR=$HOME/tract-minion 12 | CACHEDIR=$WORKDIR/cache 13 | -------------------------------------------------------------------------------- /.travis/native.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | if [ -z "$RUSTUP_TOOLCHAIN" ] 6 | then 7 | export RUSTUP_TOOLCHAIN=1.85.0 8 | fi 9 | 10 | rustup update 11 | 12 | cargo update 13 | cargo check --all-targets --workspace --exclude test-tflite --exclude test-metal --exclude tract-metal 14 | 15 | ./.travis/onnx-tests.sh 16 | ./.travis/regular-tests.sh 17 | ./.travis/test-harness.sh 18 | 19 | if [ -n "$CI" ] 20 | then 21 | cargo clean 22 | fi 23 | 24 | if [ `uname` = "Linux" ] 25 | then 26 | ./.travis/tflite.sh 27 | fi 28 | 29 | if [ `uname` = "Darwin" ] 30 | then 31 | cargo test -p test-metal 32 | fi 33 | 34 | 35 | if [ -n "$CI" ] 36 | then 37 | cargo clean 38 | fi 39 | ./.travis/cli-tests.sh 40 | -------------------------------------------------------------------------------- /.travis/onnx-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | ROOT=$(dirname $(realpath $0))/.. 6 | . $ROOT/.travis/ci-system-setup.sh 7 | 8 | opset=onnx_"${1:-1_13_0}" 9 | 10 | cargo -q test -p test-unit-core $CARGO_EXTRA -q 11 | cargo -q test -p test-onnx-core $CARGO_EXTRA -q --no-default-features --features $opset 12 | cargo -q test -p test-nnef-cycle $CARGO_EXTRA -q --no-default-features 13 | -------------------------------------------------------------------------------- /.travis/regular-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | set -x 5 | 6 | cd $(dirname $0) 7 | 8 | ./test-published-crates.sh 9 | if [ -n "$CI" ] 10 | then 11 | cargo clean 12 | fi 13 | ./test-rt.sh 14 | if [ -n "$CI" ] 15 | then 16 | cargo clean 17 | fi 18 | -------------------------------------------------------------------------------- /.travis/run-bundle.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | BUNDLE_NAME=$1 6 | 7 | tar zxf $BUNDLE_NAME.tgz 8 | ( 9 | cd $BUNDLE_NAME 10 | . ./vars 11 | ./entrypoint.sh 12 | ) 13 | # rm -rf "$BUNDLE_NAME" "$BUNDLE_NAME.tgz" 14 | -------------------------------------------------------------------------------- /.travis/run_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | `dirname $0`/native.sh 6 | cd `dirname $0`/../examples 7 | for i in * 8 | do 9 | (cd $i; cargo test --release) 10 | done 11 | -------------------------------------------------------------------------------- /.travis/tf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | if [ -z "$CACHEDIR" ] 6 | then 7 | CACHEDIR=`dirname $0`/../.cached 8 | fi 9 | 10 | 11 | (cd tensorflow; cargo test --release --features conform) 12 | -------------------------------------------------------------------------------- /.travis/tflite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | which rustup || curl https://sh.rustup.rs -sSf | sh -s -- -y 6 | 7 | PATH=$PATH:$HOME/.cargo/bin 8 | 9 | : "${RUST_VERSION:=stable}" 10 | rustup toolchain add $RUST_VERSION 11 | rustup default $RUST_VERSION 12 | 13 | rustc --version 14 | 15 | if [ `uname` = "Darwin" ] 16 | then 17 | brew install coreutils 18 | fi 19 | if [ -n "$GITHUB_ACTIONS" ] 20 | then 21 | pip install numpy 22 | fi 23 | 24 | cargo check -p tract-tflite 25 | cargo -q test -p test-tflite $CARGO_EXTRA -q 26 | -------------------------------------------------------------------------------- /.travis/tflite/Dockerfile.tensorflow-aarch64: -------------------------------------------------------------------------------- 1 | # vim: set syntax=Dockerfile: 2 | 3 | FROM tensorflow/tensorflow:devel 4 | 5 | RUN apt-get update ; apt-get upgrade -y 6 | RUN apt-get install -y crossbuild-essential-arm64 7 | COPY linux_makefile.inc /tensorflow_src/tensorflow/lite/tools/make/targets/linux_makefile.inc 8 | COPY disable_nnapi.patch /tensorflow_src 9 | 10 | WORKDIR /tensorflow_src 11 | RUN patch -p1 < disable_nnapi.patch 12 | -------------------------------------------------------------------------------- /.travis/tflite/Dockerfile.tensorflow-official-rpi: -------------------------------------------------------------------------------- 1 | # vim: set syntax=Dockerfile: 2 | 3 | FROM tensorflow/tensorflow:nightly-devel 4 | 5 | RUN apt-get update ; apt-get upgrade -y 6 | RUN apt-get -y install git crossbuild-essential-armhf 7 | 8 | WORKDIR /tensorflow 9 | RUN ./tensorflow/lite/tools/make/download_dependencies.sh 10 | 11 | 12 | -------------------------------------------------------------------------------- /.travis/tflite/Dockerfile.tensorflow-rpitools: -------------------------------------------------------------------------------- 1 | # vim: set syntax=Dockerfile: 2 | 3 | FROM tensorflow/tensorflow:nightly-devel 4 | 5 | RUN apt-get update ; apt-get upgrade -y 6 | RUN apt-get -yy install git 7 | 8 | WORKDIR /tensorflow 9 | RUN ./tensorflow/lite/tools/make/download_dependencies.sh 10 | 11 | RUN git clone https://github.com/raspberrypi/tools /raspitools 12 | ENV PATH=/raspitools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin:$PATH 13 | 14 | -------------------------------------------------------------------------------- /.travis/tflite/build_tflite_aarch64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | mkdir -p result 5 | 6 | docker build -f Dockerfile.tensorflow-aarch64 --tag tensorflow-aarch64 . 7 | docker run --rm -it \ 8 | -v `pwd`/result:/result \ 9 | tensorflow-aarch64 \ 10 | sh -c " 11 | cd /tensorflow_src ; 12 | export EXTRA_CXXFLAGS=-flax-vector-conversions 13 | export DISABLE_NNAPI=true 14 | ./tensorflow/lite/tools/make/download_dependencies.sh 15 | make -j 3 -f tensorflow/lite/tools/make/Makefile TARGET=linux TARGET_ARCH=aarch64 ; 16 | cp /tensorflow_src/tensorflow/lite/tools/make/gen/linux_aarch64/bin/benchmark_model /result/tflite_benchmark_model_aarch64 17 | " 18 | -------------------------------------------------------------------------------- /.travis/tflite/linux_makefile.inc: -------------------------------------------------------------------------------- 1 | # Settings for Linux. 2 | ifeq ($(TARGET), linux) 3 | CXXFLAGS += \ 4 | -fPIC \ 5 | -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \ 6 | -pthread 7 | # TODO(petewarden): In the future we may want to add architecture-specific 8 | # flags like -msse4.2 9 | LIBS := -lstdc++ -lpthread -lm -ldl 10 | TARGET_ARCH=aarch64 11 | TARGET_TOOLCHAIN_PREFIX := aarch64-linux-gnu- 12 | endif 13 | -------------------------------------------------------------------------------- /.travis/tflite/run_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./benchmark_model --graph=inception_v3_2016_08_28_frozen.tflite 4 | 5 | -------------------------------------------------------------------------------- /.travis/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | if [ -z "$PLATFORM" ] 6 | then 7 | .travis/native.sh 8 | else 9 | .travis/cross.sh 10 | fi 11 | -------------------------------------------------------------------------------- /.vim/coc-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.imports.granularity.group": "module" 3 | } 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /api/.gitignore: -------------------------------------------------------------------------------- 1 | *.nnef.tgz 2 | *.onnx 3 | -------------------------------------------------------------------------------- /api/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /api/c/.gitignore: -------------------------------------------------------------------------------- 1 | *.h 2 | *.so 3 | mobilenet 4 | mobilenet_v2_1.0.onnx.nnef.tgz 5 | 6 | 7 | -------------------------------------------------------------------------------- /api/c/Makefile: -------------------------------------------------------------------------------- 1 | run: mobilenet mobilenet_v2_1.0.onnx.nnef.tgz 2 | LD_LIBRARY_PATH=. ./mobilenet 3 | 4 | clean: 5 | rm -f mobilenet libtract.so tract.h 6 | 7 | mobilenet: tract.h libtract.so mobilenet.c 8 | cc mobilenet.c -o mobilenet -L. -ltract 9 | 10 | libtract.so: 11 | cargo build -p tract-ffi --release 12 | cp ../../target/release/libtract.so . 13 | 14 | tract.h: 15 | cd ../ffi ; cbindgen -l c > tract.h 16 | cp ../ffi/tract.h . 17 | 18 | mobilenet_v2_1.0.onnx.nnef.tgz: 19 | wget -q https://s3.amazonaws.com/tract-ci-builds/tests/mobilenet_v2_1.0.onnx.nnef.tgz 20 | -------------------------------------------------------------------------------- /api/c/grace_hopper_3_224_224.f32.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/api/c/grace_hopper_3_224_224.f32.raw -------------------------------------------------------------------------------- /api/ffi/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-ffi" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, neural network inference" 7 | repository = "https://github.com/snipsco/tract" 8 | keywords = [ "TensorFlow", "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | include = [ "Cargo.toml", "src/**/*.rs", "LICENSE*" ] 13 | 14 | [lib] 15 | name = "tract" 16 | crate-type = ["cdylib"] 17 | 18 | [badges] 19 | maintenance = { status = "actively-developed" } 20 | 21 | [dependencies] 22 | anyhow.workspace = true 23 | flate2.workspace = true 24 | serde.workspace = true 25 | serde_json.workspace = true 26 | tract-api.workspace = true 27 | tract-rs.workspace = true 28 | -------------------------------------------------------------------------------- /api/generate-tract-h.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | cargo install bindgen-cli 6 | cargo install cbindgen 7 | 8 | cbindgen ffi > tract.h 9 | cp tract.h c 10 | bindgen tract.h -o proxy/sys/bindings.rs --rust-target 1.85.0 --rust-edition 2024 11 | -------------------------------------------------------------------------------- /api/proxy/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-proxy" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/sonos/tract" 8 | keywords = [ "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | rust-version = "1.85" 13 | include = [ "Cargo.toml", "src/**/*.rs", "LICENSE*" ] 14 | 15 | [dependencies] 16 | anyhow.workspace = true 17 | boow.workspace = true 18 | home.workspace = true 19 | ndarray.workspace = true 20 | tract-api.workspace = true 21 | tract-proxy-sys.workspace = true 22 | 23 | [dev-dependencies] 24 | reqwest.workspace = true 25 | tempfile.workspace = true 26 | serde_json.workspace = true 27 | -------------------------------------------------------------------------------- /api/proxy/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /api/proxy/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ROOT=$(dirname $(realpath $0))/../.. 4 | 5 | set -ex 6 | 7 | cargo build --release -p tract-ffi $CARGO_EXTRA 8 | SO=$(cargo build --message-format=json --release -p tract-ffi $CARGO_EXTRA | grep cdylib | jshon -e filenames -e 0 -u) 9 | SO_PATH=$(dirname $SO) 10 | export TRACT_DYLIB_SEARCH_PATH=$SO_PATH 11 | export LD_LIBRARY_PATH=$SO_PATH 12 | 13 | cd $(dirname $(realpath $0)) 14 | cargo test $CARGO_EXTRA 15 | -------------------------------------------------------------------------------- /api/proxy/sys/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-proxy-sys" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/sonos/tract" 8 | keywords = [ "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | rust-version = "1.85" 13 | include = [ "Cargo.toml", "src/**/*.rs", "LICENSE*", "build.rs", "tract.h" ] 14 | 15 | [build-dependencies] 16 | bindgen = "0.71.0" 17 | -------------------------------------------------------------------------------- /api/proxy/sys/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | 4 | fn main() { 5 | println!("cargo:rerun-if-env-changed=TRACT_DYLIB_SEARCH_PATH"); 6 | if let Ok(path) = std::env::var("TRACT_DYLIB_SEARCH_PATH") { 7 | println!("cargo:rustc-link-search={path}"); 8 | } 9 | println!("cargo:rustc-link-lib=tract"); 10 | 11 | let bindings = bindgen::Builder::default() 12 | .header("tract.h") 13 | .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) 14 | .generate() 15 | .expect("Unable to generate bindings"); 16 | 17 | let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); 18 | bindings.write_to_file(out_path.join("bindings.rs")).expect("Couldn't write bindings!"); 19 | } 20 | -------------------------------------------------------------------------------- /api/proxy/sys/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_upper_case_globals)] 2 | #![allow(non_camel_case_types)] 3 | #![allow(non_snake_case)] 4 | #![allow(improper_ctypes)] 5 | #![allow(deref_nullptr)] 6 | #![allow(clippy::redundant_static_lifetimes)] 7 | #![allow(clippy::useless_transmute)] 8 | 9 | include!(concat!(env!("OUT_DIR"), "/bindings.rs")); 10 | -------------------------------------------------------------------------------- /api/proxy/tests/mobilenet.rs: -------------------------------------------------------------------------------- 1 | use tract_api::*; 2 | use tract_proxy::*; 3 | 4 | include!("../../tests/mobilenet/mod.rs"); 5 | -------------------------------------------------------------------------------- /api/py/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.so 3 | *.egg-info 4 | *.onnx 5 | build 6 | mobilenet_v2_1.0.onnx.nnef.tgz 7 | rust-workspace 8 | dist 9 | 10 | -------------------------------------------------------------------------------- /api/py/MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft rust-workspace 2 | graft docs 3 | -------------------------------------------------------------------------------- /api/py/docs/fact.md: -------------------------------------------------------------------------------- 1 | # Facts 2 | 3 | ::: tract.fact 4 | -------------------------------------------------------------------------------- /api/py/docs/inference_model.md: -------------------------------------------------------------------------------- 1 | # Inference model 2 | 3 | ::: tract.inference_model 4 | -------------------------------------------------------------------------------- /api/py/docs/model.md: -------------------------------------------------------------------------------- 1 | # Model (aka Typed Model) 2 | 3 | ::: tract.model 4 | -------------------------------------------------------------------------------- /api/py/docs/nnef.md: -------------------------------------------------------------------------------- 1 | # NNEF 2 | 3 | ::: tract.nnef 4 | -------------------------------------------------------------------------------- /api/py/docs/onnx.md: -------------------------------------------------------------------------------- 1 | # ONNX 2 | 3 | ::: tract.onnx 4 | -------------------------------------------------------------------------------- /api/py/docs/runnable.md: -------------------------------------------------------------------------------- 1 | # Runnable model 2 | 3 | ::: tract.runnable 4 | -------------------------------------------------------------------------------- /api/py/docs/value.md: -------------------------------------------------------------------------------- 1 | # Value 2 | 3 | ::: tract.value 4 | -------------------------------------------------------------------------------- /api/py/grace_hopper_1x3x224x244.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/api/py/grace_hopper_1x3x224x244.npy -------------------------------------------------------------------------------- /api/py/requirements-docs.txt: -------------------------------------------------------------------------------- 1 | mkdocs 2 | mkdocs-material 3 | mkdocstrings[python]==0.27.0 4 | mike 5 | -------------------------------------------------------------------------------- /api/py/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.26.4 2 | setuptools==78.1.1 3 | setuptools_rust==1.9.0 4 | toml==0.10.2 5 | -------------------------------------------------------------------------------- /api/py/tract/bindings.py: -------------------------------------------------------------------------------- 1 | from ctypes import * 2 | from pathlib import Path 3 | 4 | if len(list(Path(__file__).parent.glob("*.so"))) > 0: 5 | dylib_path = list(Path(__file__).parent.glob("*.so"))[0] 6 | elif len(list(Path(__file__).parent.glob("*.pyd"))) > 0: 7 | dylib_path = list(Path(__file__).parent.glob("*.pyd"))[0] 8 | else: 9 | raise TractError("Can not find dynamic library") 10 | 11 | lib = cdll.LoadLibrary(str(dylib_path)) 12 | 13 | lib.tract_version.restype = c_char_p 14 | lib.tract_get_last_error.restype = c_char_p 15 | lib.tract_free_cstring.restype = None 16 | 17 | class TractError(Exception): 18 | pass 19 | 20 | def check(err): 21 | if err != 0: 22 | raise TractError(str(lib.tract_get_last_error(), "utf-8")) 23 | 24 | -------------------------------------------------------------------------------- /api/rs/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /api/rs/tests/mobilenet.rs: -------------------------------------------------------------------------------- 1 | use tract_api::*; 2 | use tract_rs::*; 3 | 4 | include!("../../tests/mobilenet/mod.rs"); 5 | -------------------------------------------------------------------------------- /api/tests/grace_hopper_3_224_224.f32.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/api/tests/grace_hopper_3_224_224.f32.raw -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_body/tract-body-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_body/tract-body-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_body/tract-body-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_body/tract-body-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_body/tract-body-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_body/tract-body-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_icon/tract-icon-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_icon/tract-icon-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_icon/tract-icon-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_icon/tract-icon-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_icon/tract-icon-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_icon/tract-icon-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_logo/tract-horizontal-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_logo/tract-horizontal-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_logo/tract-horizontal-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_logo/tract-horizontal-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_logo/tract-horizontal-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_logo/tract-horizontal-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_logo/tract-vertical-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_logo/tract-vertical-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_logo/tract-vertical-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_logo/tract-vertical-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/PNG_logo/tract-vertical-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/PNG_logo/tract-vertical-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-body-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-body-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-body-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-body-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-body-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-body-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-horizontal-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-horizontal-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-horizontal-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-horizontal-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-horizontal-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-horizontal-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-icon-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-icon-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-icon-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-icon-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-icon-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-icon-white.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-vertical-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-vertical-black.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-vertical-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-vertical-blue.png -------------------------------------------------------------------------------- /assets/tract-logo/PNG/tract-vertical-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/PNG/tract-vertical-white.png -------------------------------------------------------------------------------- /assets/tract-logo/post/tract-horizontal-white copy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/assets/tract-logo/post/tract-horizontal-white copy.png -------------------------------------------------------------------------------- /ci/tract-ci-minion/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-ci-minion" 3 | version = "0.20.7-pre" 4 | edition = "2024" 5 | 6 | [workspace] 7 | members = [] 8 | 9 | [dependencies] 10 | dirs = "3" 11 | fs2 = "0" 12 | rust-s3 = { version = "0.26.4", default-features = false, features = [ "rustls-tls" ] } 13 | serde = { version = "1.0", features = ["derive"] } 14 | toml = "0.5" 15 | anyhow = "1.0.42" 16 | log = "0.4.14" 17 | env_logger = "0.9.0" 18 | pipe = "0.4.0" 19 | flate2 = "1.0.20" 20 | tar = "0.4.35" 21 | daemonize = "0.5" 22 | libc = "0.2.119" 23 | wait-timeout = "0.2.0" 24 | lazy_static = "1.4.0" 25 | simple-signal = "1" 26 | -------------------------------------------------------------------------------- /ci/tract-ci-minion/minion.toml: -------------------------------------------------------------------------------- 1 | id = "tsar" 2 | workdir = "tmp/workdir" 3 | platform = "test" 4 | graphite = { host = "...", port = 2003, prefix = "tract" } 5 | 6 | # MINION_ID= 7 | # LOCKFILE=/tmp/minion-lock 8 | # PLATFORM=raspbian 9 | # GRAPHITE_HOST=graphite-proxy.snips.net 10 | # GRAPHITE_PORT=2003 11 | # GRAPHITE_PREFIX=tract 12 | # 13 | # S3PATH_TASKS=tract-ci-builds/tasks 14 | # S3PATH_LOGS=tract-ci-builds/logs 15 | # S3PATH_RESULTS=tract-ci-builds/logs 16 | # WORKDIR=$HOME/tract-minion 17 | # CACHEDIR=$WORKDIR/cache 18 | -------------------------------------------------------------------------------- /cli/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /cli/src/cost.rs: -------------------------------------------------------------------------------- 1 | use crate::TractResult; 2 | use tract_hir::internal::*; 3 | 4 | pub fn parse_costs(spec: &str) -> TractResult> { 5 | spec.split(',') 6 | .map(|spec| { 7 | let mut toks = spec.split('='); 8 | let name = toks.next().unwrap(); 9 | let n = toks.next().unwrap().parse::().unwrap(); 10 | let c = match name { 11 | "FMA(F32)" => Cost::FMA(f32::datum_type()), 12 | "Div(F32)" => Cost::Div(f32::datum_type()), 13 | "Buffer(F32)" => Cost::Buffer(f32::datum_type()), 14 | "Params(F32)" => Cost::Params(f32::datum_type()), 15 | _ => bail!("Unknown cost specifier {}", name), 16 | }; 17 | Ok((c, n)) 18 | }) 19 | .collect() 20 | } 21 | -------------------------------------------------------------------------------- /cli/src/model.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/cli/src/model.rs -------------------------------------------------------------------------------- /cli/src/plan_options.rs: -------------------------------------------------------------------------------- 1 | use tract_core::internal::*; 2 | 3 | pub fn plan_options_from_subcommand(sub_matches: &clap::ArgMatches) -> TractResult { 4 | let skip_order_opt_ram: bool = sub_matches.is_present("skip-order-opt-ram"); 5 | if skip_order_opt_ram { 6 | log::info!("Plan options: skip_order_opt_ram -> {skip_order_opt_ram:?}"); 7 | } 8 | Ok(PlanOptions { skip_order_opt_ram, ..PlanOptions::default() }) 9 | } -------------------------------------------------------------------------------- /core/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /core/src/macros.rs: -------------------------------------------------------------------------------- 1 | #[macro_export] 2 | macro_rules! dims { 3 | ($($dim:expr),*) => { 4 | ShapeFact::from(&[$(TDim::from($dim.clone())),*]) 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /core/src/ops/dummy.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::*; 2 | 3 | #[derive(Debug, Clone, new, Hash)] 4 | pub struct Dummy; 5 | 6 | impl Op for Dummy { 7 | fn name(&self) -> Cow { 8 | "Dummy".into() 9 | } 10 | 11 | op_as_typed_op!(); 12 | } 13 | 14 | 15 | 16 | impl EvalOp for Dummy { 17 | fn is_stateless(&self) -> bool { 18 | false 19 | } 20 | 21 | fn eval(&self, _inputs: TVec) -> TractResult> { 22 | bail!("eval() called on a Dummy op. This is a bug.") 23 | } 24 | } 25 | 26 | impl TypedOp for Dummy { 27 | as_op!(); 28 | 29 | fn output_facts(&self, _inputs: &[&TypedFact]) -> TractResult> { 30 | Ok(tvec!()) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /core/src/ops/matmul/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod de_block_quant; 2 | pub mod optimized; 3 | pub mod pack; 4 | pub mod quant; 5 | 6 | use crate::internal::*; 7 | 8 | pub fn output_type(input: DatumType) -> DatumType { 9 | if input.is_float() { 10 | input 11 | } else { 12 | i32::datum_type() 13 | } 14 | } 15 | 16 | #[derive(Clone, Debug, Hash, PartialEq, Eq)] 17 | pub enum ModePicker { 18 | Single, 19 | VecVsMat, 20 | } 21 | 22 | impl ModePicker { 23 | #[inline] 24 | pub fn pick(&self, n: usize) -> TractResult { 25 | match self { 26 | ModePicker::Single => Ok(0), 27 | ModePicker::VecVsMat => Ok((n > 1) as usize), 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/ops/memory/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod force_eval; 2 | pub mod load; 3 | pub mod store; 4 | -------------------------------------------------------------------------------- /core/test_data/test_data.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/core/test_data/test_data.cfg -------------------------------------------------------------------------------- /data/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /data/benches/tensor_from_datum.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate criterion; 3 | extern crate tract_data; 4 | 5 | use criterion::Criterion; 6 | use tract_data::internal::*; 7 | 8 | fn rank_4(c: &mut Criterion) { 9 | c.bench_function("rank_4", |b| { 10 | b.iter_with_setup( 11 | || { 12 | tract_ndarray::Array4::from_shape_simple_fn((256, 35, 35, 1), || 1.0f32) 13 | .permuted_axes([3, 2, 1, 0]) 14 | }, 15 | Tensor::from, 16 | ); 17 | }); 18 | } 19 | 20 | criterion_group!(benches, rank_4); 21 | criterion_main!(benches); 22 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # Tract internals documentation 2 | 3 | This kind of documentation does not tend to age well. Use with caution. 4 | 5 | * a [tract crates introduction](intro.md) 6 | * a [tract command line cookbook](cli-recipe.md) 7 | * [graph, model, node, op, facts](graph.md) 8 | -------------------------------------------------------------------------------- /doc/nnef/tract-resource.nnef: -------------------------------------------------------------------------------- 1 | # Extension `tract_resource` exposes NNEF fragments for accessing 2 | # resources files in NNEF folder or archive. 3 | # 4 | # Add `extension tract_resource` to `graph.nnef` 5 | 6 | 7 | # Access embedded resource by key 8 | fragment tract_resource_get( 9 | # Resource label to access 10 | label: string, 11 | # Key path in resource 12 | key: string 13 | ) -> (output: tensor); 14 | 15 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | *.onnx 2 | *.nnef.tgz 3 | *.tar.gz 4 | -------------------------------------------------------------------------------- /examples/face_detection_yolov8onnx_example/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | *.onnx 3 | *.pt 4 | *.pt.1 5 | -------------------------------------------------------------------------------- /examples/face_detection_yolov8onnx_example/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "face_detection_yolov8onnx_example" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | anyhow.workspace = true 8 | clap = { version = "4.5.9", features = ["derive"] } 9 | image.workspace = true 10 | tract-onnx.workspace = true 11 | -------------------------------------------------------------------------------- /examples/face_detection_yolov8onnx_example/README.md: -------------------------------------------------------------------------------- 1 | # tract onnx face detection demo yolov8 2 | face deteciton demo using yolov8 , using models converted from [derronqi's repo](https://github.com/derronqi/yolov8-face) 3 | 4 | # getting the models 5 | 1. you can get the model and convert them yourself from [here](https://github.com/derronqi/yolov8-face) , you can follow conversion instructions [here](https://docs.ultralytics.com/integrations/onnx/) 6 | 2. you can get it preconverted from 7 | [google drive](https://drive.google.com/file/d/1PYAG1ypAuwh_rDROaUF0OdLmBqOefBGL/view?usp=sharing) 8 | 9 | # to use 10 | run `cargo run -- --input-image /path/to/image --weights /path/to/weights` 11 | -------------------------------------------------------------------------------- /examples/face_detection_yolov8onnx_example/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | # download pre-exported onnx model 5 | wget -Nq "https://tract-ci-builds.s3.amazonaws.com/model/yolov8n-face.onnx" 6 | 7 | # on win/linux 8 | cargo run -- --input-image grace_hopper.jpg --weights yolov8n-face.onnx 9 | 10 | wasmtime -V || curl https://wasmtime.dev/install.sh -sSf | bash # install wasmtime 11 | PATH=$PATH:$HOME/.wasmtime/bin 12 | rustup target install wasm32-wasip1 13 | cargo build --target wasm32-wasip1 --release 14 | wasmtime --dir . ../../target/wasm32-wasip1/release/face_detection_yolov8onnx_example.wasm --input-image grace_hopper.jpg --weights yolov8n-face.onnx 15 | 16 | rm yolov8n-face.onnx 17 | -------------------------------------------------------------------------------- /examples/face_detection_yolov8onnx_example/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/face_detection_yolov8onnx_example/grace_hopper.jpg -------------------------------------------------------------------------------- /examples/face_similarity_arcface_onnx/.gitignore: -------------------------------------------------------------------------------- 1 | *.onnx 2 | -------------------------------------------------------------------------------- /examples/face_similarity_arcface_onnx/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "face_similarity_arcface_onnx" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | anyhow.workspace = true 8 | clap = { version = "4.5.9", features = ["derive"] } 9 | image.workspace = true 10 | tract-onnx.workspace = true 11 | tract-core.workspace = true 12 | 13 | -------------------------------------------------------------------------------- /examples/face_similarity_arcface_onnx/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | # download pre-exported onnx model 6 | wget -Nq "https://tract-ci-builds.s3.amazonaws.com/model/yolov8n-face.onnx" 7 | wget -Nq "https://tract-ci-builds.s3.amazonaws.com/model/arcfaceresnet100-8.onnx" 8 | 9 | # on win/linux 10 | cargo run --release -- --face1 grace_hopper.jpeg --face2 grace_hopper2.jpeg 11 | 12 | wasmtime -V || curl https://wasmtime.dev/install.sh -sSf | bash # install wasmtime 13 | PATH=$PATH:$HOME/.wasmtime/bin 14 | rustup target install wasm32-wasip1 15 | cargo build --target wasm32-wasip1 --release 16 | wasmtime --dir . ../../target/wasm32-wasip1/release/face_similarity_arcface_onnx.wasm --face1 grace_hopper.jpeg --face2 grace_hopper2.jpeg 17 | 18 | rm yolov8n-face.onnx 19 | rm arcfaceresnet100-8.onnx 20 | -------------------------------------------------------------------------------- /examples/face_similarity_arcface_onnx/grace_hopper.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/face_similarity_arcface_onnx/grace_hopper.jpeg -------------------------------------------------------------------------------- /examples/face_similarity_arcface_onnx/grace_hopper2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/face_similarity_arcface_onnx/grace_hopper2.jpeg -------------------------------------------------------------------------------- /examples/keras-tract-tf2/.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | Anaconda3-2022.10-Linux-x86_64.sh 3 | anaconda/ 4 | venv 5 | -------------------------------------------------------------------------------- /examples/keras-tract-tf2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "keras-tract-tf2" 3 | version = "0.20.7-pre" 4 | authors = ["Matthew Alhonte "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | 12 | tract-core.workspace = true 13 | tract-onnx.workspace = true 14 | ndarray-npy.workspace = true 15 | 16 | -------------------------------------------------------------------------------- /examples/keras-tract-tf2/README.md: -------------------------------------------------------------------------------- 1 | A simple example of training a Tensorflow model with Python, check the model with tract python API, then [loading it into `tract`](src/main.rs) and compare predictions. 2 | 3 | # Python side training 4 | 5 | Setup [environment](requirements.txt). 6 | 7 | ``` 8 | pip install -r requirements.txt 9 | ``` 10 | 11 | [Train](example.py) a model, export it to ONNX along with a input and output example. 12 | 13 | ``` 14 | python example.py 15 | ``` 16 | 17 | (Outputs are commited to git, you don't need to run the python step at all.) 18 | 19 | # Rust side inference 20 | 21 | [Run](src/main.rs) the model and double check the output. 22 | 23 | ``` 24 | cargo run 25 | ``` 26 | -------------------------------------------------------------------------------- /examples/keras-tract-tf2/ci.sh.nope: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | 6 | sudo apt-get update 7 | 8 | # Install required libraries 9 | # sudo apt-get install -y libgl1-mesa-glx libegl1-mesa libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6 python3 python3-venv 10 | sudo apt-get install -y libxrandr2 libxss1 libxcursor1 libxcomposite1 libxi6 libxtst6 python3 python3-venv 11 | 12 | python3 -m venv venv 13 | . venv/bin/activate 14 | 15 | 16 | pip install -r requirements.txt 17 | python example.py 18 | cargo run 19 | -------------------------------------------------------------------------------- /examples/keras-tract-tf2/example.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/keras-tract-tf2/example.onnx -------------------------------------------------------------------------------- /examples/keras-tract-tf2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/keras-tract-tf2/io.npz -------------------------------------------------------------------------------- /examples/keras-tract-tf2/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow==2.18.0 2 | numpy==1.26.0 3 | tf2onnx==1.16.1 4 | tract==0.21.6 5 | -------------------------------------------------------------------------------- /examples/nnef-dump-mobilenet-v2/.gitignore: -------------------------------------------------------------------------------- 1 | mobilenet* 2 | -------------------------------------------------------------------------------- /examples/nnef-dump-mobilenet-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example-dump-nnef-mobilenet-v2" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | 11 | tract-nnef.workspace = true 12 | -------------------------------------------------------------------------------- /examples/nnef-dump-mobilenet-v2/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | wget -nc -q https://s3.amazonaws.com/tract-ci-builds/model/mobilenet_v2_1.4_224.tgz 6 | tar zxf mobilenet_v2_1.4_224.tgz 7 | 8 | cargo run -p tract -- mobilenet_v2_1.4_224_frozen.pb -i 1,224,224,3,f32 dump --nnef mobilenet.nnef.tgz 9 | cargo run 10 | -------------------------------------------------------------------------------- /examples/nnef-dump-mobilenet-v2/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/nnef-dump-mobilenet-v2/grace_hopper.jpg -------------------------------------------------------------------------------- /examples/nnef-mobilenet-v2/.gitignore: -------------------------------------------------------------------------------- 1 | mobilenet_v2_1.0.onnx.nnef.tgz 2 | mobilenetv2-7.onnx 3 | 4 | -------------------------------------------------------------------------------- /examples/nnef-mobilenet-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example-nnef-mobilenet-v2" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | 11 | tract-nnef.workspace = true 12 | -------------------------------------------------------------------------------- /examples/nnef-mobilenet-v2/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | wget -q https://sfo2.digitaloceanspaces.com/nnef-public/mobilenet_v2_1.0.onnx.nnef.tgz -O mobilenet_v2_1.0.onnx.nnef.tgz 6 | 7 | cargo run 8 | rm mobilenet_v2_1.0.onnx.nnef.tgz 9 | -------------------------------------------------------------------------------- /examples/nnef-mobilenet-v2/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/nnef-mobilenet-v2/grace_hopper.jpg -------------------------------------------------------------------------------- /examples/onnx-mobilenet-v2/.gitignore: -------------------------------------------------------------------------------- 1 | mobilenetv2-7.onnx 2 | -------------------------------------------------------------------------------- /examples/onnx-mobilenet-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example-onnx-mobilenet-v2" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | tract-onnx.workspace = true 11 | -------------------------------------------------------------------------------- /examples/onnx-mobilenet-v2/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | [ -e mobilenetv2-7.onnx ] || \ 6 | wget -q https://s3.amazonaws.com/tract-ci-builds/tests/mobilenetv2-7.onnx -O mobilenetv2-7.onnx 7 | 8 | # on win/linux 9 | cargo run 10 | # on wasm 11 | wasmtime -V || curl https://wasmtime.dev/install.sh -sSf | bash # install wasmtime 12 | PATH=$PATH:$HOME/.wasmtime/bin 13 | rustup target install wasm32-wasip1 14 | cargo build --target wasm32-wasip1 15 | wasmtime --dir . ../../target/wasm32-wasip1/debug/example-onnx-mobilenet-v2.wasm 16 | 17 | rm mobilenetv2-7.onnx 18 | -------------------------------------------------------------------------------- /examples/onnx-mobilenet-v2/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/onnx-mobilenet-v2/grace_hopper.jpg -------------------------------------------------------------------------------- /examples/pytorch-albert-v2/.gitignore: -------------------------------------------------------------------------------- 1 | albert 2 | -------------------------------------------------------------------------------- /examples/pytorch-albert-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pytorch-albert-v2" 3 | version = "0.20.7-pre" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | anstyle.workspace = true 10 | anstyle-parse.workspace = true 11 | anstyle-query.workspace = true 12 | ndarray.workspace = true 13 | tokenizers.workspace = true 14 | clap_builder = { version = "4.4" } 15 | clap_lex = { version = "0.7" } 16 | tract-onnx.workspace = true 17 | -------------------------------------------------------------------------------- /examples/pytorch-albert-v2/README.md: -------------------------------------------------------------------------------- 1 | A simple example of exporting a [transformer](https://huggingface.co/docs/transformers/index) model with Python, then loading it into tract to make predictions. 2 | 3 | # To Use 4 | 5 | First export the pre-trained transformer model using Python and PyTorch 6 | 7 | ``` shell 8 | python export.py 9 | ``` 10 | 11 | the exported model and tokenizer are saved in `./albert`. Then load the model into tract and make prediction. 12 | 13 | ``` rust 14 | cargo run --release 15 | ``` 16 | 17 | The output for input sentence "Paris is the [MASK] of France" should look like 18 | 19 | ``` text 20 | Result: Some("▁capital") 21 | ``` 22 | -------------------------------------------------------------------------------- /examples/pytorch-albert-v2/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | sudo apt-get install -y python3-virtualenv 5 | virtualenv venv 6 | . ./venv/bin/activate 7 | pip install -q torch transformers onnx accelerate 8 | python export.py 9 | cargo run --release 10 | rm -rf venv 11 | -------------------------------------------------------------------------------- /examples/pytorch-resnet/.gitignore: -------------------------------------------------------------------------------- 1 | resnet.onnx 2 | -------------------------------------------------------------------------------- /examples/pytorch-resnet/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example-pytorch-resnet" 3 | version = "0.20.7-pre" 4 | authors = ["Teddy Koker "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | image.workspace = true 12 | tract-onnx.workspace = true 13 | -------------------------------------------------------------------------------- /examples/pytorch-resnet/README.md: -------------------------------------------------------------------------------- 1 | Example of exporting a PyTorch to ONNX format, then performing inference with 2 | tract. 3 | 4 | **Export Model:** 5 | ```bash 6 | python export.py 7 | ``` 8 | 9 | **Inference on `elephants.jpg`:** 10 | ``` 11 | cargo run 12 | result: Some((22.08386, 102)) 13 | ``` 14 | 15 | Predicts class 102 (`tusker`). 16 | -------------------------------------------------------------------------------- /examples/pytorch-resnet/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | sudo apt-get install -y python3-virtualenv 5 | virtualenv venv 6 | . ./venv/bin/activate 7 | pip install -q torch torchvision onnx 8 | python export.py 9 | cargo run 10 | rm -rf venv 11 | -------------------------------------------------------------------------------- /examples/pytorch-resnet/elephants.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/pytorch-resnet/elephants.jpg -------------------------------------------------------------------------------- /examples/pytorch-resnet/export.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | 4 | 5 | def export(): 6 | dummy_input = torch.randn(1, 3, 224, 224) 7 | model = torchvision.models.resnet18(pretrained=True) 8 | torch.onnx.export(model, dummy_input, "resnet.onnx") 9 | 10 | 11 | if __name__ == "__main__": 12 | export() 13 | -------------------------------------------------------------------------------- /examples/pytorch-resnet/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==2.6.0 2 | torchvision==0.4.2 3 | -------------------------------------------------------------------------------- /examples/tensorflow-mobilenet-v2/.gitignore: -------------------------------------------------------------------------------- 1 | mobilenet_v2_1.4_224* 2 | target 3 | -------------------------------------------------------------------------------- /examples/tensorflow-mobilenet-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example-tensorflow-mobilenet-v2" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | tract-tensorflow.workspace = true 11 | -------------------------------------------------------------------------------- /examples/tensorflow-mobilenet-v2/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | wget -nc -q https://s3.amazonaws.com/tract-ci-builds/model/mobilenet_v2_1.4_224.tgz 6 | tar zxf mobilenet_v2_1.4_224.tgz 7 | cargo run 8 | rm -rf mobilenet* 9 | -------------------------------------------------------------------------------- /examples/tensorflow-mobilenet-v2/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/tensorflow-mobilenet-v2/grace_hopper.jpg -------------------------------------------------------------------------------- /examples/tflite-mobilenet-v3/.gitignore: -------------------------------------------------------------------------------- 1 | mobilenet_v2_1.4_224* 2 | target 3 | -------------------------------------------------------------------------------- /examples/tflite-mobilenet-v3/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example-tflite-mobilenet-v3" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | tract-tflite.workspace = true 11 | -------------------------------------------------------------------------------- /examples/tflite-mobilenet-v3/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | wget -q "https://tract-ci-builds.s3.amazonaws.com/model/mobilenet_v3_small_100_224.tflite" -O mobilenet_v3_small_100_224.tflite 6 | cargo run 7 | rm -rf mobilenet* 8 | -------------------------------------------------------------------------------- /examples/tflite-mobilenet-v3/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/examples/tflite-mobilenet-v3/grace_hopper.jpg -------------------------------------------------------------------------------- /extra/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-extra" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/snipsco/tract" 8 | keywords = [ "TensorFlow", "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | rust-version = "1.85" 13 | 14 | [badges] 15 | maintenance = { status = "actively-developed" } 16 | 17 | [dependencies] 18 | tract-nnef.workspace = true 19 | tract-pulse.workspace = true 20 | 21 | [dev-dependencies] 22 | criterion.workspace = true 23 | env_logger.workspace = true 24 | lazy_static.workspace = true 25 | proptest.workspace = true 26 | approx.workspace = true 27 | -------------------------------------------------------------------------------- /extra/src/lib.rs: -------------------------------------------------------------------------------- 1 | use tract_nnef::internal::*; 2 | 3 | mod exp_unit_norm; 4 | 5 | pub trait WithTractExtra { 6 | fn enable_tract_extra(&mut self); 7 | fn with_tract_extra(self) -> Self; 8 | } 9 | 10 | impl WithTractExtra for tract_nnef::framework::Nnef { 11 | fn enable_tract_extra(&mut self) { 12 | self.enable_tract_core(); 13 | self.registries.push(tract_extra_registry()); 14 | } 15 | 16 | fn with_tract_extra(mut self) -> Self { 17 | self.enable_tract_extra(); 18 | self 19 | } 20 | } 21 | 22 | pub fn tract_extra_registry() -> Registry { 23 | let mut reg = Registry::new("tract_extra"); 24 | exp_unit_norm::register(&mut reg); 25 | reg 26 | } 27 | 28 | pub fn register_pulsifiers() { 29 | let _ = tract_extra_registry(); 30 | } 31 | -------------------------------------------------------------------------------- /gpu/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod device; 2 | pub mod fact; 3 | pub mod memory; 4 | pub mod rewrite_rules; 5 | pub mod session_handler; 6 | pub mod sync; 7 | pub mod tensor; 8 | pub mod utils; 9 | -------------------------------------------------------------------------------- /gpu/src/memory/mod.rs: -------------------------------------------------------------------------------- 1 | mod pool; 2 | mod schema; 3 | 4 | pub use pool::DeviceMemoryPool; 5 | pub use schema::{DeviceMemSchema, DeviceResolvedMemSchema}; 6 | -------------------------------------------------------------------------------- /harness/core-proptest-pulse/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "core-proptest-pulse" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | tract-core.workspace = true 10 | tract-pulse.workspace = true 11 | 12 | [dev-dependencies] 13 | log.workspace = true 14 | proptest.workspace = true 15 | env_logger.workspace = true 16 | -------------------------------------------------------------------------------- /harness/lstm-proptest-onnx-vs-tf/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lstm-proptest-onnx-vs-tf" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | proptest.workspace = true 10 | tract-tensorflow.workspace = true 11 | tract-onnx.workspace = true 12 | -------------------------------------------------------------------------------- /harness/nnef-inceptionv3/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nnef-inceptionv3" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | flate2.workspace = true 10 | image.workspace = true 11 | tract-core.workspace = true 12 | tract-nnef.workspace = true 13 | 14 | [dev-dependencies] 15 | dinghy-test.workspace = true 16 | env_logger.workspace = true 17 | log.workspace = true 18 | -------------------------------------------------------------------------------- /harness/nnef-inceptionv3/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR=`dirname $0` 4 | 5 | $MY_DIR/../../.travis/cache_file.sh inception_v3.tfpb.nnef.tgz imagenet_slim_labels.txt 6 | -------------------------------------------------------------------------------- /harness/nnef-inceptionv3/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-inceptionv3/grace_hopper.jpg -------------------------------------------------------------------------------- /harness/nnef-test-cases/.gitignore: -------------------------------------------------------------------------------- 1 | found 2 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-bias/expected: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | fragment tract_core_properties( 4 | ) -> (properties: (string, tensor)[]) 5 | { 6 | properties = [("tract_nnef_ser_version", "0.18.3-pre"), ("tract_nnef_format_version", "beta1")]; 7 | } 8 | 9 | graph network(input) -> (output) { 10 | input = external(shape = [1, 1, 3]); 11 | kernel1 = [[[0.0, 0.0, 0.0]], [[0.0, 0.0, 0.0]]]; 12 | output_bias_rm_0 = [1.0, 1.0]; 13 | output_conv = conv(input, kernel1, output_bias_rm_0, dilation = [1], stride = [1], border = "constant", groups = 1, padding = [(0, 0)]); 14 | output = output_conv; 15 | } 16 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-bias/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph check_bias_loading(input) -> (output) 4 | { 5 | input = external(shape = [1, 1, 3]); 6 | kernel1 = [[[0.0, 0.0, 0.0]], [[0.0, 0.0, 0.0]]]; 7 | bias1 = [[[1.0, 1.0]]]; 8 | output = conv(input, kernel1, bias1, padding = [(0, 0)], border = 'constant', stride = [1], dilation = [1]); 9 | } 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-bias/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . dump -q --nnef-graph found 9 | 10 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 11 | perl -pi -e "s/$version/0.18.3-pre/" found 12 | 13 | diff -u expected found 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv2d/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv2d/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv2d/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv2d/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv2d/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel1/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_base_kernel1/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel1/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_base_kernel1/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel1/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel3/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_base_kernel3/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel3/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_base_kernel3/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel3/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel9/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_base_kernel9/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel9/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_base_kernel9/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_base_kernel9/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_dilation2/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation2/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_dilation2/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation2/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation4/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_dilation4/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation4/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_dilation4/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation4/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation8/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_dilation8/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation8/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_dilation8/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_dilation8/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_groups2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_groups2/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_groups2/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_groups2/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_groups2/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_groups4/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_groups4/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_groups4/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_groups4/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_groups4/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_insize128/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_insize128/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_insize128/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_insize128/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_insize128/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_insize64/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_insize64/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_insize64/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_insize64/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_insize64/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_stride2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_stride2/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_stride2/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_stride2/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_stride2/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_stride3/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_stride3/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_stride3/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/conv-q40/conv_stride3/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-q40/conv_stride3/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN --nnef-tract-core model.nnef.tgz -O run --approx very --input-from-bundle io.npz --assert-output-bundle io.npz 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-with-batch/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph net_2022_04_13T13_03_40(input_0) -> (output_0) 4 | { 5 | input_0 = external(shape = [2, 1, 3]); 6 | v3_weight = [[[0.0, 0.0, 0.0],[0.0,0.0,0.0],[0.0,0.0,0.0]]]; 7 | v2 = linear(input_0, v3_weight); 8 | output_0 = add(v2, input_0); 9 | } 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/conv-with-batch/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN -O . compare --allow-random-input --stage declutter 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/dyn_slice/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core dump -q 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/fixed_roll/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph network(input_0) -> (output_0) 6 | { 7 | input_0 = tract_core_external(shape = [1, 2, 3], datum_type = 'f32'); 8 | output_0_roll_l0_p1 = slice(input_0, axes = [1], begin = [1], end = [2], stride = [1]); 9 | output_0_roll_l0_p2 = slice(input_0, axes = [1], begin = [0], end = [1], stride = [1]); 10 | output_0_roll_0 = concat([output_0_roll_l0_p1, output_0_roll_l0_p2], axis = 1); 11 | output_0_roll_l1_p1 = slice(output_0_roll_0, axes = [2], begin = [2], end = [3], stride = [1]); 12 | output_0_roll_l1_p2 = slice(output_0_roll_0, axes = [2], begin = [0], end = [2], stride = [1]); 13 | output_0 = concat([output_0_roll_l1_p1, output_0_roll_l1_p2], axis = 2); 14 | } 15 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/fixed_roll/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/fixed_roll/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/fixed_roll/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | # Check result is as expected 9 | # bug appear only if model optimized 10 | $TRACT_RUN --nnef-tract-core . -O run --input-from-bundle io.npz --assert-output-bundle io.npz 11 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/load-store/expected: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | fragment tract_core_properties( 6 | ) -> (properties: (string, tensor)[]) 7 | { 8 | properties = [("tract_nnef_ser_version", "0.18.3-pre"), ("tract_nnef_format_version", "beta1")]; 9 | } 10 | 11 | graph network(input) -> (output) { 12 | input = external(shape = [1, 2, 2]); 13 | default_state = [[[0.0, 0.0], [0.0, 0.0]]]; 14 | state = tract_core_load(default_state, id = "state-0"); 15 | v2_linear_output = matmul(input, state, transposeA = false, transposeB = true); 16 | new_state_1 = add(v2_linear_output, v2_linear_output); 17 | dummy = tract_core_store(input = v2_linear_output, state = new_state_1, id = "state-0"); 18 | output = tract_core_force_eval([v2_linear_output, dummy], slots = [1]); 19 | } 20 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/load-store/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph nnet(input) -> (output) 6 | { 7 | input = external(shape = [1, 2, 2]); 8 | default_state = [[[0.0, 0.0], [0.0, 0.0]]]; 9 | state = tract_core_load(default_state, id = "state-0"); 10 | v2 = linear(input, state); 11 | new_state = add(v2, v2); 12 | dummy = tract_core_store(input = v2, state = new_state, id = "state-0"); 13 | output = tract_core_force_eval([v2, dummy], slots = [1]); 14 | } 15 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/load-store/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core dump -q --nnef-graph found 9 | 10 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 11 | perl -pi -e "s/$version/0.18.3-pre/" found 12 | 13 | diff -u expected found 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/memory-arena/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ `uname` = "Darwin" ] && ! ( sysctl -n machdep.cpu.brand_string | grep -q "(Virtual)" ) 4 | then 5 | 6 | ROOT=$(dirname $(realpath $0))/../../.. 7 | . $ROOT/.travis/ci-system-setup.sh 8 | 9 | cd `dirname $0` 10 | set -ex 11 | 12 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 13 | 14 | model=OpenELM-270M 15 | q=q40f16 16 | id="apple--$model-$q" 17 | generation=current 18 | nnef="llm/$generation/$id/$id.nnef.tgz" 19 | $CACHE_FILE $nnef 20 | 21 | $TRACT_RUN -v --nnef-tract-core --metal $MODELS/$nnef dump --set S=1024 --set P=0 --memory-arena found.json 22 | 23 | diff -u expected.json found.json 24 | else 25 | echo "Skipped (memory arena test requires apple hardware)" 26 | fi 27 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/pool-padding/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph check_full_padding(input) -> (output) 4 | { 5 | input = external(shape = [1, 1, 3, 3]); 6 | max = max_pool(input, size = [1, 1, 3, 3],padding = [(0, 0), (0, 0), (1, 1), (1, 1)], border = 'constant', stride = [1, 1, 1, 1], dilation = [1, 1, 1, 1]); 7 | add_value = [[[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]]]; 8 | output = add(max, add_value); 9 | } 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/pool-padding/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . dump -q 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/q40_linear_followed_slice/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph net_2024_08_07_bug_slice_bubble_up(input_0) -> (output_0) 6 | { 7 | input_0 = tract_core_external(shape = [10, 96], datum_type = 'f32'); 8 | linear__var0 = variable(label = 'linear__var0_q4_0', shape = [16, 96]); 9 | linear0 = linear(input_0, linear__var0, 0.0); 10 | slice_a = slice(linear0, axes = [1], begin = [0], end = [8], stride = [1]); 11 | slice_b = slice(linear0, axes = [1], begin = [8], end = [16], stride = [1]); 12 | output_0 = mul(slice_a, slice_b); 13 | } 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/q40_linear_followed_slice/linear__var0_q4_0.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/q40_linear_followed_slice/linear__var0_q4_0.dat -------------------------------------------------------------------------------- /harness/nnef-test-cases/q40_linear_followed_slice/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core dump -q 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/qmul/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph net_2024_06_26T18_12_43(input_0) -> (output_0) 6 | { 7 | input_0 = external(shape = [2, 3]); 8 | v2 = tract_core_cast(input_0); 9 | v1 = mul(v2, v2); 10 | output_0 = tract_core_cast(v1, to = 'f32'); 11 | } 12 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/qmul/graph.quant: -------------------------------------------------------------------------------- 1 | "v2": zero_point_linear_quantize(zero_point = 0, scale = 0.13444413244724274, bits = 8, signed = false, symmetric = false); 2 | "v1": zero_point_linear_quantize(scale = 4.611433506011963, zero_point = 0, bits = 8, signed = false, symmetric = false); 3 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/qmul/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/qmul/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/qmul/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | rm -rf found 9 | $TRACT_RUN --nnef-tract-core . run --input-from-bundle io.npz --steps --assert-output-bundle io.npz 10 | 11 | # version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 12 | # perl -pi -e "s/$version/0.16.10-pre/" found/graph.nnef 13 | # 14 | # diff expected found 15 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/range-slice-dyn-tile/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/range-slice-dyn-tile/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/range-slice-dyn-tile/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN model.nnef.tgz --nnef-tract-core dump -q 9 | $TRACT_RUN model.nnef.tgz --nnef-tract-core --nnef-cycle dump -q 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/reshape/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | extension tract_pulse_streaming_symbol; 5 | 6 | graph net_2024_06_18T12_07_11(input_0) -> (output_0) 7 | { 8 | input_0 = external(shape = [3, 1, 2]); 9 | foo = [1, 2]; 10 | output_0 = reshape(input_0, shape = [1, foo, 3]); 11 | } 12 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/reshape/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core dump -q 9 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/slice-over-slice-optim-loop/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph network(input) -> (o00, o01, o10, o11) 6 | { 7 | input = tract_core_external(shape = [2, 2], datum_type = 'f32'); 8 | o0 = slice(input, axes = [0], begin = [0], end = [1], stride = [1]); 9 | o1 = slice(input, axes = [0], begin = [1], end = [2], stride = [1]); 10 | o00 = slice(o0, axes = [1], begin = [0], end = [1], stride = [1]); 11 | o01 = slice(o0, axes = [1], begin = [1], end = [2], stride = [1]); 12 | o10 = slice(o1, axes = [1], begin = [0], end = [1], stride = [1]); 13 | o11 = slice(o1, axes = [1], begin = [1], end = [2], stride = [1]); 14 | } 15 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/slice-over-slice-optim-loop/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | # no timeout during recompilation 9 | $TRACT_RUN --version 10 | 11 | timeout 3 $TRACT_RUN --nnef-tract-core . 12 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-change-axis-1/expected: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | fragment tract_core_properties( 4 | ) -> (properties: (string, tensor)[]) 5 | { 6 | properties = [("tract_nnef_ser_version", "0.16.10-pre"), ("tract_nnef_format_version", "beta1")]; 7 | } 8 | 9 | graph network(input) -> (output) { 10 | input = external(shape = [2, 1, 3]); 11 | softmax1 = softmax(input, axes = [2]); 12 | output = softmax1; 13 | } 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-change-axis-1/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph add_softmax_rm(input) -> (output) 4 | { 5 | input = external(shape = [2, 1, 3]); 6 | tmp1 = unsqueeze( input, axes = [1] ); 7 | softmax1 = softmax( tmp1, axes = [3] ); 8 | output = squeeze( softmax1, axes = [1] ); 9 | } 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-change-axis-1/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . dump -q --nnef-graph found 9 | 10 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 11 | perl -pi -e "s/$version/0.16.10-pre/" found 12 | 13 | diff expected found 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-change-axis/expected: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | fragment tract_core_properties( 4 | ) -> (properties: (string, tensor)[]) 5 | { 6 | properties = [("tract_nnef_ser_version", "0.16.10-pre"), ("tract_nnef_format_version", "beta1")]; 7 | } 8 | 9 | graph network(input) -> (output) { 10 | input = external(shape = [2, 1, 3]); 11 | softmax1 = softmax(input, axes = [2]); 12 | output = softmax1; 13 | } 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-change-axis/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph rm_softmax_add(input) -> (output) 4 | { 5 | input = external(shape = [2, 1, 3]); 6 | tmp1 = squeeze( input, axes = [1] ); 7 | softmax1 = softmax( tmp1, axes = [1] ); 8 | output = unsqueeze( softmax1, axes = [1] ); 9 | } 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-change-axis/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . dump -q --nnef-graph found 9 | 10 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 11 | perl -pi -e "s/$version/0.16.10-pre/" found 12 | 13 | diff expected found 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-quant/expected/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | fragment tract_core_properties( 4 | ) -> (properties: (string, tensor)[]) 5 | { 6 | properties = [("tract_nnef_ser_version", "0.16.10-pre"), ("tract_nnef_format_version", "beta1")]; 7 | } 8 | 9 | graph network(input) -> (output) { 10 | input = external(shape = [2, 1, 3]); 11 | output = softmax(input, axes = [3]); 12 | } 13 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-quant/expected/graph.quant: -------------------------------------------------------------------------------- 1 | "input": zero_point_linear_quantize(zero_point = 128, scale = 0.007812500, bits = 8, signed = false, symmetric = false); 2 | "output": zero_point_linear_quantize(zero_point = 0, scale = 0.003906250, bits = 8, signed = false, symmetric = true); 3 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-quant/model/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph softmax_quant(input) -> (output) 4 | { 5 | input = external(shape = [2, 1, 3]); 6 | output = softmax( input, axes = [3] ); 7 | } 8 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-quant/model/graph.quant: -------------------------------------------------------------------------------- 1 | "input": zero_point_linear_quantize(zero_point = 128, scale = 0.0078125, bits = 8, signed = false, symmetric = false); 2 | "output": zero_point_linear_quantize(zero_point = 0, scale = 0.00390625, bits = 8, signed = false, symmetric = false); -------------------------------------------------------------------------------- /harness/nnef-test-cases/softmax/softmax-quant/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | rm -rf found 9 | $TRACT_RUN model dump -q --nnef-dir found 10 | 11 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 12 | perl -pi -e "s/$version/0.16.10-pre/" found/graph.nnef 13 | 14 | diff expected found 15 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/expected: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | fragment tract_core_properties( 6 | ) -> (properties: (string, tensor)[]) 7 | { 8 | properties = [("tract_nnef_ser_version", "0.18.3-pre"), ("tract_nnef_format_version", "beta1")]; 9 | } 10 | 11 | graph network(input) -> (output) { 12 | input = external(shape = [1, 1, 5]); 13 | nnet1 = tract_core_submodel(input, label = "nnet1"); 14 | nnet_1_out_quant = tract_core_cast(nnet1, to = "u8"); 15 | nnet2 = tract_core_submodel(nnet_1_out_quant, label = "nnet2"); 16 | nnet_2_out = tract_core_cast(nnet2, to = "f32"); 17 | output = nnet_2_out; 18 | } 19 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph nnet(input) -> (output) 6 | { 7 | input = external(shape = [1, 1, 5]); 8 | 9 | # First model is loaded from a tgz archive 10 | nnet_1_out = tract_core_submodel(input, label = 'nnet1'); 11 | 12 | # Second model is a quantized model and loaded from a subfolder 13 | nnet_1_out_quant = tract_core_cast(nnet_1_out); 14 | nnet_2_out_quant = tract_core_submodel(nnet_1_out_quant, label = 'nnet2'); 15 | nnet_2_out = tract_core_cast(nnet_2_out_quant, to = 'f32'); 16 | 17 | output = nnet_2_out; 18 | } 19 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/graph.quant: -------------------------------------------------------------------------------- 1 | "nnet_1_out_quant": zero_point_linear_quantize(zero_point = 0, scale = 0.064853981, bits = 8, signed = false, symmetric = true); 2 | "nnet_2_out_quant": zero_point_linear_quantize(zero_point = 0, scale = 0.064853981, bits = 8, signed = false, symmetric = true); 3 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/nnet2/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | fragment tract_core_properties( 4 | ) -> (properties: (string, tensor)[]) 5 | { 6 | properties = [("tract_nnef_format_version", "beta1"), ("tract_nnef_ser_version", "0.18.4"), ("tract_nnef_ser_version", "0.18.4"), ("tract_nnef_format_version", "beta1")]; 7 | } 8 | 9 | graph network( input ) -> ( output ) { 10 | input = external(shape = [1, 2, 3]); 11 | conv_weights = [[[0, 0, 0], [0, 0, 0]]]; 12 | conv_bias = [[[1]]]; 13 | output_conv = conv(input, conv_weights, conv_bias, dilation = [1], stride = [1], border = "constant", groups = 1, padding = [(0, 0)]); 14 | output = output_conv; 15 | } 16 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/nnet2/graph.quant: -------------------------------------------------------------------------------- 1 | "output": zero_point_linear_quantize(zero_point = 0, scale = 0.064853981, bits = 8, signed = false, symmetric = true); 2 | "conv_weights": zero_point_linear_quantize(zero_point = 128, scale = 0.000184978, bits = 8, signed = false, symmetric = false); 3 | "input": zero_point_linear_quantize(zero_point = 0, scale = 0.064853981, bits = 8, signed = false, symmetric = true); 4 | "output_conv": zero_point_linear_quantize(zero_point = 0, scale = 0.064853981, bits = 8, signed = false, symmetric = true); 5 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/nnet2/output_bias.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/submodel/nnet2/output_bias.dat -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/nnet2/output_weigths.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/submodel/nnet2/output_weigths.dat -------------------------------------------------------------------------------- /harness/nnef-test-cases/submodel/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core dump -q --nnef-graph found 9 | 10 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 11 | perl -pi -e "s/$version/0.18.3-pre/" found 12 | 13 | diff -u expected found 14 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/tdim-cmp/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_core; 4 | 5 | graph dummy_net(input_0) -> (v90) 6 | { 7 | input_0 = external(shape = [1, 6]); 8 | v37 = tract_core_range(0, 6, step = 1); 9 | v131 = add(v37, 1); 10 | v90 = lt(v37, v131); 11 | } 12 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/tdim-cmp/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core run --allow-random-input 9 | 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/test_manage_gru_states/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/test_manage_gru_states/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/test_manage_gru_states/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/test_manage_gru_states/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/test_manage_gru_states/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | # Check result is as expected 9 | # bug appear only if model optimized and input-fact-from-bundle 10 | $TRACT_RUN --nnef-tract-core ./model.nnef.tgz -O --input-facts-from-bundle ./io.npz run --input-from-bundle io.npz --assert-output-bundle io.npz 11 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/test_upcast_f32_attn/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/test_upcast_f32_attn/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/test_upcast_f32_attn/model.nnef.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/test_upcast_f32_attn/model.nnef.tgz -------------------------------------------------------------------------------- /harness/nnef-test-cases/test_upcast_f32_attn/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | # Check result is as expected 9 | # bug appear only if model optimized 10 | $TRACT_RUN --nnef-tract-core ./model.nnef.tgz -O run --input-from-bundle io.npz --assert-output-bundle io.npz --allow-float-casts 11 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/tile-with-tdim/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . --nnef-tract-core dump -q 9 | 10 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/uniform-mul/expected: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | fragment tract_core_properties( 4 | ) -> (properties: (string, tensor)[]) 5 | { 6 | properties = [("tract_nnef_ser_version", "0.18.3-pre"), ("tract_nnef_format_version", "beta1")]; 7 | } 8 | 9 | graph network(input) -> (output) { 10 | input = external(shape = [2, 2, 33]); 11 | a = [[[2.0], [3.0]]]; 12 | mul_1_0 = mul(input, a); 13 | b = [[[2.0], [3.0]], [[2.0], [3.0]]]; 14 | mul_2_0 = mul(input, b); 15 | output_1 = add(mul_1_0, mul_2_0); 16 | output = output_1; 17 | } 18 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/uniform-mul/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | graph check_uniform_mul_not_applied(input) -> (output) 4 | { 5 | input = external(shape = [2, 2, 33]); 6 | 7 | # a_shape = [1, 2, 1] 8 | a = [[[2.0], [3.0]]]; 9 | # This mul shouldn't be optimized with a mul_by_scalar as input_dim[0] != a_dim[0] 10 | mul_1 = mul(input, a); 11 | 12 | # b_shape = [2, 2, 1] 13 | b = [[[2.0], [3.0]], [[2.0], [3.0]]]; 14 | 15 | # This mul should be optimized with a mul_by_scalar as input_dim[0] == b_dim[0] + num_elements > 32 16 | mul_2 = mul(input, b); 17 | 18 | output = mul_1 + mul_2; 19 | } 20 | -------------------------------------------------------------------------------- /harness/nnef-test-cases/uniform-mul/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/nnef-test-cases/uniform-mul/io.npz -------------------------------------------------------------------------------- /harness/nnef-test-cases/uniform-mul/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | set -ex 5 | 6 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 7 | 8 | $TRACT_RUN . dump -q --nnef-graph found 9 | 10 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 11 | perl -pi -e "s/$version/0.18.3-pre/" found 12 | 13 | diff -u expected found 14 | 15 | # Check result is as expected 16 | $TRACT_RUN --nnef-tract-core . run --input-from-bundle io.npz --steps --assert-output-bundle io.npz 17 | -------------------------------------------------------------------------------- /harness/pre-optimized-graphes/.gitignore: -------------------------------------------------------------------------------- 1 | found 2 | -------------------------------------------------------------------------------- /harness/pre-optimized-graphes/hey_snips_v4_model17/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd `dirname $0` 4 | 5 | ROOT=$(dirname $(realpath $0))/../../.. 6 | . $ROOT/.travis/ci-system-setup.sh 7 | 8 | set -ex 9 | 10 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 11 | 12 | $CACHE_FILE hey_snips_v4_model17.pb 13 | $TRACT_RUN $MODELS/hey_snips_v4_model17.pb -i S,20,f32 --pulse 8 --nnef-tract-pulse --nnef-extended-identifier dump -q --nnef-graph found 14 | 15 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 16 | perl -pi -e "s/$version/0.19.3-pre/" found 17 | 18 | diff -u expected found 19 | -------------------------------------------------------------------------------- /harness/pre-optimized-graphes/mdl-en-2019-Q3-librispeech/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | cd `dirname $0` 6 | 7 | ROOT=$(dirname $(realpath $0))/../../.. 8 | . $ROOT/.travis/ci-system-setup.sh 9 | 10 | : ${TRACT_RUN:=cargo run -p tract $CARGO_OPTS --} 11 | 12 | $CACHE_FILE mdl-en-2019-Q3-librispeech.onnx 13 | $TRACT_RUN $MODELS/mdl-en-2019-Q3-librispeech.onnx -i S,40 --output-node output --pulse 24 --nnef-tract-pulse --nnef-extended-identifier dump -q --nnef-graph found 14 | 15 | version=`cargo metadata --format-version 1 | jq -r '.packages | map(select( (.name) == "tract-core") | .version) | .[] '` 16 | perl -pi -e "s/$version/0.19.3-pre/" found 17 | 18 | diff -u expected found 19 | -------------------------------------------------------------------------------- /harness/tf-inceptionv3/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tf-inceptionv3" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | tract-tensorflow.workspace = true 11 | 12 | [features] 13 | conform = [ "tract-tensorflow/conform" ] 14 | 15 | [dev-dependencies] 16 | criterion.workspace = true 17 | dinghy-test.workspace = true 18 | env_logger.workspace = true 19 | log.workspace = true 20 | 21 | [[bench]] 22 | harness = false 23 | name = "inceptionv3" 24 | -------------------------------------------------------------------------------- /harness/tf-inceptionv3/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR=`dirname $0` 4 | 5 | $MY_DIR/../../.travis/cache_file.sh inception_v3_2016_08_28_frozen.pb imagenet_slim_labels.txt 6 | -------------------------------------------------------------------------------- /harness/tf-inceptionv3/grace_hopper.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/harness/tf-inceptionv3/grace_hopper.jpg -------------------------------------------------------------------------------- /harness/tf-mobilenet-v2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tf-mobilenet-v2" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | tract-tensorflow.workspace = true 11 | 12 | [dev-dependencies] 13 | dinghy-test.workspace = true 14 | -------------------------------------------------------------------------------- /harness/tf-mobilenet-v2/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR=`dirname $0` 4 | 5 | $MY_DIR/../../.travis/cache_file.sh \ 6 | mobilenet_v2_1.4_224_frozen.pb \ 7 | imagenet_slim_labels.txt \ 8 | grace_hopper.jpg 9 | -------------------------------------------------------------------------------- /harness/tf-moz-deepspeech/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tf-moz-deepspeech" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | edition = "2024" 6 | license = "MIT OR Apache-2.0" 7 | 8 | [dependencies] 9 | log.workspace = true 10 | tract-tensorflow.workspace = true 11 | 12 | [dev-dependencies] 13 | env_logger.workspace = true 14 | -------------------------------------------------------------------------------- /harness/tf-moz-deepspeech/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR=`dirname $0` 4 | 5 | $MY_DIR/../../.travis/cache_file.sh deepspeech-0.4.1.pb deepspeech-0.4.1-smoketest.txt 6 | -------------------------------------------------------------------------------- /harness/tf-moz-deepspeech/smoketest-builder/.gitignore: -------------------------------------------------------------------------------- 1 | tensorflow 2 | DeepSpeech 3 | -------------------------------------------------------------------------------- /harness/tf-moz-deepspeech/smoketest-builder/run-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker run -ti -v `pwd`:/work -w /work deepspeech bash 4 | -------------------------------------------------------------------------------- /harness/tf-moz-deepspeech/smoketest-builder/run_me_inside_docker.sh: -------------------------------------------------------------------------------- 1 | 2 | set -ex 3 | 4 | cd /work/tensorflow 5 | bazel build \ 6 | --config=monolithic -c opt --copt=-O3 --copt="-D_GLIBCXX_USE_CXX11_ABI=0" \ 7 | --copt=-fvisibility=hidden \ 8 | //native_client:libdeepspeech.so \ 9 | //native_client:generate_trie 10 | 11 | cd /work/DeepSpeech/native_client 12 | make install 13 | ldconfig 14 | -------------------------------------------------------------------------------- /harness/tf-moz-deepspeech/smoketest-builder/setup-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ ! -e DeepSpeech ] 4 | then 5 | git clone https://github.com/mozilla/DeepSpeech.git 6 | fi 7 | 8 | if [ ! -e tensorflow ] 9 | then 10 | git clone https://github.com/mozilla/tensorflow.git 11 | (cd tensorflow ; 12 | git checkout r1.13 ; 13 | ln -s ../DeepSpeech/native_client . 14 | ) 15 | fi 16 | 17 | docker build --tag deepspeech . 18 | -------------------------------------------------------------------------------- /harness/tfl-mobilenet-v2-q/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tfl-mobilenet-v2-q" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2024" 7 | 8 | [dependencies] 9 | image.workspace = true 10 | tract-tflite.workspace = true 11 | 12 | [dev-dependencies] 13 | dinghy-test.workspace = true 14 | -------------------------------------------------------------------------------- /harness/tfl-mobilenet-v2-q/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR=`dirname $0` 4 | 5 | $MY_DIR/../../.travis/cache_file.sh \ 6 | mobilenetv2_ptq_single_img.tflite \ 7 | imagenet_slim_labels.txt \ 8 | grace_hopper.jpg 9 | -------------------------------------------------------------------------------- /hir/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-hir" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/snipsco/tract" 8 | keywords = [ "TensorFlow", "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | 13 | [badges] 14 | maintenance = { status = "actively-developed" } 15 | 16 | [dependencies] 17 | derive-new.workspace = true 18 | log.workspace = true 19 | 20 | tract-core.workspace = true 21 | 22 | [dev-dependencies] 23 | env_logger.workspace = true 24 | -------------------------------------------------------------------------------- /hir/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /hir/src/framework.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /hir/src/ops/cast.rs: -------------------------------------------------------------------------------- 1 | use tract_core::internal::*; 2 | use crate::infer::*; 3 | 4 | use tract_core::ops::cast::Cast; 5 | pub use tract_core::ops::cast::cast; 6 | 7 | impl InferenceRulesOp for Cast { 8 | fn rules<'r, 'p: 'r, 's: 'r>( 9 | &'s self, 10 | s: &mut Solver<'r>, 11 | inputs: &'p [TensorProxy], 12 | outputs: &'p [TensorProxy], 13 | ) -> InferenceResult { 14 | check_input_arity(inputs, 1)?; 15 | check_output_arity(outputs, 1)?; 16 | s.equals(&inputs[0].shape, &outputs[0].shape)?; 17 | s.equals(&outputs[0].datum_type, self.to)?; 18 | Ok(()) 19 | } 20 | 21 | as_op!(); 22 | to_typed!(); 23 | } 24 | -------------------------------------------------------------------------------- /hir/src/ops/cnn/mod.rs: -------------------------------------------------------------------------------- 1 | mod conv; 2 | mod pools; 3 | 4 | pub use conv::Conv; 5 | pub use pools::{HirMaxPool, HirSumPool}; 6 | pub use tract_core::ops::cnn::{PaddingSpec, PoolSpec}; 7 | -------------------------------------------------------------------------------- /hir/src/ops/dummy.rs: -------------------------------------------------------------------------------- 1 | use crate::infer::*; 2 | use crate::internal::*; 3 | 4 | pub use tract_core::ops::dummy::Dummy; 5 | 6 | impl InferenceRulesOp for Dummy { 7 | fn rules<'r, 'p: 'r, 's: 'r>( 8 | &'s self, 9 | _s: &mut Solver<'r>, 10 | _inputs: &'p [TensorProxy], 11 | _outputs: &'p [TensorProxy], 12 | ) -> InferenceResult { 13 | Ok(()) 14 | } 15 | 16 | as_op!(); 17 | to_typed!(); 18 | } 19 | -------------------------------------------------------------------------------- /hir/src/ops/identity.rs: -------------------------------------------------------------------------------- 1 | use crate::infer::*; 2 | use crate::internal::*; 3 | 4 | pub use tract_core::ops::identity::Identity; 5 | 6 | impl InferenceRulesOp for Identity { 7 | fn rules<'r, 'p: 'r, 's: 'r>( 8 | &'s self, 9 | s: &mut Solver<'r>, 10 | inputs: &'p [TensorProxy], 11 | outputs: &'p [TensorProxy], 12 | ) -> InferenceResult { 13 | check_input_arity(inputs, 1)?; 14 | check_output_arity(outputs, 1)?; 15 | s.equals(&inputs[0].datum_type, &outputs[0].datum_type)?; 16 | s.equals(&inputs[0].shape, &outputs[0].shape)?; 17 | Ok(()) 18 | } 19 | 20 | as_op!(); 21 | to_typed!(); 22 | } 23 | -------------------------------------------------------------------------------- /hir/src/ops/konst.rs: -------------------------------------------------------------------------------- 1 | use crate::infer::*; 2 | use crate::internal::*; 3 | 4 | pub use tract_core::ops::konst::*; 5 | 6 | impl InferenceRulesOp for Const { 7 | fn rules<'r, 'p: 'r, 's: 'r>( 8 | &'s self, 9 | s: &mut Solver<'r>, 10 | inputs: &'p [TensorProxy], 11 | outputs: &'p [TensorProxy], 12 | ) -> InferenceResult { 13 | check_input_arity(inputs, 0)?; 14 | check_output_arity(outputs, 1)?; 15 | s.equals(&outputs[0].value, self.val().clone().bex())?; 16 | Ok(()) 17 | } 18 | 19 | as_op!(); 20 | to_typed!(); 21 | } 22 | -------------------------------------------------------------------------------- /hir/src/ops/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod activations; 2 | pub mod array; 3 | pub mod binary; 4 | pub mod cast; 5 | pub mod cnn; 6 | pub mod downsample; 7 | pub mod dummy; 8 | pub mod element_wise; 9 | pub mod expandable; 10 | pub mod identity; 11 | pub mod konst; 12 | pub mod logic; 13 | pub use tract_core::ops::math; 14 | pub mod matmul; 15 | pub mod nn; 16 | pub use tract_core::ops::quant; 17 | 18 | pub mod scan; 19 | pub mod source; 20 | pub mod unimpl; 21 | 22 | -------------------------------------------------------------------------------- /hir/src/ops/nn/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod global_pools; 2 | pub mod layer_max; 3 | pub mod reduce; 4 | pub mod softmax; 5 | 6 | pub use global_pools::*; 7 | pub use layer_max::*; 8 | pub use reduce::{Reduce, Reducer}; 9 | pub use softmax::Softmax; 10 | 11 | pub use tract_core::ops::nn::{hard_swish, sigmoid, DataFormat}; 12 | -------------------------------------------------------------------------------- /hir/src/ops/quant.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/hir/src/ops/quant.rs -------------------------------------------------------------------------------- /hir/src/ops/unimpl.rs: -------------------------------------------------------------------------------- 1 | use crate::infer::*; 2 | use crate::internal::*; 3 | 4 | pub use tract_core::ops::unimpl::UnimplementedOp; 5 | 6 | impl InferenceRulesOp for UnimplementedOp { 7 | fn rules<'r, 'p: 'r, 's: 'r>( 8 | &'s self, 9 | _: &mut Solver<'r>, 10 | _: &'p [TensorProxy], 11 | _: &'p [TensorProxy], 12 | ) -> InferenceResult { 13 | Ok(()) 14 | } 15 | 16 | as_op!(); 17 | 18 | fn to_typed( 19 | &self, 20 | _source: &InferenceModel, 21 | _node: &InferenceNode, 22 | _target: &mut TypedModel, 23 | _mapping: &HashMap, 24 | ) -> TractResult> { 25 | bail!("Operator can not be made a TypedOp.") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /libcli/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::collapsible_if)] 2 | #[macro_use] 3 | extern crate log; 4 | 5 | pub mod annotations; 6 | pub mod display_params; 7 | pub mod draw; 8 | pub mod export; 9 | pub mod model; 10 | pub mod profile; 11 | pub mod tensor; 12 | pub mod terminal; 13 | pub mod time; 14 | -------------------------------------------------------------------------------- /linalg/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_mmm_f32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_min", op:"vmin.f32", mr:mr, from:from, to:to %} 4 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_max", op:"vmax.f32", mr:mr, from:from, to:to %} 5 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_mul", op:"vmul.f32", mr:mr, from:from, to:to %} 6 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_add", op:"vadd.f32", mr:mr, from:from, to:to %} 7 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub", op:"vsub.f32", mr:mr, from:from, to:to %} 8 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsub.f32", mr:mr, from:from, to:to, flipped: true%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_mmm_f32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_min", op:"vmin.f32", mr:mr, from:from, to:to %} 4 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_max", op:"vmax.f32", mr:mr, from:from, to:to %} 5 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_mul", op:"vmul.f32", mr:mr, from:from, to:to %} 6 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_add", op:"vadd.f32", mr:mr, from:from, to:to %} 7 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub", op:"vsub.f32", mr:mr, from:from, to:to %} 8 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsub.f32", mr:mr, from:from, to:to, flipped: true%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_mmm_i32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_min", op:"vmin.s32", mr:mr, from:from, to:to %} 4 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_max", op:"vmax.s32", mr:mr, from:from, to:to %} 5 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_mul", op:"vmul.s32", mr:mr, from:from, to:to %} 6 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_add", op:"vadd.s32", mr:mr, from:from, to:to %} 7 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub", op:"vsub.s32", mr:mr, from:from, to:to %} 8 | {% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsub.s32", mr:mr, from:from, to:to, flipped:true%} 9 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_mmm_i32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_min", op:"vmin.s32", mr:mr, from:from, to:to %} 4 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_max", op:"vmax.s32", mr:mr, from:from, to:to %} 5 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_mul", op:"vmul.s32", mr:mr, from:from, to:to %} 6 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_add", op:"vadd.s32", mr:mr, from:from, to:to %} 7 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub", op:"vsub.s32", mr:mr, from:from, to:to %} 8 | {% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsub.s32", mr:mr, from:from, to:to, flipped:true%} 9 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_mmm_q_per_row.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .{{label}}: 4 | 5 | {% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%} 6 | {% capture mr_over_4_min_1 %}{{ mr | divided_by: 4 | minus: 1}}{%endcapture%} 7 | 8 | {% for reg in (0..mr_over_4_min_1) %} 9 | vldmia r3!, { q{{reg}} } 10 | {% endfor %} 11 | 12 | {% if flipped %} 13 | {% for acc in (from..to) %} 14 | {% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%} 15 | {{op}} q{{acc}}, q{{acc}}, q{{other}} 16 | {% endfor %} 17 | {% else %} 18 | {% for acc in (from..to) %} 19 | {% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%} 20 | {{op}} q{{acc}}, q{{other}}, q{{acc}} 21 | {% endfor %} 22 | {% endif %} 23 | 24 | b .non_linear_loop 25 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_mmm_q_scalar.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .{{label}}: 4 | vmov s0, r3 5 | vdup.32 q0, d0[0] 6 | {% if flipped %} 7 | {% for reg in (from..to) %} 8 | {{op}} q{{reg}}, q{{reg}}, q0 9 | {% endfor %} 10 | {% else %} 11 | {% for reg in (from..to) %} 12 | {{op}} q{{reg}}, q0, q{{reg}} 13 | {% endfor %} 14 | {% endif %} 15 | b .non_linear_loop 16 | -------------------------------------------------------------------------------- /linalg/arm32/armv7neon/armv7neon_prefetch.tmpl: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .arm 4 | .text 5 | .global armv7neon_prefetch_{{suffix}} 6 | .type armv7neon_prefetch_{{suffix}}, %function 7 | 8 | armv7neon_prefetch_{{suffix}}: 9 | loop: 10 | pld [r0] 11 | pld [r0, #32] 12 | pld [r0, #64] 13 | pld [r0, #96] 14 | pld [r0, #128] 15 | pld [r0, #160] 16 | pld [r0, #192] 17 | pld [r0, #224] 18 | add r0, r0, #256 19 | cmp r0, r1 20 | blt loop 21 | 22 | bx lr 23 | -------------------------------------------------------------------------------- /linalg/arm64/apple_amx/dispatcher.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .non_linear: 4 | sub x0, x0, 40 5 | 6 | .non_linear_loop: 7 | add x0, x0, 40 8 | ldr x2, [x0] 9 | 10 | mov x4, #{{ jump_table | size }} 11 | 12 | cmp x2, #{{ jump_table | size }} 13 | csel x2, x2, x4, lt 14 | cmp x2, #0 15 | csel x2, x4, x2, lt 16 | 17 | adr x3, .jmp_table 18 | add x3, x3, x2, LSL#2 19 | br x3 20 | 21 | .jmp_table: 22 | {% for j in jump_table %} 23 | b .{{j}} 24 | {% endfor %} 25 | b .unsupported 26 | 27 | add x0, x2, #4000 28 | b .return 29 | 30 | .unsupported: 31 | mov x0, #1 32 | b .return 33 | 34 | .done: 35 | mov x0, 0 36 | b .return 37 | 38 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/arm64fp16_mmm_8h_scalar.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .{{label}}: 4 | add x2, x0, #8 5 | ld1 {v0.h}[0], [ x2 ] 6 | dup v0.8h, v0.h[0] 7 | {% if flipped %} 8 | {% for reg in (from..to) %} 9 | {{op}} v{{reg}}.8h, v{{reg}}.8h, v0.8h 10 | {% endfor %} 11 | {% else %} 12 | {% for reg in (from..to) %} 13 | {{op}} v{{reg}}.8h, v0.8h, v{{reg}}.8h 14 | {% endfor %} 15 | {% endif %} 16 | 17 | b .non_linear_loop 18 | 19 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/arm64fp16_mmm_f16_16x8/loop1/naive.tmpli: -------------------------------------------------------------------------------- 1 | 2 | fmla v16.8h, v0.8h, v4.h[0] 3 | fmla v17.8h, v1.8h, v4.h[0] 4 | fmla v18.8h, v0.8h, v4.h[1] 5 | fmla v19.8h, v1.8h, v4.h[1] 6 | fmla v20.8h, v0.8h, v4.h[2] 7 | fmla v21.8h, v1.8h, v4.h[2] 8 | fmla v22.8h, v0.8h, v4.h[3] 9 | fmla v23.8h, v1.8h, v4.h[3] 10 | 11 | fmla v24.8h, v0.8h, v4.h[4] 12 | fmla v25.8h, v1.8h, v4.h[4] 13 | fmla v26.8h, v0.8h, v4.h[5] 14 | fmla v27.8h, v1.8h, v4.h[5] 15 | fmla v28.8h, v0.8h, v4.h[6] 16 | fmla v29.8h, v1.8h, v4.h[6] 17 | fmla v30.8h, v0.8h, v4.h[7] 18 | fmla v31.8h, v1.8h, v4.h[7] 19 | 20 | ld1 {{ v0.8h, v1.8h }}, [x1], #32 21 | ld1 {{ v4.8h }}, [x2], #16 22 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/arm64fp16_mmm_f16_32x4/loop1/naive.tmpli: -------------------------------------------------------------------------------- 1 | 2 | fmla v16.8h, v0.8h, v4.h[0] 3 | fmla v17.8h, v1.8h, v4.h[0] 4 | fmla v18.8h, v2.8h, v4.h[0] 5 | fmla v19.8h, v3.8h, v4.h[0] 6 | fmla v20.8h, v0.8h, v4.h[1] 7 | fmla v21.8h, v1.8h, v4.h[1] 8 | fmla v22.8h, v2.8h, v4.h[1] 9 | fmla v23.8h, v3.8h, v4.h[1] 10 | 11 | fmla v24.8h, v0.8h, v4.h[2] 12 | fmla v25.8h, v1.8h, v4.h[2] 13 | fmla v26.8h, v2.8h, v4.h[2] 14 | fmla v27.8h, v3.8h, v4.h[2] 15 | fmla v28.8h, v0.8h, v4.h[3] 16 | fmla v29.8h, v1.8h, v4.h[3] 17 | fmla v30.8h, v2.8h, v4.h[3] 18 | fmla v31.8h, v3.8h, v4.h[3] 19 | 20 | ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [ x1 ], #64 21 | ldr d4, [x2], #8 22 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/arm64fp16_mmm_f16_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_min", op:"fmin", mr:mr, from:from, to:to %} 4 | {% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_max", op:"fmax", mr:mr, from:from, to:to %} 5 | {% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_mul", op:"fmul", mr:mr, from:from, to:to %} 6 | {% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_add", op:"fadd", mr:mr, from:from, to:to %} 7 | {% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_sub", op:"fsub", mr:mr, from:from, to:to %} 8 | {% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/arm64fp16_mmm_f16_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_min", op:"fmin", mr:mr, from:from, to:to %} 4 | {% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_max", op:"fmax", mr:mr, from:from, to:to %} 5 | {% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_mul", op:"fmul", mr:mr, from:from, to:to %} 6 | {% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_add", op:"fadd", mr:mr, from:from, to:to %} 7 | {% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_sub", op:"fsub", mr:mr, from:from, to:to %} 8 | {% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/arm64fp16_mmm_load_tile.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .load_tile: 4 | ldr x2, [ x0, #8 ] 5 | {% for reg in (from..to) %} 6 | ld1 { v{{reg}}.4s }, [ x2 ], #16 7 | {% endfor %} 8 | 9 | b .non_linear_loop 10 | 11 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/dispatcher.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .non_linear: 4 | sub x0, x0, 40 5 | 6 | .non_linear_loop: 7 | add x0, x0, 40 8 | ldr x2, [x0] 9 | 10 | mov x4, #{{ jump_table | size }} 11 | 12 | cmp x2, #{{ jump_table | size }} 13 | csel x2, x2, x4, lt 14 | cmp x2, #0 15 | csel x2, x4, x2, lt 16 | 17 | adr x3, .jmp_table 18 | add x3, x3, x2, LSL#2 19 | br x3 20 | 21 | .jmp_table: 22 | {% for j in jump_table %} 23 | b .{{j}} 24 | {% endfor %} 25 | b .unsupported 26 | 27 | add x0, x2, #4000 28 | b .return 29 | 30 | .unsupported: 31 | mov x0, #1 32 | b .return 33 | 34 | .done: 35 | mov x0, 0 36 | b .return 37 | 38 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/dummy_fmla_no_pragma.S: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | // serves as a canary build file to figure out which flag combination will accept half precision fmla 4 | 5 | .text 6 | .align 4 7 | 8 | // .cpu generic+fp+simd+fp16 9 | .global foo 10 | foo: 11 | fmla v16.8h, v0.8h, v8.h[0] 12 | ret 13 | 14 | -------------------------------------------------------------------------------- /linalg/arm64/arm64fp16/dummy_fmla_pragma.S: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | // serves as a canary build file to figure out which flag combination will accept half precision fmla 4 | 5 | .text 6 | .align 4 7 | 8 | .cpu generic+fp+simd+fp16 9 | .global foo 10 | foo: 11 | fmla v16.8h, v0.8h, v8.h[0] 12 | ret 13 | 14 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_4s_scalar.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .{{label}}: 4 | add x2, x0, #8 5 | ld1 {v0.s}[0], [ x2 ] 6 | dup v0.4s, v0.s[0] 7 | {% if flipped %} 8 | {% for reg in (from..to) %} 9 | {{op}} v{{reg}}.4s, v{{reg}}.4s, v0.4s 10 | {% endfor %} 11 | {% else %} 12 | {% for reg in (from..to) %} 13 | {{op}} v{{reg}}.4s, v0.4s, v{{reg}}.4s 14 | {% endfor %} 15 | {% endif %} 16 | 17 | b .non_linear_loop 18 | 19 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_f32_16x4/packed_packed_loop1/naive.tmpli: -------------------------------------------------------------------------------- 1 | 2 | fmla v16.4s, v0.4s, v4.s[0] 3 | fmla v17.4s, v1.4s, v4.s[0] 4 | fmla v18.4s, v2.4s, v4.s[0] 5 | fmla v19.4s, v3.4s, v4.s[0] 6 | fmla v20.4s, v0.4s, v4.s[1] 7 | fmla v21.4s, v1.4s, v4.s[1] 8 | fmla v22.4s, v2.4s, v4.s[1] 9 | fmla v23.4s, v3.4s, v4.s[1] 10 | 11 | fmla v24.4s, v0.4s, v4.s[2] 12 | fmla v25.4s, v1.4s, v4.s[2] 13 | fmla v26.4s, v2.4s, v4.s[2] 14 | fmla v27.4s, v3.4s, v4.s[2] 15 | fmla v28.4s, v0.4s, v4.s[3] 16 | fmla v29.4s, v1.4s, v4.s[3] 17 | fmla v30.4s, v2.4s, v4.s[3] 18 | fmla v31.4s, v3.4s, v4.s[3] 19 | 20 | ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64 21 | ld1 {{ v4.4s }}, [ x2 ], #16 22 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_f32_8x8/packed_packed_loop1/naive.tmpli: -------------------------------------------------------------------------------- 1 | 2 | fmla v16.4s, v0.4s, v4.s[0] 3 | fmla v17.4s, v1.4s, v4.s[0] 4 | fmla v18.4s, v0.4s, v4.s[1] 5 | fmla v19.4s, v1.4s, v4.s[1] 6 | fmla v20.4s, v0.4s, v4.s[2] 7 | fmla v21.4s, v1.4s, v4.s[2] 8 | fmla v22.4s, v0.4s, v4.s[3] 9 | fmla v23.4s, v1.4s, v4.s[3] 10 | 11 | fmla v24.4s, v0.4s, v5.s[0] 12 | fmla v25.4s, v1.4s, v5.s[0] 13 | fmla v26.4s, v0.4s, v5.s[1] 14 | fmla v27.4s, v1.4s, v5.s[1] 15 | fmla v28.4s, v0.4s, v5.s[2] 16 | fmla v29.4s, v1.4s, v5.s[2] 17 | fmla v30.4s, v0.4s, v5.s[3] 18 | fmla v31.4s, v1.4s, v5.s[3] 19 | 20 | ld1 {{ v0.4s, v1.4s }}, [x1], #32 21 | ld1 {{ v4.4s, v5.4s }}, [x2], #32 22 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_f32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_min", op:"fmin", mr:mr, from:from, to:to %} 4 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_max", op:"fmax", mr:mr, from:from, to:to %} 5 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_mul", op:"fmul", mr:mr, from:from, to:to %} 6 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_add", op:"fadd", mr:mr, from:from, to:to %} 7 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub", op:"fsub", mr:mr, from:from, to:to %} 8 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_f32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_min", op:"fmin", mr:mr, from:from, to:to %} 4 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_max", op:"fmax", mr:mr, from:from, to:to %} 5 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_mul", op:"fmul", mr:mr, from:from, to:to %} 6 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_add", op:"fadd", mr:mr, from:from, to:to %} 7 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub", op:"fsub", mr:mr, from:from, to:to %} 8 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_i32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_min", op:"smin", mr:mr, from:from, to:to %} 4 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_max", op:"smax", mr:mr, from:from, to:to %} 5 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_mul", op:"mul", mr:mr, from:from, to:to %} 6 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_add", op:"add", mr:mr, from:from, to:to %} 7 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub", op:"sub", mr:mr, from:from, to:to %} 8 | {% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub_flipped", op:"sub", mr:mr, from:from, to:to, flipped: true %} 9 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_i32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_min", op:"smin", mr:mr, from:from, to:to %} 4 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_max", op:"smax", mr:mr, from:from, to:to %} 5 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_mul", op:"mul", mr:mr, from:from, to:to %} 6 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_add", op:"add", mr:mr, from:from, to:to %} 7 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub", op:"sub", mr:mr, from:from, to:to %} 8 | {% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub_flipped", op:"sub", mr:mr, from:from, to:to, flipped: true %} 9 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/arm64simd_mmm_load_tile.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .load_tile: 4 | ldr x2, [ x0, #8 ] 5 | {% for reg in (from..to) %} 6 | ld1 { v{{reg}}.4s }, [ x2 ], #16 7 | {% endfor %} 8 | 9 | b .non_linear_loop 10 | 11 | -------------------------------------------------------------------------------- /linalg/arm64/arm64simd/dispatcher.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: ft=arm 2 | 3 | .non_linear: 4 | sub x0, x0, 40 5 | 6 | .non_linear_loop: 7 | add x0, x0, 40 8 | ldr x2, [x0] 9 | 10 | mov x4, #{{ jump_table | size }} 11 | 12 | cmp x2, #{{ jump_table | size }} 13 | csel x2, x2, x4, lt 14 | cmp x2, #0 15 | csel x2, x4, x2, lt 16 | 17 | adr x3, .jmp_table 18 | add x3, x3, x2, LSL#2 19 | br x3 20 | 21 | .jmp_table: 22 | {% for j in jump_table %} 23 | b .{{j}} 24 | {% endfor %} 25 | b .unsupported 26 | 27 | add x0, x2, #4000 28 | b .return 29 | 30 | .unsupported: 31 | mov x0, #1 32 | b .return 33 | 34 | .done: 35 | mov x0, 0 36 | b .return 37 | 38 | -------------------------------------------------------------------------------- /linalg/benches/mm_for_wavenet_hw.rs: -------------------------------------------------------------------------------- 1 | use criterion::*; 2 | 3 | mod utils; 4 | use utils::*; 5 | 6 | fn s16x60x8(c: &mut Criterion) { 7 | packed_packed(c, "wavenet", 32, 32, 8); // postproc 8 | packed_packed(c, "wavenet", 16, 60, 8); 9 | } 10 | 11 | criterion_group!(benches, s16x60x8); 12 | criterion_main!(benches); 13 | -------------------------------------------------------------------------------- /linalg/benches/sigmoid.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate criterion; 3 | extern crate tract_linalg; 4 | use criterion::Criterion; 5 | 6 | fn ssigmoid(c: &mut Criterion, n: usize) { 7 | c.bench_function(&format!("ssigmoid_{n}"), move |be| { 8 | let mut s = (0..n).map(|i| i as f32 / 10.0).collect::>(); 9 | let op = &(tract_linalg::ops().sigmoid_f32)(); 10 | be.iter(|| op.run(&mut s)); 11 | }); 12 | } 13 | 14 | fn bs(c: &mut Criterion) { 15 | ssigmoid(c, 4); 16 | ssigmoid(c, 8); 17 | ssigmoid(c, 128); 18 | ssigmoid(c, 1024); 19 | } 20 | 21 | criterion_group!(benches, bs); 22 | criterion_main!(benches); 23 | -------------------------------------------------------------------------------- /linalg/cost_model/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cost_model" 3 | version = "0.20.7-pre" 4 | edition = "2024" 5 | 6 | [workspace] 7 | members = [] 8 | 9 | [dependencies] 10 | lazy_static = "1.4.0" 11 | clap = "3.0.7" 12 | scan_fmt = "0.2.6" 13 | rand = "0.8.4" 14 | colorous = "1.0.6" 15 | nu-ansi-term = "0.50" 16 | pbr = "1" 17 | readings-probe = "0.1.4" 18 | tract-linalg = { path = ".."} 19 | tract-data = { path = "../../data" } 20 | -------------------------------------------------------------------------------- /linalg/cost_model/train/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.22.0 2 | torch==2.7.0 3 | -------------------------------------------------------------------------------- /linalg/cost_model/train/runme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | device_name=$1 4 | dataset_name=$2 5 | platform=$3 6 | 7 | [ -e venv ] || virtualenv venv 8 | . venv/bin/activate 9 | 10 | pip install -r requirements.txt 11 | 12 | set -ex 13 | mkdir -p tmp 14 | ( 15 | cd tmp 16 | aws s3 cp s3://tract-ci-builds/products/$device_name/$dataset_name.tgz . 17 | tar zxf $dataset_name.tgz 18 | data=`ls -1 $dataset_name.$device_name` 19 | python ../train.py -N 15 --platform=$platform $dataset_name.$device_name/$data $platform.rs 20 | ) 21 | mv tmp/$platform.rs . 22 | -------------------------------------------------------------------------------- /linalg/matmul-bench/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "matmul-bench" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | edition = "2024" 6 | 7 | [workspace] 8 | members = [] 9 | 10 | [dependencies] 11 | cblas = { version = "0.3", optional = true } 12 | accelerate-src = { version = "0.3", optional = true } 13 | blis-src = { version = "0.2", features = ["static"], optional = true } 14 | matrixmultiply = "*" 15 | tract-data.workspace = true 16 | tract-linalg.workspace = true 17 | 18 | 19 | [features] 20 | default = [] 21 | blas = ["cblas"] 22 | blis = ["blis-src", "blas"] 23 | accelerate = ["accelerate-src", "blas"] 24 | 25 | [build-dependencies] 26 | cc = "1.0" 27 | 28 | [dev-dependencies] 29 | criterion = "*" 30 | 31 | [[bench]] 32 | name = "matmul" 33 | harness = false 34 | -------------------------------------------------------------------------------- /linalg/matmul-bench/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let mut cc = cc::Build::new(); 3 | cc 4 | .file("c/tile_1x1.c") 5 | .file("c/tile_2x2.c") 6 | .file("c/tile_4x4.c") 7 | .file("c/packed_tile_4x4.c") 8 | .file("c/tile_8x8.c") 9 | .file("c/packed_tile_8x8.c"); 10 | if std::env::var("TARGET").unwrap().starts_with("aarch64") { 11 | cc.flag("-mtune=cortex-a53"); 12 | } else { 13 | cc.flag("-mtune=haswell"); 14 | } 15 | cc.flag("-funsafe-math-optimizations").compile("libmatmulbench"); 16 | } 17 | -------------------------------------------------------------------------------- /linalg/matmul-bench/c/tile_1x1.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | void c_tile_1x1(size_t m, size_t k, size_t n, float *a, float *b, float *c) { 5 | for(size_t row = 0 ; row < m ; row++) { 6 | for(size_t col = 0 ; col < n ; col++) { 7 | float sum00 = 0.0; 8 | for(size_t i = 0; i < k ; i++) { 9 | float a0 = a[row * k + i]; 10 | float b0 = b[i * n + col]; 11 | sum00 += a0 * b0; 12 | } 13 | c[row * n + col] = sum00; 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /linalg/src/arm32/armvfpv2.rs: -------------------------------------------------------------------------------- 1 | use crate::frame::mmm::ImplementationQuality::ManuallyOptimized; 2 | use crate::frame::mmm::*; 3 | use crate::Ops; 4 | 5 | MMMExternKernel!(armvfpv2_mmm_f32_4x4(4, 4)@(4, 4) quality(ManuallyOptimized)); 6 | 7 | pub fn plug(ops: &mut Ops) { 8 | log::info!("armvfpv2 activated for smmm"); 9 | ops.mmm_f32 = Box::new(|_, _, _| armvfpv2_mmm_f32_4x4.mmm()); 10 | ops.mmm_impls.push(armvfpv2_mmm_f32_4x4.mmm()); 11 | } 12 | -------------------------------------------------------------------------------- /linalg/src/arm64/cortex_a72.rs: -------------------------------------------------------------------------------- 1 | use crate::frame::mmm::cost_model::CostModel; 2 | pub fn models() -> Vec<(&'static str, CostModel<'static>)> { 3 | vec![] 4 | } 5 | -------------------------------------------------------------------------------- /linalg/src/arm64/cortex_a73.rs: -------------------------------------------------------------------------------- 1 | use crate::frame::mmm::cost_model::CostModel; 2 | pub fn models() -> Vec<(&'static str, CostModel<'static>)> { 3 | vec![] 4 | } 5 | -------------------------------------------------------------------------------- /linalg/src/frame/mod.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | pub mod block_quant; 3 | #[macro_use] 4 | pub mod element_wise; 5 | pub mod element_wise_helper; 6 | #[macro_use] 7 | pub mod unicast; 8 | #[macro_use] 9 | pub mod by_scalar; 10 | #[macro_use] 11 | pub mod leaky_relu; 12 | #[macro_use] 13 | pub mod lut; 14 | #[macro_use] 15 | pub mod mmm; 16 | #[macro_use] 17 | pub mod pack; 18 | #[macro_use] 19 | pub mod reduce; 20 | #[macro_use] 21 | pub mod sigmoid; 22 | #[macro_use] 23 | pub mod tanh; 24 | #[macro_use] 25 | pub mod weights; 26 | -------------------------------------------------------------------------------- /linalg/src/x86_64_fma/intel.rs: -------------------------------------------------------------------------------- 1 | use crate::frame::mmm::cost_model::CostModel; 2 | #[allow(dead_code)] 3 | pub fn models() -> Vec<(&'static str, CostModel<'static>)> { 4 | vec!( 5 | )} 6 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/1x1/packed_packed_loop1/avx-512.tmpli: -------------------------------------------------------------------------------- 1 | vbroadcastss zmm15, dword ptr [rcx] 2 | 3 | vmovups zmm8, [rax] 4 | vfmadd231ps zmm0, zmm15, zmm8 5 | 6 | add rcx, 4 7 | add rax, 64 8 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/1x1/packed_packed_loop1/unroll-4.tmpli: -------------------------------------------------------------------------------- 1 | // slow 2 | vbroadcastss xmm16, dword ptr [rcx] 3 | vbroadcastss xmm17, dword ptr [rcx + 4] 4 | vbroadcastss xmm18, dword ptr [rcx + 8] 5 | vbroadcastss xmm19, dword ptr [rcx + 12] 6 | 7 | // fast 8 | vmovups xmm31, [rcx] 9 | vbroadcastss zmm16, xmm31 10 | valignd xmm17, xmm31, xmm31, 1 11 | vbroadcastss zmm17, xmm17 12 | valignd xmm18, xmm31, xmm31, 2 13 | vbroadcastss zmm18, xmm18 14 | valignd xmm19, xmm31, xmm31, 3 15 | vbroadcastss zmm19, xmm19 16 | 17 | // commmon 18 | vfmadd231ps zmm0, zmm16, [rax + 0] 19 | vfmadd231ps zmm1, zmm17, [rax + 64] 20 | vfmadd231ps zmm2, zmm18, [rax + 128] 21 | vfmadd231ps zmm3, zmm19, [rax + 192] 22 | 23 | add rcx, 16 24 | add rax, 256 25 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/1x1/packed_packed_loop1/unroll.tmpli: -------------------------------------------------------------------------------- 1 | vbroadcastss zmm15, dword ptr [rcx] 2 | 3 | vmovaps zmm8, [rax + 0] 4 | vfmadd231ps zmm0, zmm15, zmm8 5 | 6 | vbroadcastss zmm16, dword ptr [rcx + 4] 7 | vmovaps zmm9, [rax + 64] 8 | vfmadd231ps zmm1, zmm16, zmm9 9 | 10 | add rcx, 8 11 | add rax, 128 12 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/7x1/packed_packed_loop1/avx-512.tmpli: -------------------------------------------------------------------------------- 1 | // Tile size: 7x1 2 | // Accumulators: 0-6 3 | // Col regs: 6-13 4 | // Row regs: 15 5 | vbroadcastss zmm15, dword ptr [rcx] 6 | 7 | vmovaps zmm7, [rax + 0] 8 | vmovaps zmm8, [rax + 64] 9 | vmovaps zmm9, [rax + 128] 10 | vmovaps zmm10, [rax + 192] 11 | vmovaps zmm11, [rax + 256] 12 | vmovaps zmm12, [rax + 320] 13 | vmovaps zmm13, [rax + 384] 14 | 15 | vfmadd231ps zmm0, zmm7, zmm15 16 | vfmadd231ps zmm1, zmm8, zmm15 17 | vfmadd231ps zmm2, zmm9, zmm15 18 | vfmadd231ps zmm3, zmm10, zmm15 19 | vfmadd231ps zmm4, zmm11, zmm15 20 | vfmadd231ps zmm5, zmm12, zmm15 21 | vfmadd231ps zmm6, zmm13, zmm15 22 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/avx512_mmm_load_tile.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {{L}}load_tile: 4 | mov r8, [rdi + 8] 5 | {% for reg in (from..to) %} 6 | vmovups zmm{{reg}}, zmmword ptr [r8 + {{ reg|minus:from|times:64 }}] 7 | {% endfor %} 8 | 9 | jmp {{L}}non_linear_loop 10 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/f32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "zmm_per_col.tmpliq" label:"per_col_min", op:"vminps", mr:mr, from:from, to:to %} 4 | {% include "zmm_per_col.tmpliq" label:"per_col_max", op:"vmaxps", mr:mr, from:from, to:to %} 5 | {% include "zmm_per_col.tmpliq" label:"per_col_add", op:"vaddps", mr:mr, from:from, to:to %} 6 | {% include "zmm_per_col.tmpliq" label:"per_col_mul", op:"vmulps", mr:mr, from:from, to:to %} 7 | {% include "zmm_per_col.tmpliq" label:"per_col_sub", op:"vsubps", from:from, to:to %} 8 | {% include "zmm_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsubps", from:from, to:to, flipped: true %} 9 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/f32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "zmm_per_row.tmpliq" label:"per_row_min", op:"vminps", mr:mr, from:from, to:to %} 4 | {% include "zmm_per_row.tmpliq" label:"per_row_max", op:"vmaxps", mr:mr, from:from, to:to %} 5 | {% include "zmm_per_row.tmpliq" label:"per_row_add", op:"vaddps", mr:mr, from:from, to:to %} 6 | {% include "zmm_per_row.tmpliq" label:"per_row_mul", op:"vmulps", mr:mr, from:from, to:to %} 7 | {% include "zmm_per_row.tmpliq" label:"per_row_sub", op:"vsubps", from:from, to:to %} 8 | {% include "zmm_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsubps", from:from, to:to, flipped: true %} 9 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/i32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "zmm_per_col.tmpliq" label:"per_col_min", op:"vpminsd", mr:mr, from:from, to:to%} 4 | {% include "zmm_per_col.tmpliq" label:"per_col_max", op:"vpmaxsd", mr:mr, from:from, to:to%} 5 | {% include "zmm_per_col.tmpliq" label:"per_col_add", op:"vpaddd", mr:mr, from:from, to:to%} 6 | {% include "zmm_per_col.tmpliq" label:"per_col_mul", op:"vpmulld", mr:mr, from:from, to:to%} 7 | {% include "zmm_per_col.tmpliq" label:"per_col_sub", op:"vpsubd", from:from, to:to%} 8 | {% include "zmm_per_col.tmpliq" label:"per_col_sub_flipped", op:"vpsubd", from:from, to:to, flipped: true%} 9 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/i32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "zmm_per_row.tmpliq" label:"per_row_min", op:"vpminsd", mr:mr, from:from, to:to%} 4 | {% include "zmm_per_row.tmpliq" label:"per_row_max", op:"vpmaxsd", mr:mr, from:from, to:to%} 5 | {% include "zmm_per_row.tmpliq" label:"per_row_add", op:"vpaddd", mr:mr, from:from, to:to%} 6 | {% include "zmm_per_row.tmpliq" label:"per_row_mul", op:"vpmulld", mr:mr, from:from, to:to%} 7 | {% include "zmm_per_row.tmpliq" label:"per_row_sub", op:"vpsubd", from:from, to:to%} 8 | {% include "zmm_per_row.tmpliq" label:"per_row_sub_flipped", op:"vpsubd", from:from, to:to, flipped: true%} 9 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/i32_scalars.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | {% unless arch %} 3 | {% assign arch = "ymm" %} 4 | {% endunless %} 5 | {% include "zmm_scalar.tmpliq" label:"scalar_min", op:"vpminsd", from:from, to:to, arch:arch %} 6 | {% include "zmm_scalar.tmpliq" label:"scalar_max", op:"vpmaxsd", from:from, to:to, arch:arch %} 7 | {% include "zmm_scalar.tmpliq" label:"scalar_mul", op:"vpmulld", from:from, to:to, arch:arch %} 8 | {% include "zmm_scalar.tmpliq" label:"scalar_add", op:"vpaddd", from:from, to:to, arch:arch %} 9 | {% include "zmm_scalar.tmpliq" label:"scalar_sub", op:"vpsubd", from:from, to:to, arch:arch %} 10 | {% include "zmm_scalar.tmpliq" label:"scalar_sub_flipped", op:"vpsubd", from:from, to:to, flipped: true, arch:arch %} 11 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/postamble.tmpliq: -------------------------------------------------------------------------------- 1 | {{L}}return: 2 | ldmxcsr [rsp + 4] 3 | add rsp, 8 4 | 5 | pop r15 6 | pop r14 7 | pop r13 8 | pop r12 9 | pop rbx 10 | 11 | {% if family == "windows" %} 12 | pop rsi 13 | pop rdi 14 | 15 | vmovaps xmm15, [rsp+16*9] 16 | vmovaps xmm14, [rsp+16*8] 17 | vmovaps xmm13, [rsp+16*7] 18 | vmovaps xmm12, [rsp+16*6] 19 | vmovaps xmm11, [rsp+16*5] 20 | vmovaps xmm10, [rsp+16*4] 21 | vmovaps xmm9, [rsp+16*3] 22 | vmovaps xmm8, [rsp+16*2] 23 | vmovaps xmm7, [rsp+16*1] 24 | vmovaps xmm6, [rsp] 25 | {% endif %} 26 | 27 | mov rsp, rbp 28 | pop rbp 29 | ret 30 | 31 | {% if msvc %} 32 | {{arch}}_mmm_f32_{{size}}_{{suffix}} endp 33 | _text ends 34 | end 35 | 36 | {% else %} 37 | .cfi_endproc 38 | {% endif %} 39 | -------------------------------------------------------------------------------- /linalg/x86_64/avx512/zmm_scalar.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {{L}}{{label}}: 4 | vbroadcastss zmm12, dword ptr [rdi + 8] 5 | {% if flipped %} 6 | {% for reg in (from..to) %} 7 | {{op}} zmm{{reg}}, zmm{{reg}}, zmm12 8 | {% endfor %} 9 | {% else %} 10 | {% for reg in (from..to) %} 11 | {{op}} zmm{{reg}}, zmm12, zmm{{reg}} 12 | {% endfor %} 13 | {% endif %} 14 | 15 | jmp {{L}}non_linear_loop 16 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/7x1/packed_packed_loop1/avx.tmpli: -------------------------------------------------------------------------------- 1 | // Tile size: 6x1 2 | // Accumulators: 0-5 3 | // Col regs: 6-11 4 | // Row regs: 15 5 | vbroadcastss ymm15, dword ptr [rcx] 6 | 7 | vmovaps ymm6, [rax + 0] 8 | vmovaps ymm7, [rax + 32] 9 | vmovaps ymm8, [rax + 64] 10 | vmovaps ymm9, [rax + 96] 11 | 12 | vfmadd231ps ymm0, ymm6, ymm15 13 | vfmadd231ps ymm1, ymm7, ymm15 14 | 15 | vmovaps ymm10, [rax + 128] 16 | vfmadd231ps ymm2, ymm8, ymm15 17 | 18 | vmovaps ymm11, [rax + 160] 19 | vfmadd231ps ymm3, ymm9, ymm15 20 | 21 | vfmadd231ps ymm4, ymm10, ymm15 22 | vfmadd231ps ymm5, ymm11, ymm15 23 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/fma_mmm_f32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_min", op:"vminps", mr:mr, from:from, to:to, type:type%} 4 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_max", op:"vmaxps", mr:mr, from:from, to:to, type:type%} 5 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_add", op:"vaddps", mr:mr, from:from, to:to, type:type%} 6 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_mul", op:"vmulps", mr:mr, from:from, to:to, type:type%} 7 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_sub", op:"vsubps", from:from, to:to, type:type %} 8 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsubps", from:from, to:to, flipped: true, type:type%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/fma_mmm_f32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_min", op:"vminps", mr:mr, from:from, to:to, type: type%} 4 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_max", op:"vmaxps", mr:mr, from:from, to:to, type: type%} 5 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_add", op:"vaddps", mr:mr, from:from, to:to, type: type%} 6 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_mul", op:"vmulps", mr:mr, from:from, to:to, type: type%} 7 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_sub", op:"vsubps", from:from, to:to, type: type%} 8 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsubps", from:from, to:to, flipped: true, type: type%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/fma_mmm_i32_per_cols.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_min", op:"vpminsd", mr:mr, from:from, to:to, type:"i32"%} 4 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_max", op:"vpmaxsd", mr:mr, from:from, to:to, type:"i32"%} 5 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_add", op:"vpaddd", mr:mr, from:from, to:to, type:"i32"%} 6 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_mul", op:"vpmulld", mr:mr, from:from, to:to, type:"i32"%} 7 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_sub", op:"vpsubd", from:from, to:to, type:"i32"%} 8 | {% include "fma_mmm_ymm_per_col.tmpliq" label:"per_col_sub_flipped", op:"vpsubd", from:from, to:to, flipped: true, type:"i32"%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/fma_mmm_i32_per_rows.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_min", op:"vpminsd", mr:mr, from:from, to:to, type:"i32"%} 4 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_max", op:"vpmaxsd", mr:mr, from:from, to:to, type:"i32"%} 5 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_add", op:"vpaddd", mr:mr, from:from, to:to, type:"i32"%} 6 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_mul", op:"vpmulld", mr:mr, from:from, to:to, type:"i32"%} 7 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_sub", op:"vpsubd", from:from, to:to, type:"i32"%} 8 | {% include "fma_mmm_ymm_per_row.tmpliq" label:"per_row_sub_flipped", op:"vpsubd", from:from, to:to, flipped: true, type:"i32"%} 9 | 10 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/fma_mmm_load_tile.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {{L}}load_tile: 4 | mov r8, [rdi + 8] 5 | {% for reg in (from..to) %} 6 | vmovups ymm{{reg}}, ymmword ptr [r8 + {{ reg|minus:from|times:32 }}] 7 | {% endfor %} 8 | 9 | jmp {{L}}non_linear_loop 10 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/fma_mmm_ymm_scalar.tmpliq: -------------------------------------------------------------------------------- 1 | // vim: set syntax=asm : 2 | 3 | {{L}}{{label}}: 4 | {% if type == "f16" %} 5 | pinsrw xmm12, word ptr [rdi + 8], 0 6 | vcvtph2ps ymm12, xmm12 7 | vbroadcastss ymm12, xmm12 8 | {% else %} 9 | vbroadcastss ymm12, dword ptr [rdi + 8] 10 | {% endif %} 11 | 12 | {% if flipped %} 13 | {% for reg in (from..to) %} 14 | {{op}} ymm{{reg}}, ymm{{reg}}, ymm12 15 | {% endfor %} 16 | {% else %} 17 | {% for reg in (from..to) %} 18 | {{op}} ymm{{reg}}, ymm12, ymm{{reg}} 19 | {% endfor %} 20 | {% endif %} 21 | 22 | jmp {{L}}non_linear_loop 23 | -------------------------------------------------------------------------------- /linalg/x86_64/fma/postamble.tmpliq: -------------------------------------------------------------------------------- 1 | {{L}}return: 2 | ldmxcsr [rsp + 4] 3 | add rsp, 8 4 | 5 | pop r15 6 | pop r14 7 | pop r13 8 | pop r12 9 | pop rbx 10 | 11 | {% if family == "windows" %} 12 | pop rsi 13 | pop rdi 14 | 15 | vmovaps xmm15, [rsp+16*9] 16 | vmovaps xmm14, [rsp+16*8] 17 | vmovaps xmm13, [rsp+16*7] 18 | vmovaps xmm12, [rsp+16*6] 19 | vmovaps xmm11, [rsp+16*5] 20 | vmovaps xmm10, [rsp+16*4] 21 | vmovaps xmm9, [rsp+16*3] 22 | vmovaps xmm8, [rsp+16*2] 23 | vmovaps xmm7, [rsp+16*1] 24 | vmovaps xmm6, [rsp] 25 | {% endif %} 26 | 27 | mov rsp, rbp 28 | pop rbp 29 | ret 30 | 31 | {% if msvc %} 32 | fma_mmm_{{type}}_{{size}}_{{suffix}} endp 33 | _text ends 34 | end 35 | 36 | {% else %} 37 | .cfi_endproc 38 | {% endif %} 39 | -------------------------------------------------------------------------------- /metal/README.md: -------------------------------------------------------------------------------- 1 | # tract-metal 2 | 3 | ## Updating Metal Flash Attention library 4 | 5 | ``` 6 | git clone https://github.com/philipturner/metal-flash-attention.git 7 | cd metal-flash-attention 8 | 9 | # for iOS 10 | swift build.swift --platform iOS --xcode-path /Applications/Xcode.app 11 | cp build/lib/libMetalFlashAttention.metallib path/to/tract/metal/src/kernels/libMetalFlashAttention-ios.metallib 12 | 13 | # for MacOS 14 | swift build.swift --platform macOS --xcode-path /Applications/Xcode.app 15 | cp build/lib/libMetalFlashAttention.metallib path/to/tract/metal/src/kernels/libMetalFlashAttention-macos.metallib 16 | ``` -------------------------------------------------------------------------------- /metal/src/kernels/matmul/mfa/libMetalFlashAttention-ios.metallib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/metal/src/kernels/matmul/mfa/libMetalFlashAttention-ios.metallib -------------------------------------------------------------------------------- /metal/src/kernels/matmul/mfa/libMetalFlashAttention-macos.metallib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/metal/src/kernels/matmul/mfa/libMetalFlashAttention-macos.metallib -------------------------------------------------------------------------------- /metal/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod command_buffer; 2 | mod context; 3 | mod encoder; 4 | mod func_constants; 5 | pub mod kernels; 6 | mod ops; 7 | mod rewrite_rules; 8 | mod tests; 9 | mod transform; 10 | mod utils; 11 | 12 | use crate::func_constants::{ConstantValues, Value}; 13 | pub use crate::kernels::matmul::MetalGemmImplKind; 14 | use crate::kernels::LibraryName; 15 | 16 | pub use crate::context::{MetalContext, MetalStream, METAL_STREAM}; 17 | pub use crate::transform::MetalTransform; 18 | -------------------------------------------------------------------------------- /metal/src/rewrite_rules/mod.rs: -------------------------------------------------------------------------------- 1 | mod add_matmul_broadcast; 2 | mod fuse_axis_op; 3 | mod untranspose_matmul_output; 4 | mod rms_norm; 5 | 6 | pub use add_matmul_broadcast::add_broadcast_pre_matmul; 7 | pub use fuse_axis_op::{fuse_axis_op, fuse_move_axis}; 8 | pub use untranspose_matmul_output::untranspose_matmul_output; 9 | pub use rms_norm::remove_rms_norm_cast; 10 | 11 | pub use tract_gpu::rewrite_rules::{next_node, previous_node, previous_nodes}; 12 | -------------------------------------------------------------------------------- /nnef/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /nnef/cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-nnef-cli" 3 | version = "0.21.8-pre" 4 | authors = [ 5 | "Mathieu Poumeyrol ", 6 | "Hubert de La Jonquière " 7 | ] 8 | license = "MIT OR Apache-2.0" 9 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 10 | repository = "https://github.com/snipsco/tract" 11 | keywords = [ "NeuralNetworks", "NNEF" ] 12 | categories = [ "science" ] 13 | autobenches = false 14 | edition = "2024" 15 | rust-version = "1.85" 16 | 17 | [dependencies] 18 | anyhow.workspace = true 19 | env_logger.workspace = true 20 | log.workspace = true 21 | structopt.workspace = true 22 | tract-nnef.workspace = true 23 | tract-pulse.workspace = true 24 | tract-onnx-opl.workspace = true 25 | -------------------------------------------------------------------------------- /nnef/nnef-resources/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-nnef-resources" 3 | version = "0.21.14-pre" 4 | authors = [ 5 | "Mathieu Poumeyrol ", 6 | "Hubert de La Jonquière " 7 | ] 8 | license = "MIT OR Apache-2.0" 9 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 10 | repository = "https://github.com/snipsco/tract" 11 | keywords = [ "NeuralNetworks", "NNEF" ] 12 | categories = [ "science" ] 13 | autobenches = false 14 | edition = "2024" 15 | rust-version = "1.85" 16 | 17 | [badges] 18 | maintenance = { status = "actively-developed" } 19 | 20 | [dependencies] 21 | serde_json.workspace = true 22 | serde.workspace = true 23 | anyhow.workspace = true 24 | nom.workspace = true 25 | tract-nnef.workspace = true 26 | 27 | 28 | -------------------------------------------------------------------------------- /nnef/nnef-resources/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod json_loader; 2 | 3 | pub mod internal { 4 | pub use crate::json_loader::{JsonLoader, JsonResource}; 5 | } 6 | -------------------------------------------------------------------------------- /nnef/nnef-resources/tests/nnef_with_json/graph.nnef: -------------------------------------------------------------------------------- 1 | version 1.0; 2 | 3 | extension tract_registry tract_resource; 4 | 5 | graph afe_graph(src) -> (output) 6 | { 7 | w = tract_resource_get(label = "src_config", key = "$.width"); 8 | h = tract_resource_get(label = "src_config", key = "$.height"); 9 | src = external(shape = [w, h]); 10 | output = src; 11 | } -------------------------------------------------------------------------------- /nnef/nnef-resources/tests/nnef_with_json/src_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "width": 2, 3 | "height": 10 4 | } -------------------------------------------------------------------------------- /nnef/nnef-resources/tests/test_json_resource.rs: -------------------------------------------------------------------------------- 1 | use tract_nnef::internal::*; 2 | use tract_nnef_resources::internal::JsonLoader; 3 | 4 | #[test] 5 | fn load_model_with_json_resource() -> TractResult<()> { 6 | let model = tract_nnef::nnef() 7 | .with_tract_core() 8 | .with_tract_resource() 9 | .with_resource_loader(JsonLoader) 10 | .model_for_path("tests/nnef_with_json")?; 11 | 12 | assert_eq!(model.input_fact(0)?.shape.as_concrete().unwrap(), &vec![2, 10]); 13 | assert_eq!(model.output_fact(0)?.shape.as_concrete().unwrap(), &vec![2, 10]); 14 | Ok(()) 15 | } -------------------------------------------------------------------------------- /onnx-opl/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /onnx-opl/src/is_nan.rs: -------------------------------------------------------------------------------- 1 | use tract_nnef::internal::*; 2 | 3 | element_wise_oop!(is_nan, IsNan, 4 | [f32] => bool |_, xs, ys| { 5 | xs.iter().zip(ys.iter_mut()).for_each(|(x,y)| *y = x.is_nan()); 6 | Ok(()) 7 | }, 8 | [f16] => bool |_, xs, ys| { 9 | xs.iter().zip(ys.iter_mut()).for_each(|(x,y)| *y = x.is_nan()); 10 | Ok(()) 11 | }; 12 | prefix: "onnx." 13 | ); 14 | -------------------------------------------------------------------------------- /onnx-opl/src/ml/mod.rs: -------------------------------------------------------------------------------- 1 | use tract_nnef::internal::*; 2 | 3 | pub mod category_mapper; 4 | pub mod tree; 5 | pub mod tree_ensemble_classifier; 6 | 7 | pub use category_mapper::{DirectLookup, ReverseLookup}; 8 | 9 | pub fn register(registry: &mut Registry) { 10 | category_mapper::register(registry); 11 | tree_ensemble_classifier::register(registry); 12 | } 13 | -------------------------------------------------------------------------------- /onnx/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /onnx/build-proto.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let _ = std::fs::create_dir_all("src/prost"); 3 | std::env::set_var("PROTOC", protobuf_src::protoc()); 4 | prost_build::Config::new() 5 | .out_dir("src/prost") 6 | .compile_protos(&["protos/onnx/onnx.proto3"], &["protos/"]) 7 | .unwrap(); 8 | } 9 | -------------------------------------------------------------------------------- /onnx/src/ops/ml/mod.rs: -------------------------------------------------------------------------------- 1 | mod category_mapper; 2 | mod tree_ensemble_classifier; 3 | 4 | use crate::model::OnnxOpRegister; 5 | 6 | pub fn register_all_ops(reg: &mut OnnxOpRegister) { 7 | category_mapper::register_all_ops(reg); 8 | tree_ensemble_classifier::register_all_ops(reg); 9 | } 10 | -------------------------------------------------------------------------------- /onnx/src/ops/rec.rs: -------------------------------------------------------------------------------- 1 | use crate::model::OnnxOpRegister; 2 | 3 | pub mod common; 4 | pub mod gru; 5 | pub mod lstm; 6 | pub mod rnn; 7 | pub mod scan; 8 | 9 | pub fn register_all_ops(reg: &mut OnnxOpRegister) { 10 | reg.insert("GRU", gru::gru); 11 | reg.insert("LSTM", lstm::lstm); 12 | reg.insert("RNN", rnn::rnn); 13 | reg.insert("Scan", scan::scan); 14 | } 15 | -------------------------------------------------------------------------------- /onnx/test_cases/byte_sb_bidi_lstm/README.md: -------------------------------------------------------------------------------- 1 | # Byte-level sentence boundary detection with a bidirectional LSTM 2 | 3 | See [here](https://app.wandb.ai/bminixhofer/nnsplit/runs/3ty2i90r/overview) for the code used to generate `model.onnx`. 4 | -------------------------------------------------------------------------------- /onnx/test_cases/byte_sb_bidi_lstm/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/byte_sb_bidi_lstm/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/byte_sb_bidi_lstm/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/byte_sb_bidi_lstm/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/deconv_group/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/deconv_group/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/deconv_group/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/deconv_group/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/deconv_group/vars.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/deconv_group/vars.sh -------------------------------------------------------------------------------- /onnx/test_cases/lgbm_classifier_tensor/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/lgbm_classifier_tensor/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/lgbm_classifier_tensor/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/lgbm_classifier_tensor/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/lgbm_classifier_tensor/vars.sh: -------------------------------------------------------------------------------- 1 | IGNORE= 2 | OPTIONS="--output-node probabilities" 3 | -------------------------------------------------------------------------------- /onnx/test_cases/lgbm_regressor_tensor/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/lgbm_regressor_tensor/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/lgbm_regressor_tensor/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/lgbm_regressor_tensor/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/lgbm_regressor_tensor/vars.sh: -------------------------------------------------------------------------------- 1 | IGNORE="plain decl opti nnef" 2 | -------------------------------------------------------------------------------- /onnx/test_cases/qlstm_3-2-3_T3_S1/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qlstm_3-2-3_T3_S1/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qlstm_3-2-3_T3_S1/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qlstm_3-2-3_T3_S1/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qlstm_3-2-3_T3_S1/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qlstm_3-2-3_T3_S1/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qlstm_3-2-3_T3_S1/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qlstm_3-2-3_T3_S1/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qlstm_3-2-3_T3_S1/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_1/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_1/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_1/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_1/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_1/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_1/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_1/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_1/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_1/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_2/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_2/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_2/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_2/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_2/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_2/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qrelu_2/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qrelu_2/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_1/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_1/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_1/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_1/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_1/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_1/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_1/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_1/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_1/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_2/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_2/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_2/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_2/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_2/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_2/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qsigmoid_2/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qsigmoid_2/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_1/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_1/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_1/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_1/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_1/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_1/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_1/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_1/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_1/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_2/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_2/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_2/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_2/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_2/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_2/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtanh_2/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qtanh_2/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=0 2 | right_context=0 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/qtdnn_10x5_101_i32_biases/final.mdl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtdnn_10x5_101_i32_biases/final.mdl -------------------------------------------------------------------------------- /onnx/test_cases/qtdnn_10x5_101_i32_biases/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtdnn_10x5_101_i32_biases/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/qtdnn_10x5_101_i32_biases/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtdnn_10x5_101_i32_biases/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/qtdnn_10x5_101_i32_biases/model.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/qtdnn_10x5_101_i32_biases/model.raw -------------------------------------------------------------------------------- /onnx/test_cases/qtdnn_10x5_101_i32_biases/vars.sh: -------------------------------------------------------------------------------- 1 | left_context=1 2 | right_context=1 3 | subsampling=1 4 | IGNORE=nnef 5 | -------------------------------------------------------------------------------- /onnx/test_cases/tinyyolov2/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/tinyyolov2/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/tinyyolov2/vars.sh: -------------------------------------------------------------------------------- 1 | CACHE_FILES=tinyyolov2-7-model.onnx 2 | MODEL=$CACHEDIR/tinyyolov2-7-model.onnx 3 | IGNORE="nnef" 4 | -------------------------------------------------------------------------------- /onnx/test_cases/transformer-mlm/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/transformer-mlm/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/transformer-mlm/vars.sh: -------------------------------------------------------------------------------- 1 | CACHE_FILES=transformer-mlm-model.onnx 2 | MODEL=$CACHEDIR/transformer-mlm-model.onnx 3 | IGNORE="nnef" 4 | -------------------------------------------------------------------------------- /onnx/test_cases/xgboost_classifier_tree/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/xgboost_classifier_tree/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/xgboost_classifier_tree/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/xgboost_classifier_tree/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/xgboost_classifier_tree/vars.sh: -------------------------------------------------------------------------------- 1 | IGNORE= 2 | OPTIONS="--output-node probabilities --nnef-tract-onnx" 3 | -------------------------------------------------------------------------------- /onnx/test_cases/xgboost_regressor_tree/io.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/xgboost_regressor_tree/io.npz -------------------------------------------------------------------------------- /onnx/test_cases/xgboost_regressor_tree/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/onnx/test_cases/xgboost_regressor_tree/model.onnx -------------------------------------------------------------------------------- /onnx/test_cases/xgboost_regressor_tree/vars.sh: -------------------------------------------------------------------------------- 1 | IGNORE="plain decl opti nnef" 2 | -------------------------------------------------------------------------------- /post-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | VERSION=$1 4 | . ./.all_crates.sh 5 | 6 | if [ `uname` = "Darwin" ] 7 | then 8 | SED=gsed 9 | else 10 | SED=sed 11 | fi 12 | 13 | if [ -z "$VERSION" ] 14 | then 15 | echo "Usage: $0 " 16 | exit 1 17 | fi 18 | 19 | for path in $ALL_CRATES_PATH 20 | do 21 | crate=$(tomato get package.name $path/Cargo.toml) 22 | echo $crate 23 | tomato set package.version $VERSION $path/Cargo.toml > /dev/null 24 | tomato set workspace.dependencies.$crate.version $VERSION Cargo.toml 25 | done 26 | 27 | git commit . -m "post-release $VERSION" 28 | git push 29 | -------------------------------------------------------------------------------- /pulse-opl/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-pulse-opl" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/snipsco/tract" 8 | keywords = [ "TensorFlow", "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | 13 | [badges] 14 | maintenance = { status = "actively-developed" } 15 | 16 | [dependencies] 17 | downcast-rs.workspace = true 18 | lazy_static.workspace = true 19 | tract-nnef.workspace = true 20 | 21 | [features] 22 | complex = [ "tract-nnef/complex" ] 23 | -------------------------------------------------------------------------------- /pulse-opl/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /pulse/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-pulse" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/snipsco/tract" 8 | keywords = [ "TensorFlow", "NeuralNetworks" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | 13 | [badges] 14 | maintenance = { status = "actively-developed" } 15 | 16 | [dependencies] 17 | downcast-rs.workspace = true 18 | lazy_static.workspace = true 19 | log.workspace = true 20 | tract-pulse-opl.workspace = true 21 | 22 | -------------------------------------------------------------------------------- /pulse/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /pulse/src/ops/array/mask.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::*; 2 | use tract_pulse_opl::ops::PulseMask; 3 | 4 | impl PulsedOp for PulseMask { 5 | fn pulsed_output_facts(&self, inputs: &[&PulsedFact]) -> TractResult> { 6 | Ok(inputs.iter().cloned().cloned().collect()) 7 | } 8 | 9 | as_op!(); 10 | pulsed_op_to_typed_op!(); 11 | } 12 | -------------------------------------------------------------------------------- /pulse/src/ops/array/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::*; 2 | 3 | mod broadcast; 4 | mod concat; 5 | mod mask; 6 | mod pad; 7 | mod slice; 8 | 9 | register_all_mod!(broadcast, concat, pad, slice); 10 | -------------------------------------------------------------------------------- /pulse/src/ops/cnn/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::*; 2 | 3 | mod conv; 4 | mod deconv; 5 | mod pools; 6 | 7 | register_all_mod!(conv, deconv, pools); 8 | -------------------------------------------------------------------------------- /pulse/src/ops/dummy.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::*; 2 | use tract_core::ops::dummy::Dummy; 3 | 4 | impl PulsedOp for Dummy { 5 | fn pulsed_output_facts(&self, _inputs: &[&PulsedFact]) -> TractResult> { 6 | Ok(tvec!()) 7 | } 8 | as_op!(); 9 | pulsed_op_to_typed_op!(); 10 | } 11 | -------------------------------------------------------------------------------- /pulse/src/ops/identity.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | impl PulsedOp for Identity { 4 | fn pulsed_output_facts(&self, inputs: &[&PulsedFact]) -> TractResult> { 5 | Ok(tvec!(inputs[0].clone())) 6 | } 7 | 8 | as_op!(); 9 | pulsed_op_to_typed_op!(); 10 | } 11 | -------------------------------------------------------------------------------- /pulse/src/ops/mask.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/pulse/src/ops/mask.rs -------------------------------------------------------------------------------- /pulse/src/ops/slice.rs: -------------------------------------------------------------------------------- 1 | use crate::internal::*; 2 | use tract_pulse_opl::ops::PulsedAxisSlice; 3 | 4 | impl PulsedOp for PulsedAxisSlice { 5 | fn pulsed_output_facts(&self, inputs: &[&PulsedFact]) -> TractResult> { 6 | let mut fact = inputs[0].clone(); 7 | let stream = fact.stream.as_mut().unwrap(); 8 | stream.delay += self.skip; 9 | stream.dim = self.take.clone(); 10 | Ok(tvec!(fact)) 11 | } 12 | 13 | as_op!(); 14 | pulsed_op_to_typed_op!(); 15 | } 16 | 17 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | use_small_heuristics = "Max" 2 | use_field_init_shorthand = true 3 | use_try_shorthand = true 4 | edition = "2018" 5 | -------------------------------------------------------------------------------- /tensorflow/LICENSE: -------------------------------------------------------------------------------- 1 | ## License 2 | 3 | Licensed under either of 4 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 5 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 6 | at your option. 7 | 8 | ### Contribution 9 | 10 | Unless you explicitly state otherwise, any contribution intentionally submitted 11 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall 12 | be dual licensed as above, without any additional terms or conditions. 13 | -------------------------------------------------------------------------------- /tensorflow/benches/hey_snips_3.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/tensorflow/benches/hey_snips_3.pb -------------------------------------------------------------------------------- /tensorflow/build-proto.rs: -------------------------------------------------------------------------------- 1 | use std::{env, fs, path}; 2 | 3 | fn main() -> std::io::Result<()> { 4 | env::set_var("PROTOC", protobuf_src::protoc()); 5 | let inputs: Vec = { 6 | let mut inputs: Vec = vec![]; 7 | 8 | for dir in &["protos/tensorflow/core/framework", "protos/tensorflow/core/protobuf"] { 9 | for pb in fs::read_dir(dir)? { 10 | inputs.push(pb?.path()) 11 | } 12 | } 13 | 14 | inputs.sort(); 15 | inputs 16 | }; 17 | 18 | let gen = "src/prost"; 19 | let _ = fs::create_dir_all(&gen); 20 | prost_build::Config::new() 21 | .out_dir(gen) 22 | .compile_protos(&inputs, &[path::PathBuf::from("protos/")])?; 23 | 24 | Ok(()) 25 | } 26 | -------------------------------------------------------------------------------- /tensorflow/src/ops/random/mod.rs: -------------------------------------------------------------------------------- 1 | mod philox; 2 | mod random_uniform; 3 | 4 | use crate::model::TfOpRegister; 5 | 6 | pub fn register_all_ops(reg: &mut TfOpRegister) { 7 | reg.insert("RandomUniform", random_uniform::random_uniform); 8 | reg.insert("RandomUniformInt", random_uniform::random_uniform_int); 9 | } 10 | -------------------------------------------------------------------------------- /tensorflow/src/ops/rec/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::model::TfOpRegister; 2 | 3 | pub mod block_lstm; 4 | 5 | pub fn register_all_ops(reg: &mut TfOpRegister) { 6 | reg.insert("BlockLSTM", block_lstm::block_lstm); 7 | } 8 | -------------------------------------------------------------------------------- /tensorflow/src/prost/google.protobuf.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonos/tract/e60cf63dd9a723b5570ce70aa8489d9cce5b63d8/tensorflow/src/prost/google.protobuf.rs -------------------------------------------------------------------------------- /tensorflow/tests/models/plus3.pb: -------------------------------------------------------------------------------- 1 | 2 | ! 3 | input Placeholder* 4 | dtype0 5 | 6 6 | threeConst* 7 | dtype0* 8 | valueB*@@ 9 | $ 10 | outputAddinputthree* 11 | T0 -------------------------------------------------------------------------------- /test-rt/infra/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "infra" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | anyhow.workspace = true 10 | downcast-rs.workspace = true 11 | dyn-clone.workspace = true 12 | env_logger.workspace = true 13 | itertools.workspace = true 14 | lazy_static.workspace = true 15 | tract-core.workspace = true 16 | 17 | [target.'cfg(not(target_family = "wasm"))'.dependencies] 18 | proptest.workspace = true 19 | 20 | [target.'cfg(target_family = "wasm")'.dependencies] 21 | # Wasm doesn't support the `fork` feature of proptest. 22 | proptest = { version = "1.0.0", default-features = false, features = ["std", "bit-set"] } 23 | -------------------------------------------------------------------------------- /test-rt/suite-onnx/simple.txt: -------------------------------------------------------------------------------- 1 | # test_shrink example shape not consistent with network not-nnef 2 | test_expand_shape_model1 input:X 3 | test_expand_shape_model2 input:X 4 | test_expand_shape_model3 input:X 5 | test_expand_shape_model4 input:X 6 | test_shrink since:10 7 | test_sign_model 8 | test_single_relu_model 9 | -------------------------------------------------------------------------------- /test-rt/suite-unit/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "suite-unit" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | tract-core.workspace = true 10 | tract-transformers.workspace = true 11 | infra = { path = "../infra" } 12 | 13 | [target.'cfg(not(target_family = "wasm"))'.dependencies] 14 | proptest.workspace = true 15 | 16 | [target.'cfg(target_family = "wasm")'.dependencies] 17 | # Wasm doesn't support the `fork` feature of proptest. 18 | proptest = { version = "1.0.0", default-features = false, features = ["std", "bit-set"] } 19 | -------------------------------------------------------------------------------- /test-rt/test-blas/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "test-blas" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | 8 | [build-dependencies] 9 | infra = { path = "../infra" } 10 | itertools.workspace = true 11 | lazy_static.workspace = true 12 | suite-onnx = { path = "../suite-onnx" } 13 | suite-unit = { path = "../suite-unit" } 14 | tract-core = { workspace = true, features = [ "blis" ] } 15 | 16 | [dev-dependencies] 17 | infra = { path = "../infra" } 18 | itertools.workspace = true 19 | lazy_static.workspace = true 20 | log.workspace = true 21 | suite-onnx = { path = "../suite-onnx" } 22 | suite-unit = { path = "../suite-unit" } 23 | tract-core = { workspace = true, features = [ "blis" ] } 24 | tract-onnx-opl.workspace = true 25 | -------------------------------------------------------------------------------- /test-rt/test-blas/build.rs: -------------------------------------------------------------------------------- 1 | #[path = "suite.rs"] 2 | mod suite; 3 | 4 | fn main() { 5 | suite::suite().test_runtime( 6 | "as_blas", 7 | "suite::suite()", 8 | "as_blas()", 9 | "Approximation::Approximate", 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /test-rt/test-blas/suite.rs: -------------------------------------------------------------------------------- 1 | use infra::Test; 2 | 3 | pub fn suite() -> &'static infra::TestSuite { 4 | lazy_static::lazy_static! { 5 | static ref SUITE: infra::TestSuite = mk_suite(); 6 | }; 7 | &SUITE 8 | } 9 | 10 | #[allow(clippy::needless_update)] 11 | fn mk_suite() -> infra::TestSuite { 12 | let mut onnx = suite_onnx::suite().clone(); 13 | onnx.ignore(&ignore_onnx); 14 | 15 | let mut unit = suite_unit::suite().unwrap().clone(); 16 | unit.ignore_case(&ignore_unit); 17 | 18 | infra::TestSuite::default().with("onnx", onnx).with("unit", unit) 19 | } 20 | 21 | fn ignore_onnx(_t: &[String]) -> bool { 22 | false 23 | } 24 | 25 | fn ignore_unit(_t: &[String], _tc: &dyn Test) -> bool { 26 | false 27 | } 28 | -------------------------------------------------------------------------------- /test-rt/test-f16/build.rs: -------------------------------------------------------------------------------- 1 | #[path = "suite.rs"] 2 | mod suite; 3 | 4 | fn main() { 5 | suite::suite().test_runtime( 6 | "tests", 7 | "suite::suite()", 8 | "runtime()", 9 | "Approximation::SuperApproximate", 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /test-rt/test-metal/build.rs: -------------------------------------------------------------------------------- 1 | #[path = "ggml_suite.rs"] 2 | mod ggml_suite; 3 | #[path = "suite.rs"] 4 | mod suite; 5 | 6 | fn main() { 7 | suite::suite().test_runtime("mlx", "suite::suite()", "runtime()", "Approximation::Approximate"); 8 | 9 | suite::suite().test_runtime("mfa", "suite::suite()", "runtime()", "Approximation::Approximate"); 10 | 11 | ggml_suite::suite().test_runtime( 12 | "ggml", 13 | "ggml_suite::suite()", 14 | "runtime()", 15 | "Approximation::Approximate", 16 | ); 17 | 18 | ggml_suite::suite().test_runtime( 19 | "none", 20 | "ggml_suite::suite()", 21 | "runtime()", 22 | "Approximation::Approximate", 23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /test-rt/test-nnef-cycle/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "test-nnef-cycle" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | 8 | [build-dependencies] 9 | infra = { path = "../infra" } 10 | itertools.workspace = true 11 | lazy_static.workspace = true 12 | suite-onnx = { path = "../suite-onnx" } 13 | suite-unit = { path = "../suite-unit" } 14 | tract-core.workspace = true 15 | tract-transformers.workspace = true 16 | 17 | [dev-dependencies] 18 | infra = { path = "../infra" } 19 | itertools.workspace = true 20 | lazy_static.workspace = true 21 | log.workspace = true 22 | suite-onnx = { path = "../suite-onnx" } 23 | suite-unit = { path = "../suite-unit" } 24 | tract-core.workspace = true 25 | tract-transformers.workspace = true 26 | tract-onnx-opl.workspace = true 27 | tract-nnef.workspace = true 28 | -------------------------------------------------------------------------------- /test-rt/test-nnef-cycle/build.rs: -------------------------------------------------------------------------------- 1 | #[path = "suite.rs"] 2 | mod suite; 3 | 4 | fn main() { 5 | suite::suite().test_runtime( 6 | "nnef_cycle", 7 | "suite::suite()", 8 | "runtime()", 9 | "Approximation::Approximate", 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /test-rt/test-onnx-core/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let suite = suite_onnx::suite(); 3 | suite.test_runtime("default", "suite_onnx::suite()", "default()", "Approximation::Approximate"); 4 | suite.test_runtime( 5 | "unoptimized", 6 | "suite_onnx::suite()", 7 | "unoptimized()", 8 | "Approximation::Approximate", 9 | ); 10 | suite.test_runtime( 11 | "as_blas", 12 | "suite_onnx::suite()", 13 | "as_blas()", 14 | "Approximation::Approximate", 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /test-rt/test-onnx-core/debug-utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "debug-utils" 3 | version = "0.20.7-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | edition = "2024" 6 | 7 | [workspace] 8 | members = [] 9 | 10 | [dependencies] 11 | protobuf = "*" 12 | tract-onnx.workspace = true 13 | -------------------------------------------------------------------------------- /test-rt/test-onnx-core/debug-utils/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | cargo run -- ../../../.cached/onnx/onnx/backend/test/data/real/test_inception_v1/inception_v1/model.onnx inception_v1_all_outputs.onnx 4 | 5 | virtualenv -p python3 ort 6 | source ./ort/bin/activate 7 | pip install numpy onnx onnxruntime 8 | 9 | python ./save_all.py inception_v1_all_outputs.onnx inception_v1_all_outputs.npz ../../../.cached/onnx/onnx/backend/test/data/real/test_inception_v1/inception_v1/test_data_0.npz 10 | -------------------------------------------------------------------------------- /test-rt/test-onnx-core/include-passing-ignored.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cargo run 4 | -------------------------------------------------------------------------------- /test-rt/test-tflite/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "test-tflite" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | 8 | [build-dependencies] 9 | home.workspace = true 10 | lazy_static.workspace = true 11 | regex.workspace = true 12 | infra = { path = "../infra" } 13 | tract-core.workspace = true 14 | suite-onnx = { path = "../suite-onnx" } 15 | suite-unit = { path = "../suite-unit" } 16 | 17 | [dev-dependencies] 18 | home.workspace = true 19 | regex.workspace = true 20 | lazy_static.workspace = true 21 | log.workspace = true 22 | tflitec.workspace = true 23 | tract-core.workspace = true 24 | tract-tflite.workspace = true 25 | tract-onnx-opl.workspace = true 26 | infra = { path = "../infra" } 27 | suite-onnx = { path = "../suite-onnx" } 28 | suite-unit = { path = "../suite-unit" } 29 | -------------------------------------------------------------------------------- /test-rt/test-tflite/build.rs: -------------------------------------------------------------------------------- 1 | #[path = "suite.rs"] 2 | mod suite; 3 | 4 | fn main() { 5 | suite::suite().test_runtime( 6 | "tests", 7 | "suite::suite()", 8 | "runtime()", 9 | "Approximation::Approximate", 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /test-rt/test-unit-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "test-unit-core" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | tract-core.workspace = true 10 | 11 | [dev-dependencies] 12 | suite-unit = { path = "../suite-unit" } 13 | 14 | [build-dependencies] 15 | suite-unit = { path = "../suite-unit" } 16 | -------------------------------------------------------------------------------- /test-rt/test-unit-core/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let suite = suite_unit::suite().unwrap(); 3 | suite.test_runtime( 4 | "raw", 5 | "suite_unit::suite().unwrap()", 6 | "raw()", 7 | "Approximation::Approximate", 8 | ); 9 | suite.test_runtime( 10 | "decluttered", 11 | "suite_unit::suite().unwrap()", 12 | "decluttered()", 13 | "Approximation::Approximate", 14 | ); 15 | suite.test_runtime( 16 | "optimized", 17 | "suite_unit::suite().unwrap()", 18 | "optimized()", 19 | "Approximation::Approximate", 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /test-rt/test-unit-core/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /tflite/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-tflite" 3 | version = "0.21.14-pre" 4 | authors = ["Mathieu Poumeyrol "] 5 | license = "MIT OR Apache-2.0" 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/snipsco/tract" 8 | edition = "2021" 9 | 10 | [dependencies] 11 | derive-new.workspace = true 12 | flatbuffers.workspace = true 13 | tract-core.workspace = true 14 | 15 | [features] 16 | complex = [] 17 | -------------------------------------------------------------------------------- /transformers/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tract-transformers" 3 | version = "0.21.14-pre" 4 | license = "MIT OR Apache-2.0" 5 | authors = ["Mathieu Poumeyrol ", "Louis Chouraki "] 6 | description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference" 7 | repository = "https://github.com/snipsco/tract" 8 | keywords = [ "TensorFlow", "NeuralNetworks", "Transformers" ] 9 | categories = [ "science" ] 10 | autobenches = false 11 | edition = "2024" 12 | rust-version = "1.85" 13 | 14 | [badges] 15 | maintenance = { status = "actively-developed" } 16 | 17 | [dependencies] 18 | tract-nnef.workspace = true 19 | -------------------------------------------------------------------------------- /yank.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | VERSION=$1 4 | . ./.all_crates.sh 5 | 6 | if [ `uname` = "Darwin" ] 7 | then 8 | SED=gsed 9 | else 10 | SED=sed 11 | fi 12 | 13 | if [ -z "$VERSION" ] 14 | then 15 | echo "Usage: $0 " 16 | exit 1 17 | fi 18 | 19 | for path in $ALL_CRATES_PATH 20 | do 21 | crate=$(tomato get package.name $path/Cargo.toml) 22 | cargo yank --version $VERSION $crate 23 | done 24 | --------------------------------------------------------------------------------