├── .github └── workflows │ ├── build-docs.yaml │ ├── build-lambda.yaml │ ├── pypi.yaml │ └── python-tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── docs └── images │ ├── 68747470733a2f2f64726976652e676f6f676c652e636f6d2f75633f6578706f72743d766965772669643d3141305235795371446e48715939624650677163546f6b347735416a516c666572.png │ ├── Ax.png │ ├── attention.png │ ├── basics.png │ ├── conv_jac.png │ ├── expectation.png │ ├── func_max.png │ ├── hess_ce.png │ ├── hessian.png │ ├── hessian_yaroslaw.png │ ├── l2_grad_w.png │ ├── l2_grad_w_single_step.png │ ├── ts_hybrid.png │ ├── ts_pp.png │ ├── ts_simple.png │ ├── ts_tree.png │ └── uCrOg.png ├── examples ├── main.py └── mnist.py ├── paper ├── .gitignore ├── chapters │ ├── advanced_derivatives.tex │ ├── appendix.tex │ ├── decompositions.tex │ ├── determinant.tex │ ├── functions.tex │ ├── intro.tex │ ├── kronecker.tex │ ├── ml.tex │ ├── simple_derivatives.tex │ ├── special_matrices.tex │ ├── statistics.tex │ ├── tensor_algos.tex │ ├── tensorgrad.tex │ ├── tikz-macros.tex │ └── tikz-styles.tex ├── cookbook.pdf ├── cookbook.tex ├── figures │ ├── Y.pdf │ ├── Y.pdf_tex │ ├── Y.svg │ ├── aXXb.svg │ ├── attention.png │ ├── attention.svg │ ├── backprop.svg │ ├── basic_graph.pdf │ ├── basic_graph.pdf_tex │ ├── basic_graph.svg │ ├── basic_graph.svg.svg │ ├── basics.svg │ ├── canonicals.svg │ ├── chain_rule_broadcast.pdf │ ├── chain_rule_broadcast.pdf_tex │ ├── chainrule2.svg │ ├── contraction_1.tex │ ├── contraction_2.tex │ ├── cos.pdf │ ├── cos.svg │ ├── cubic_expectation.svg │ ├── dTdU.pdf │ ├── dTdU.pdf_tex │ ├── dTdU.svg │ ├── det.svg │ ├── diag.svg │ ├── drawing-1.svg │ ├── drawing.svg │ ├── expectations.svg │ ├── extensive-gave.svg │ ├── flatK.svg │ ├── flip.svg │ ├── fold-unfold.svg │ ├── fourier-function.svg │ ├── frontpage.svg │ ├── functions.svg │ ├── hWX.svg │ ├── isomorph.svg │ ├── kron.pdf │ ├── kron.pdf_tex │ ├── linearityOfExpectation.pdf │ ├── linearityOfExpectation.pdf_tex │ ├── linearityOfExpectation.svg │ ├── linseq.svg │ ├── max-function.svg │ ├── multi_input_functions.pdf │ ├── other_functions.pdf │ ├── path1.pdf │ ├── path1.pdf_tex │ ├── path2.pdf │ ├── path2.pdf_tex │ ├── path3.pdf │ ├── path3.pdf_tex │ ├── path8.pdf │ ├── path8.pdf_tex │ ├── power.pdf │ ├── power.pdf_tex │ ├── product.pdf │ ├── product.pdf_tex │ ├── product.svg │ ├── recursion.svg │ ├── se_answer.svg │ ├── se_answer2.svg │ ├── se_answer3.svg │ ├── shampoo.svg │ ├── steins.pdf │ ├── steins.pdf_tex │ ├── steins.svg │ ├── strassen.svg │ └── traceab.svg ├── old_examples.tex └── references.bib ├── pyproject.toml ├── scripts ├── analyze_profile.py ├── benchmark.py ├── benchmark_to_numpy.py ├── profile_ops.py ├── profile_test.py ├── run_profile.py └── simple_benchmark.py ├── server ├── Dockerfile ├── __init__.py ├── drawTensors.py ├── index.html ├── pyproject.toml └── tests │ ├── __init__.py │ ├── pytest.ini │ ├── test_app.py │ ├── test_docker.py │ ├── test_docker_debug.py │ └── test_docker_quick.py ├── tensorgrad ├── __init__.py ├── extras │ ├── __init__.py │ ├── _convolution.py │ ├── evaluate.py │ ├── expectation.py │ ├── polynomials.py │ ├── to_formula_old.py │ ├── to_index.py │ ├── to_latex.py │ ├── to_numpy.py │ ├── to_numpy_optimization_notes.md │ ├── to_numpy_optimized.py │ ├── to_pytorch.py │ └── to_tikz.py ├── functions.py ├── imgtools.py ├── tensor.py ├── testutils.py └── utils.py ├── tests ├── extras │ ├── test_evaluate.py │ ├── test_expectation.py │ ├── test_polynomials.py │ ├── test_tikz.py │ ├── test_to_index.py │ ├── test_to_latex.py │ └── test_to_numpy_optimized.py ├── test_functions.py ├── test_grads.py ├── test_graph.py ├── test_isomorphism.py ├── test_jacobians.py ├── test_matrix_calc.py ├── test_ml.py ├── test_powers.py ├── test_product.py ├── test_rename.py ├── test_simplify.py ├── test_tensor.py ├── test_to_code.py └── test_variable.py └── uv.lock /.github/workflows/build-docs.yaml: -------------------------------------------------------------------------------- 1 | name: Deploy API Documentation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main # or your primary branch 7 | 8 | jobs: 9 | deploy-docs: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | # 1. Checkout the repository. 14 | - name: Checkout repository 15 | uses: actions/checkout@v3 16 | 17 | # 2. Set up Python. 18 | - name: Set up Python 3.x 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: '3.x' 22 | 23 | # 3. Install pydoctor. 24 | - name: Install pdoc 25 | run: | 26 | #pip install pydoctor 27 | pip install . # Install deps 28 | pip install pdoc 29 | 30 | # 4. Build the API documentation using pydoctor. 31 | # This command tells pydoctor to generate HTML docs in docs/api for the module tensorgrad. 32 | - name: Build API Documentation with Pdoc 33 | run: | 34 | # pydoctor --html-output=docs/api tensorgrad/ 35 | pdoc tensorgrad -o docs/api 36 | 37 | - name: Deploy API Documentation to GitHub Pages 38 | uses: peaceiris/actions-gh-pages@v3 39 | with: 40 | github_token: ${{ secrets.GITHUB_TOKEN }} 41 | publish_dir: docs/api 42 | publish_branch: gh-pages 43 | destination_dir: docs/api 44 | -------------------------------------------------------------------------------- /.github/workflows/build-lambda.yaml: -------------------------------------------------------------------------------- 1 | name: Update AWS Lambda with ECR Image 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | update_lambda: 10 | runs-on: ubuntu-latest 11 | env: 12 | AWS_REGION: us-east-1 13 | LAMBDA_FUNCTION_NAME: drawTensors 14 | ECR_REPOSITORY: tensorgrad 15 | TEST_PAYLOAD: > 16 | {"code": 17 | "i = sp.symbols('i'); 18 | x = tg.Delta(i, 'i', 'j'); 19 | y = x * 2; 20 | save_steps(y); 21 | "} 22 | steps: 23 | # Checkout the repository (fetches Dockerfile and code) 24 | - name: Checkout code 25 | uses: actions/checkout@v3 26 | 27 | # Build the Docker image locally using Docker Buildx 28 | - name: Build Docker image 29 | run: | 30 | docker buildx build -t tensorgrad -f server/Dockerfile . 31 | 32 | # Test the Docker image by running it locally and invoking the function 33 | - name: Test Lambda container locally 34 | run: | 35 | docker run -d --name lambda_local_test -p 9000:8080 \ 36 | -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ 37 | -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ 38 | tensorgrad 39 | sleep 5 40 | curl --fail -X POST \ 41 | "http://localhost:9000/2015-03-31/functions/function/invocations" \ 42 | -H "Content-Type: application/json" \ 43 | -d "$TEST_PAYLOAD" 44 | docker stop lambda_local_test 45 | 46 | # Configure AWS credentials (from GitHub Secrets) for AWS CLI and Docker actions 47 | - name: Configure AWS credentials 48 | uses: aws-actions/configure-aws-credentials@v4 49 | with: 50 | aws-region: ${{ env.AWS_REGION }} 51 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 52 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 53 | 54 | # Retrieve AWS account ID dynamically (avoids hard-coding the account ID in the workflow) 55 | - name: Retrieve AWS Account ID 56 | id: get-account-id 57 | run: | 58 | ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) 59 | echo "ACCOUNT_ID=$ACCOUNT_ID" >> $GITHUB_ENV 60 | 61 | # Log in to Amazon ECR (authenticates Docker to the AWS ECR registry) 62 | - name: Login to Amazon ECR 63 | id: login-ecr 64 | uses: aws-actions/amazon-ecr-login@v2 65 | 66 | - name: Set ECR_REGISTRY environment variable 67 | run: echo "ECR_REGISTRY=${{ steps.login-ecr.outputs.registry }}" >> $GITHUB_ENV 68 | 69 | # Set up Docker Buildx (builder) for multi-architecture image building 70 | - name: Set up Docker Buildx 71 | uses: docker/setup-buildx-action@v2 72 | 73 | # Build and push the Docker image for ARM64 with caching enabled. 74 | - name: Build and push Docker image (ARM64) with caching 75 | uses: docker/build-push-action@v6 76 | with: 77 | context: . 78 | file: server/Dockerfile 79 | platforms: linux/arm64 80 | push: true 81 | tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest 82 | cache-from: type=gha,scope=update-lambda-cache 83 | cache-to: type=gha,scope=update-lambda-cache,mode=max,ignore-error=true 84 | provenance: false 85 | 86 | # Update the Lambda function to use the new image from ECR 87 | - name: Update Lambda function code 88 | run: | 89 | aws lambda update-function-code \ 90 | --function-name ${{ env.LAMBDA_FUNCTION_NAME }} \ 91 | --image-uri ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest 92 | 93 | # (Optional) Invoke the Lambda function and fetch logs to verify the new code 94 | - name: Invoke Lambda function (and fetch logs) 95 | run: | 96 | echo "Invoking function '${{ env.LAMBDA_FUNCTION_NAME }}' and printing logs:" 97 | aws lambda invoke \ 98 | --function-name ${{ env.LAMBDA_FUNCTION_NAME }} \ 99 | --cli-binary-format raw-in-base64-out \ 100 | --payload "$TEST_PAYLOAD" /tmp/response.json \ 101 | --log-type Tail \ 102 | --query 'LogResult' \ 103 | --output text \ 104 | | base64 -d 105 | 106 | # (Optional) Test the deployed API endpoint via cURL 107 | - name: Test API Gateway endpoint 108 | run: | 109 | sleep 5 110 | echo "Testing API Gateway endpoint for function..." 111 | curl -f -X POST \ 112 | https://4kqy5zmzdi3aghjn32orugt7vm0kgzts.lambda-url.us-east-1.on.aws/execute \ 113 | -H 'Content-Type: application/json' \ 114 | -d "$TEST_PAYLOAD" 115 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Publish to PyPI 2 | 3 | on: 4 | # Trigger when the python-tests workflow completes. 5 | workflow_run: 6 | workflows: ["python-tests"] 7 | types: [completed] 8 | # Also allow manual triggering. 9 | workflow_dispatch: 10 | inputs: 11 | force: 12 | description: 'Force publish regardless of version check' 13 | required: false 14 | default: 'false' 15 | 16 | jobs: 17 | build-and-publish: 18 | # Only run if the triggering workflow (python-tests) succeeded. 19 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - name: Checkout repository 24 | uses: actions/checkout@v2 25 | with: 26 | # Check out the commit that triggered the tests. 27 | ref: ${{ github.event.workflow_run.head_sha }} 28 | 29 | - name: Check if new version compared to PyPI or force publish 30 | id: version_check 31 | run: | 32 | PACKAGE_NAME="tensorgrad" # Update this if your PyPI package name is different. 33 | FORCE="${{ github.event.inputs.force }}" 34 | if [ "$FORCE" = "true" ]; then 35 | echo "Force publish enabled." 36 | echo "version_changed=true" >> $GITHUB_OUTPUT 37 | else 38 | # Extract the version from pyproject.toml (expects a line like: version = "1.2.3") 39 | current_version=$(grep '^version' pyproject.toml | head -n 1 | cut -d'"' -f2) 40 | echo "Current version in pyproject.toml: $current_version" 41 | # Get the latest version from PyPI using its JSON API. 42 | latest_pypi_version=$(curl -s "https://pypi.org/pypi/${PACKAGE_NAME}/json" | jq -r '.info.version') 43 | # Default to 0.0.0 if nothing is returned. 44 | if [ "$latest_pypi_version" = "null" ] || [ -z "$latest_pypi_version" ]; then 45 | latest_pypi_version="0.0.0" 46 | fi 47 | echo "Latest version on PyPI: $latest_pypi_version" 48 | # Compare versions using dpkg (ensures proper version ordering). 49 | if dpkg --compare-versions "$current_version" gt "$latest_pypi_version"; then 50 | echo "New version detected, ready to publish." 51 | echo "version_changed=true" >> $GITHUB_OUTPUT 52 | else 53 | echo "No new version detected. Skipping publish." 54 | echo "version_changed=false" >> $GITHUB_OUTPUT 55 | fi 56 | fi 57 | shell: bash 58 | 59 | - name: Set up Python 60 | if: steps.version_check.outputs.version_changed == 'true' 61 | uses: actions/setup-python@v2 62 | with: 63 | python-version: '3.x' 64 | 65 | - name: Upgrade pip and install build tools 66 | if: steps.version_check.outputs.version_changed == 'true' 67 | run: | 68 | python -m pip install --upgrade pip 69 | pip install build twine 70 | 71 | - name: Build the package 72 | if: steps.version_check.outputs.version_changed == 'true' 73 | run: python -m build 74 | 75 | - name: Publish package to PyPI 76 | if: steps.version_check.outputs.version_changed == 'true' 77 | env: 78 | TWINE_USERNAME: __token__ 79 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 80 | run: python -m twine upload dist/* 81 | -------------------------------------------------------------------------------- /.github/workflows/python-tests.yml: -------------------------------------------------------------------------------- 1 | name: Run Pytest on Push 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v3 16 | 17 | - name: Set up Python 18 | uses: actions/setup-python@v4 19 | with: 20 | python-version: '3.11' 21 | 22 | - name: Install Python dependencies 23 | run: | 24 | pip install . 25 | pip install pytest 26 | 27 | - name: Run Tests 28 | run: | 29 | pytest tests/ --maxfail=5 --disable-warnings 30 | 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | output_files/ 2 | *.swp 3 | media/ 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | cover/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | .pybuilder/ 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | # For a library or package, you might want to ignore these files since the code is 91 | # intended to run in multiple environments; otherwise, check them in: 92 | # .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # poetry 102 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 103 | # This is especially recommended for binary packages to ensure reproducibility, and is more 104 | # commonly ignored for libraries. 105 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 106 | #poetry.lock 107 | 108 | # pdm 109 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 110 | #pdm.lock 111 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 112 | # in version control. 113 | # https://pdm.fming.dev/#use-with-ide 114 | .pdm.toml 115 | 116 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 117 | __pypackages__/ 118 | 119 | # Celery stuff 120 | celerybeat-schedule 121 | celerybeat.pid 122 | 123 | # SageMath parsed files 124 | *.sage.py 125 | 126 | # Environments 127 | .env 128 | .venv 129 | env/ 130 | venv/ 131 | ENV/ 132 | env.bak/ 133 | venv.bak/ 134 | 135 | # Spyder project settings 136 | .spyderproject 137 | .spyproject 138 | 139 | # Rope project settings 140 | .ropeproject 141 | 142 | # mkdocs documentation 143 | /site 144 | 145 | # mypy 146 | .mypy_cache/ 147 | .dmypy.json 148 | dmypy.json 149 | 150 | # Pyre type checker 151 | .pyre/ 152 | 153 | # pytype static type analyzer 154 | .pytype/ 155 | 156 | # Cython debug symbols 157 | cython_debug/ 158 | 159 | # PyCharm 160 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 161 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 162 | # and can be added to the global gitignore or merged into this file. For a more nuclear 163 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 164 | #.idea/ 165 | .vscode/settings.json 166 | iso.py 167 | .aider* 168 | 169 | # 170 | .DS_Store 171 | combined_image.png 172 | data/ 173 | out 174 | examples/mnist_torch.py 175 | steps.png 176 | *.prof 177 | -------------------------------------------------------------------------------- /docs/images/68747470733a2f2f64726976652e676f6f676c652e636f6d2f75633f6578706f72743d766965772669643d3141305235795371446e48715939624650677163546f6b347735416a516c666572.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/68747470733a2f2f64726976652e676f6f676c652e636f6d2f75633f6578706f72743d766965772669643d3141305235795371446e48715939624650677163546f6b347735416a516c666572.png -------------------------------------------------------------------------------- /docs/images/Ax.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/Ax.png -------------------------------------------------------------------------------- /docs/images/attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/attention.png -------------------------------------------------------------------------------- /docs/images/basics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/basics.png -------------------------------------------------------------------------------- /docs/images/conv_jac.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/conv_jac.png -------------------------------------------------------------------------------- /docs/images/expectation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/expectation.png -------------------------------------------------------------------------------- /docs/images/func_max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/func_max.png -------------------------------------------------------------------------------- /docs/images/hess_ce.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/hess_ce.png -------------------------------------------------------------------------------- /docs/images/hessian.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/hessian.png -------------------------------------------------------------------------------- /docs/images/hessian_yaroslaw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/hessian_yaroslaw.png -------------------------------------------------------------------------------- /docs/images/l2_grad_w.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/l2_grad_w.png -------------------------------------------------------------------------------- /docs/images/l2_grad_w_single_step.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/l2_grad_w_single_step.png -------------------------------------------------------------------------------- /docs/images/ts_hybrid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/ts_hybrid.png -------------------------------------------------------------------------------- /docs/images/ts_pp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/ts_pp.png -------------------------------------------------------------------------------- /docs/images/ts_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/ts_simple.png -------------------------------------------------------------------------------- /docs/images/ts_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/ts_tree.png -------------------------------------------------------------------------------- /docs/images/uCrOg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/docs/images/uCrOg.png -------------------------------------------------------------------------------- /examples/mnist.py: -------------------------------------------------------------------------------- 1 | from torchvision import datasets, transforms 2 | from torch.utils.data import DataLoader 3 | from sympy import symbols 4 | import tqdm 5 | import torch 6 | import argparse 7 | 8 | from tensorgrad import Variable 9 | from tensorgrad.testutils import rand_values 10 | from tensorgrad.extras.to_pytorch import compile_to_callable as compile_torch 11 | from tensorgrad.extras.to_numpy import compile_to_callable as compile_numpy 12 | import tensorgrad.functions as F 13 | 14 | 15 | def main(args): 16 | n_epochs = 10 17 | batch_size = 32 18 | lr = 1e-2 19 | 20 | # Load data 21 | transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))]) 22 | mnist_args = dict(root="./data", download=True, transform=transform) 23 | train_dataset = datasets.MNIST(train=True, **mnist_args) 24 | train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) 25 | # test_dataset = datasets.MNIST(train=False, **mnist_args) 26 | # test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) 27 | 28 | # Create model 29 | batch, c0, w0, h0, out, kernel_size = symbols("batch c0 w0 h0 out ks") 30 | data = Variable("data", batch, c0, w0, h0) 31 | targets = Variable("targets", batch, out) 32 | shapes = { 33 | batch: batch_size, 34 | c0: 1, 35 | w0: 28, 36 | h0: 28, 37 | kernel_size: 3, 38 | out: 10, 39 | } 40 | 41 | layers = [] 42 | 43 | def conv_layer(channels: int): 44 | # Declare heigth and weidth convolutions 45 | i = len(layers) 46 | c_in, c_out, h_in, h_out, w_in, w_out = symbols(f"c{i} c{i+1} h{i} h{i+1}, w{i} w{i+1}") 47 | h_conv = F.Convolution(h_in, h_out, hk=kernel_size) 48 | w_conv = F.Convolution(w_in, w_out, wk=kernel_size) 49 | kernel = Variable(f"kernel_{i}", c_in, c_out, hk=kernel_size, wk=kernel_size) 50 | # Save the layer and shapes of the inner dimensions 51 | layers.append(kernel) 52 | shapes[c_out] = channels 53 | shapes[h_out] = shapes[h_in] - shapes[kernel_size] + 1 54 | shapes[w_out] = shapes[w_in] - shapes[kernel_size] + 1 55 | # Apply the convolution 56 | return kernel @ h_conv @ w_conv 57 | 58 | # Build the mode 59 | x = data 60 | 61 | if args.model == "conv-2": 62 | x = F.relu(x @ conv_layer(channels=2)).simplify() 63 | x = F.relu(x @ conv_layer(channels=3)).simplify() 64 | c2, h2, w2 = symbols("c2 h2 w2") 65 | layers.append(linear := Variable("lin", c2, h2, w2, out)) 66 | logits = x @ linear 67 | 68 | elif args.model == "conv-1": 69 | x = F.relu(x @ conv_layer(channels=2)).simplify() 70 | c1, h1, w1 = symbols("c1 h1 w1") 71 | layers.append(linear := Variable("lin", c1, h1, w1, out)) 72 | logits = x @ linear 73 | 74 | elif args.model == "linear-2": 75 | shapes[mid := symbols("mid")] = 40 76 | layers.append(linear1 := Variable("lin1", c0, h0, w0, mid)) 77 | layers.append(linear2 := Variable("lin2", mid, out)) 78 | x = F.relu(x @ linear1) 79 | logits = x @ linear2 80 | 81 | elif args.model == "linear-1": 82 | layers.append(linear := Variable("lin", c0, w0, h0, out)) 83 | logits = x @ linear 84 | 85 | # y = F.cross_entropy(logits, targets, dim='out') 86 | logits = logits.full_simplify(expand=False) 87 | y = F.mean((logits - targets) ** 2, dim="out") 88 | y = F.mean(y, dim="batch") 89 | # y = y.full_simplify() 90 | prediction = F.argmax(logits, dim="out") 91 | 92 | print("Computing and simplifying gradients") 93 | grad_tensors = [y.grad(param).full_simplify(expand=False) for param in layers] 94 | 95 | compile_func = compile_numpy if args.backend == "numpy" else compile_torch 96 | backprop = compile_func(prediction, y, *grad_tensors, verbose=True, torch_compile=True) 97 | 98 | # Train 99 | print("Training...") 100 | parameters = rand_values(layers, shapes) 101 | parameters = {s: t / sum(t.shape) ** 0.5 for s, t in parameters.items()} 102 | for _ in range(n_epochs): 103 | total_loss = 0 104 | corr = 0 105 | batches = 0 106 | for t_data, t_target in tqdm.tqdm(train_loader): 107 | shapes[batch] = t_data.shape[0] 108 | input_params = {t: p.clone() for t, p in parameters.items()} 109 | input_params[data] = t_data.rename("batch", "c0", "w0", "h0") 110 | input_params[targets] = torch.eye(10)[t_target].rename("batch", "out") 111 | 112 | # Forward and backward pass 113 | pred_out, y_out, *grad_outputs = backprop(input_params, shapes) 114 | 115 | # Grad update 116 | for layer, grad in zip(layers, grad_outputs): 117 | g = grad.align_to(*parameters[layer].names) 118 | parameters[layer] -= lr * g 119 | 120 | # Forward pass 121 | total_loss += y_out / shapes[batch] 122 | corr += (pred_out == t_target).sum() / shapes[batch] 123 | batches += 1 124 | 125 | print(f"Loss: {total_loss/batches}") 126 | print(f"Acc: {corr/batches}") 127 | 128 | 129 | if __name__ == "__main__": 130 | parser = argparse.ArgumentParser() 131 | parser.add_argument( 132 | "model", type=str, default="linear-1", 133 | choices=["conv-1", "conv-2", "linear-1", "linear-2"] 134 | ) 135 | parser.add_argument("--backend", type=str, default="torch", choices=["torch", "numpy"]) 136 | args = parser.parse_args() 137 | main(args) 138 | -------------------------------------------------------------------------------- /paper/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /paper/chapters/advanced_derivatives.tex: -------------------------------------------------------------------------------- 1 | 2 | \chapter{Advanced Derivatives} 3 | 4 | \section{Derivatives of vector norms} 5 | 6 | \subsection{Two-norm} 7 | \begin{align} 8 | \frac{d}{dx}\|x-a\|_2 &= \frac{x-a}{\|x-a\|_2} 9 | \\ 10 | \frac{d}{dx}\frac{x-a}{\|x-a\|_2} &= \frac{I}{\|x-a\|_2} - \frac{(x-a)(x-a)^T}{\|x-a\|_2^3} 11 | \\ 12 | \frac{d}{dx}\|x\|_2^2 = \frac{d}{dx}\|x^T x\|_2 = 2x 13 | \end{align} 14 | 15 | \section{Derivatives of matrix norms} 16 | 17 | \section{Derivatives of Structured Matrices} 18 | 19 | \subsection{Symmetric} 20 | \subsection{Diagonal} 21 | \subsection{Toeplitz} 22 | 23 | \section{Derivatives of a Determinant} 24 | \section{General forms} 25 | \section{Linear forms} 26 | \section{Square forms} 27 | 28 | \section{From Stack Exchange} 29 | 30 | Let $F(t)=|1+t A|$ then 31 | $F'(t) = \mathrm{Tr}(A)$ and 32 | $F''(t) = \mathrm{Tr}(A)^2 - \mathrm{Tr}(A^2)$. 33 | 34 | \section{Derivatives of an Inverse} 35 | 36 | 37 | \subsection{Trace Identities} 38 | $$ 39 | \frac{\partial}{\partial \mathbf{X}} \operatorname{Tr}\left(\mathbf{A} \mathbf{X}^{-1} \mathbf{B}\right)=-\left(\mathbf{X}^{-1} \mathbf{B} \mathbf{A} \mathbf{X}^{-1}\right)^T=-\mathbf{X}^{-T} \mathbf{A}^T \mathbf{B}^T \mathbf{X}^{-T} 40 | $$ 41 | 42 | Assume $\mathbf{B}$ and $\mathbf{C}$ to be symmetric, then 43 | $$ 44 | \begin{aligned} 45 | \frac{\partial}{\partial \mathbf{X}} \operatorname{Tr}\left[\left(\mathbf{X}^T \mathbf{C X}\right)^{-1} \mathbf{A}\right] & =-\left(\mathbf{C X}\left(\mathbf{X}^T \mathbf{C X}\right)^{-1}\right)\left(\mathbf{A}+\mathbf{A}^T\right)\left(\mathbf{X}^T \mathbf{C X}\right)^{-1} \\ 46 | \frac{\partial}{\partial \mathbf{X}} \operatorname{Tr}\left[\left(\mathbf{X}^T \mathbf{C X}\right)^{-1}\left(\mathbf{X}^T \mathbf{B X}\right)\right] & =-2 \mathbf{C X}\left(\mathbf{X}^T \mathbf{C X}\right)^{-1} \mathbf{X}^T \mathbf{B X}\left(\mathbf{X}^T \mathbf{C X}\right)^{-1} \\ 47 | \frac{\partial}{\partial \mathbf{X}} \operatorname{Tr}\left[\left(\mathbf{A}+\mathbf{X}^T \mathbf{C X}\right)^{-1}\left(\mathbf{X}^T \mathbf{B X}\right)\right] & =-2 \mathbf{B X}\left(\mathbf{X}^T \mathbf{C X}\right)^{-1} \\ 48 | & -2 \mathbf{C X}\left(\mathbf{A}+\mathbf{X}^T \mathbf{C X}\right)^{-1} \mathbf{X}^T \mathbf{B X}\left(\mathbf{A}+\mathbf{X}^T \mathbf{C X}\right)^{-1} \\ 49 | & +2 \mathbf{B X}\left(\mathbf{A}+\mathbf{X}^T \mathbf{C X}\right)^{-1} 50 | \end{aligned} 51 | $$ 52 | 53 | \section{Derivatives of Eigenvalues} 54 | 55 | \section{Exercises} 56 | \begin{exercise} 57 | Find the derivative of 58 | \[ 59 | f(x,\Sigma,\mu) = \frac{1}{\sqrt{(2\pi)^k|\Sigma|}} 60 | \exp\left(-\frac12(x-\mu)^T \Sigma^{-1} (x-\mu)\right) 61 | \] 62 | with respect to $\Sigma$. 63 | \end{exercise} 64 | -------------------------------------------------------------------------------- /paper/chapters/appendix.tex: -------------------------------------------------------------------------------- 1 | 2 | \chapter{Appendix} 3 | Contains some proofs, such as of equation 524 or 571. 4 | They are pretty long and could be useful for contrasting with the diagram proofs. 5 | -------------------------------------------------------------------------------- /paper/chapters/ml.tex: -------------------------------------------------------------------------------- 1 | 2 | \chapter{Machine Learning Applications} 3 | 4 | \section{Least Squares} 5 | 6 | \section{Hessian of Cross Entropy Loss} 7 | 8 | \section{Convolutional Neural Networks} 9 | 10 | \section{Transformers / Attention} 11 | 12 | \section{Tensor Sketch} 13 | 14 | \section{Reinforcement Learning} 15 | 16 | When considering imperfect information games, 17 | Let's try to approximate each player's strategy with a low rank tensor. 18 | We can use the tensor sketch to approximate the tensor. 19 | Basically that means picking a random outer product of vectors and multiplying on the derivative. 20 | This should be fast... 21 | -------------------------------------------------------------------------------- /paper/chapters/tensor_algos.tex: -------------------------------------------------------------------------------- 1 | 2 | \chapter{Tensor Algorithms} 3 | 4 | \section{Tensor Contraction Orders}\label{sec:opt_contr} 5 | 6 | Throughout the previous chapters, we drew a \emph{lot} of tensor networks. While the theoretical aspect is sufficient for some applications, there are indeed research areas, mostly computational ones such as quantum circuit simulation, where one wants to actually compute the tensor underlying the tensor network. 7 | This is done by the so-called \emph{tensor contractions}. 8 | To see what this means, consider the following 2-tensor diagram, where we contract two tensors $A$ and $B$: 9 | \begin{center} 10 | \resizebox{0.44\textwidth}{!}{ 11 | \input{figures/contraction_1} 12 | } 13 | \end{center} 14 | Note that we also write the sizes of the respective dimensions, i.e., $A$ is a $2\times 2\times 4$-tensor, while $B$ is a $4 \times 2$-matrix. 15 | Now, the \emph{contraction cost} is defined as the number of FLOPS performed during the contraction; as a convention, this is the number of scalar multiplications.\footnote{This assumes a naive implementation of the operation, i.e., nested loops over the dimensions.} 16 | In our example, the contraction cost is $2 \times 2 \times 4 \times 2 = 32$, i.e., we simply multiply the dimension sizes. 17 | 18 | The previous example was rather small. 19 | However, tensor networks in the wild tend to have hundreds of tensors. 20 | Naturally, these also need to be contracted to a single tensor. 21 | Now comes the interesting part: The \emph{order} in which we perform these contraction can have a tremendous impact on the execution time. 22 | To get an intuition for this, consider an extended example of a 3-tensor diagram: 23 | \begin{center} 24 | \resizebox{0.35\textwidth}{!}{ 25 | \input{figures/contraction_2} 26 | } 27 | \end{center} 28 | If we were to contract $A$ with $B$ first (and the resulting tensor with $C$), we would have a cost of $2^2 \times 4\times 2 + 2^3 \times 1 \times 2^2 = 32 + 32 = 64$ multiplications, whereas performing the contraction between $B$ and $C$ at the beginning would result in $2^3 \times 1 \times 2^2 + 2^2 \times 4 \times 2^3 = 32 + 128 = 160$ multiplications in total. Hence, the first order is much better. 29 | 30 | It is thus natural to ask: \emph{Can we always find the optimal contraction order?} 31 | 32 | \subsection{Algorithms} 33 | 34 | We summarize well-known results and frameworks to find good contraction orders. 35 | 36 | \subsubsection{Optimal Algorithms} 37 | 38 | Indeed, finding the \emph{optimal} contraction order for arbitrary tensor network shapes is pretty hard; better said, NP-hard~\cite{ordering_np_hard}. There is a well-known exact algorithm running in $O(3^n)$-time~\cite{ordering_exact_algo}, where $n$ is the number of tensors in the network. This finds the optimal contraction order with respect to the total contraction cost. If one is interested in minimizing the size of the largest intermediate tensor, i.e., to optimize for the memory used during the execution, this can be done faster in $O(2^n n^3)$-time~\cite{dpconv}. 39 | 40 | The good news is that for some restricted shapes of tensor networks, there are indeed efficient algorithms. A classic example is that dynamic programming solution for the matrix-chain problem~\cite{cormen}, which is just \emph{our} problem, but only for matrices. The naive algorithm runs in $O(n^3)$-time, but can be implemented in $O(n^2)$-time~\cite{mc_yao} (or even in $O(n \log n)$-time~\cite{mc_nlogn_1, mc_nlogn_2}). Another shape for which polynomial-time algorithms exist is that of \emph{tree} tensor networks~\cite{ttn_size, ttn_cost}. 41 | 42 | Another prominent way to optimize contraction orders is via the tree decomposition of the \emph{line graph} representation of the tensor network~\cite{markov_shi, dudek_tree_decomp, roman_tree_decomp}. In particular, this results in a contraction order with a maximal intermediate tensor rank equal to the treewidth of the tree decomposition. Loosely speaking, treewidth measures how tree-like a graph is. This does not directly solve our problem since finding the tree decompositions of the smallest treewidth is itself hard~\cite{treewidth_approx}. Two well-known frameworks to find good tree decompositions are \texttt{QuickBB}~\cite{quick_bb} and \texttt{FlowCutter}~\cite{flow_cutter_1, flow_cutter_2}. 43 | 44 | \subsubsection{Best-Effort Algorithms} 45 | 46 | % TODO: @Thomas, refer to the `einsum` section. 47 | However, once we want to contract \emph{arbitrary} network shapes, the best we can do is to fall back on heuristics or approximations. Two well-known frameworks are \texttt{opt\_einsum}~\cite{opt_einsum} and \texttt{cotengra}~\cite{cotengra}, which aim to optimize the contraction order (also referred to as ``contraction path'') of arbitrary einsums: For tensor networks where the optimal algorithm would be too slow, \texttt{opt\_einsum} applies an ad-hoc greedy algorithm, while \texttt{cotengra} uses hypergraph partitioning~\cite{kahypar}, along with a Bayesian optimization approach, which has been later refined~\cite{jena_tn_cut}. Other algorithms adopted from database query optimization are implemented in \texttt{netzwerk}~\cite{ttn_cost}. Another option is to \emph{learn} the best contraction orders, e.g., using reinforcement learning~\cite{meirom_tn_rl}. 48 | 49 | Naturally, the above heuristics do not come with an optimality guarantee. There exists a $(1+\eps)$-approximation $O^*(2^n / \eps)$-time algorithm that minimizes the sum of the intermediate tensor sizes~\cite{dpconv}. 50 | 51 | Worth mentioning is the SQL-view of einsum~\cite{einsum_as_sql}: It allows to perform the entire tensor network contraction on any database engine. Indeed, for some einsum classes, running the contractions on state-of-the-art database engines, such as Hyper~\cite{hyper}, can be much faster than using the de-facto \texttt{numpy}-array implementation. 52 | -------------------------------------------------------------------------------- /paper/chapters/tikz-macros.tex: -------------------------------------------------------------------------------- 1 | 2 | \pgfmathsetmacro{\outerRadius}{.7} 3 | \pgfmathsetmacro{\innerRadius}{.3} 4 | 5 | \newcounter{csvcounter} 6 | \newcommand{\CountCSV}[2]{% 7 | \setcounter{csvcounter}{0}% 8 | \foreach \x in #2 {% 9 | \stepcounter{csvcounter}% 10 | }% 11 | \pgfmathsetmacro{#1}{\value{csvcounter}}% 12 | } 13 | \newcommand{\drawPartitionAtAngle}[5][]{% 14 | \def\globaln{#3}% 15 | \begin{scope}[shift={#2}]% 16 | \node[label, anchor=east] at (-\outerRadius,0) {#4};% 17 | \CountCSV{\blockCount}{#5}% 18 | 19 | % First optimize the assignment of blocks to hub positions 20 | \def\bestTotalDist{1000}% 21 | \def\bestOffset{0}% 22 | 23 | % Try different rotational offsets for the hub arrangement 24 | \foreach \offset in {42, 62, 82} {% 25 | \pgfmathsetmacro{\totalDist}{0}% 26 | % For each block in the partition: 27 | \foreach [count=\blockIndex from 0] \block in #5 {% 28 | \pgfmathsetmacro{\angleStep}{30+360/\blockCount}% 29 | \pgfmathsetmacro{\hubAngle}{mod(\blockIndex * \angleStep + \offset, 360)}% 30 | % For each element in this block: 31 | \foreach \elem in \block {% 32 | \pgfmathsetmacro{\elemAngle}{mod((\elem - 1)*360/\globaln, 360)}% 33 | \pgfmathsetmacro{\dist}{min(abs(mod(\elemAngle - \hubAngle,360)),360 - abs(mod(\elemAngle - \hubAngle,360)))}% 34 | % Update the accumulator globally: 35 | \pgfmathparse{\totalDist + \dist/360}% 36 | \global\edef\totalDist{\pgfmathresult}% 37 | }% 38 | }% 39 | \ifdim\totalDist pt < \bestTotalDist pt% 40 | \xdef\bestTotalDist{\totalDist}% 41 | \xdef\bestOffset{\offset}% 42 | \fi% 43 | }% 44 | 45 | % Now draw each block using the optimized hub positions: 46 | \pgfmathsetmacro{\angleStep}{360/\blockCount}% 47 | \ifnum\blockCount=1 48 | \def\hubRadius{0}% 49 | \else 50 | \def\hubRadius{\innerRadius}% 51 | \fi 52 | \foreach [count=\i from 0] \block in #5 {% 53 | \pgfmathsetmacro{\currentAngle}{\i * \angleStep + \bestOffset}% 54 | \drawBlock[#1]{\currentAngle:\hubRadius}{\block}{\i}% 55 | }% 56 | \end{scope}% 57 | } 58 | \newcommand{\drawBlock}[3][]{% 59 | \coordinate (hub) at (#2);% 60 | % For each element in this block, draw a spoke: 61 | \foreach \elem in #3 {% 62 | \pgfmathsetmacro{\thisAngle}{30+(\elem - 1)*360/\globaln}% 63 | \path (hub) (\thisAngle:\outerRadius) coordinate (node\elem);% 64 | \draw[thin] (hub) -- (node\elem);% 65 | }% 66 | \if\relax\detokenize{#1}\relax 67 | \node[inner sep=1pt, circle, fill, minimum size=2pt] at (hub) {};% 68 | \else 69 | \CountCSV{\blockSize}{#3}% 70 | \node[label, inner sep=0pt, fill=white] at (hub) {\scriptsize$K_{\blockSize}$};% 71 | \fi 72 | } 73 | -------------------------------------------------------------------------------- /paper/chapters/tikz-styles.tex: -------------------------------------------------------------------------------- 1 | \usetikzlibrary{ 2 | arrows.meta, 3 | automata, 4 | positioning, 5 | shadows, 6 | calc, 7 | math, 8 | shapes.geometric, 9 | decorations, 10 | decorations.pathmorphing, 11 | decorations.pathreplacing, 12 | decorations.shapes, 13 | decorations.markings, 14 | graphs 15 | } 16 | 17 | \tikzset{% 18 | dot/.style={ 19 | circle, 20 | inner sep=0mm, 21 | outer sep=0mm, 22 | minimum size=2mm, 23 | draw=black, 24 | fill=black 25 | }, 26 | triple/.style={ 27 | double distance=2pt, 28 | postaction={draw} 29 | }, 30 | quadruple/.style={ 31 | double, 32 | double distance=2pt, 33 | postaction={ 34 | draw, 35 | transform canvas={yshift=-.4pt}, 36 | }, 37 | postaction={ 38 | draw, 39 | transform canvas={yshift=.4pt}, 40 | } 41 | }, 42 | every loop/.style={}, 43 | d/.style={ 44 | Circle[]-, 45 | shorten <=-2pt, 46 | transform canvas={shift={(-3pt, 4pt)}} 47 | }, 48 | d0/.style={ 49 | Circle[]-, 50 | shorten <=-2pt, 51 | }, 52 | d1/.style={ 53 | Circle[]-, 54 | }, 55 | dn/.style={ 56 | draw, circle, minimum size=1.3em 57 | }, 58 | ddn/.style={ 59 | draw, circle, double, minimum size=1.3em 60 | }, 61 | dddn/.style={ 62 | draw, circle, double, minimum size=1.3em, 63 | double distance=2pt, 64 | postaction={draw} 65 | }, 66 | triangle/.style={ 67 | regular polygon, 68 | regular polygon sides=3, 69 | rotate=270, 70 | scale=.5, 71 | inner sep=3pt, 72 | draw 73 | }, 74 | } 75 | 76 | 77 | 78 | \newcommand{\trace}[2]{% #1 is the list of items, #2 is the looseness for the loop 79 | %\pgfmathsetmacro{\negspace}{-1.0em - 0.5em*#2} % Calculate negative space 80 | \hspace{-1em} 81 | \begin{tikzpicture}[baseline=(node1.base), inner sep=1pt] 82 | % Initialize a counter for tracking the number of items 83 | \newcount\itemcount 84 | \itemcount=0 85 | 86 | % Define nodes 87 | \def\lastnode{node1} 88 | \foreach \i [count=\c] in {#1} { 89 | \ifnum \c=1 90 | \node (node\c) {$\i$}; % First node 91 | \else 92 | \node[right=1em of \lastnode] (node\c) {$\i$}; % Subsequent nodes 93 | \draw (\lastnode) -- (node\c); % Draw edge from last node to current 94 | \fi 95 | \global\advance\itemcount by 1 96 | \xdef\lastnode{node\c} 97 | } 98 | 99 | % Draw the loop edge 100 | \ifnum \itemcount=1 101 | \path (node1) edge [out=160, in=20, loop] (); 102 | \else 103 | \path (node1) edge [out=160, in=20, looseness=#2] (\lastnode); 104 | \fi 105 | \end{tikzpicture} 106 | \hspace{-1em} 107 | } 108 | 109 | \def\matmul#1{ 110 | \vecmatvec{.5em}{}{#1}{} 111 | } 112 | 113 | \def\vecmatvec#1#2#3#4{ 114 | \begin{tikzpicture}[baseline=-.25em, inner sep=1pt] 115 | \node (node0) {$#2$}; 116 | \xdef\lastnode{node0}; 117 | \foreach \i [count=\c] in {#3} { 118 | \node[right=#1 of \lastnode] (node\c) {$\i$}; 119 | \draw (\lastnode.east) -- (node\c); 120 | \xdef\lastnode{node\c}; 121 | } 122 | \node[right=#1 of \lastnode] (last) {$#4$}; 123 | \draw (\lastnode.east) -- (last); 124 | \end{tikzpicture} 125 | } 126 | 127 | \def\detstack#1{ 128 | \mathbin{\begin{tikzpicture}[baseline=(a0.base), inner sep=1pt] 129 | \node (a0) {#1}; 130 | \node[right=.5em of a0] (dots) {$\cdots$}; 131 | \node[right=.5em of dots] (a1) {#1}; 132 | \draw (a0.north) -- ++(0,.2) coordinate (a0top); 133 | \draw (a1.north) -- ++(0,.2) coordinate (a1top); 134 | \draw (a0.south) -- ++(0,-.2) coordinate (a0bot); 135 | \draw (a1.south) -- ++(0,-.2) coordinate (a1bot); 136 | \draw[line width=2pt] (a0top -| a0.west) -- (a1top -| a1.east); 137 | \draw[line width=2pt] (a0bot -| a0.west) -- (a1bot -| a1.east); 138 | \end{tikzpicture}} 139 | } 140 | 141 | % Define a new command for drawing the ellipse 142 | \NewDocumentCommand{\drawellipse}{m m m m m o}{ 143 | \def\centerX{#1} 144 | \def\centerY{#2} 145 | \def\widthR{#3} 146 | \def\heightR{#4} 147 | \def\angle{#5} 148 | 149 | \draw (\centerX,\centerY) ellipse [x radius=\widthR, y radius=\heightR]; 150 | \fill ({\centerX + \widthR*cos(\angle)},{\centerY + \heightR*sin(\angle)}) circle [radius=0.075]; 151 | 152 | % If target node is provided, use it, otherwise fall back to default behavior 153 | \IfNoValueTF{#6}{ 154 | \draw ({\centerX + \widthR*cos(\angle)},{\centerY + \heightR*sin(\angle)}) -- ({\centerX + .5 + \widthR*cos(\angle)}, {\centerY + \heightR*sin(\angle)}); 155 | }{ 156 | \draw ({\centerX + \widthR*cos(\angle)},{\centerY + \heightR*sin(\angle)}) -- (#6); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /paper/cookbook.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/cookbook.pdf -------------------------------------------------------------------------------- /paper/cookbook.tex: -------------------------------------------------------------------------------- 1 | \documentclass[oneside]{book} 2 | \usepackage{ifthen} 3 | \usepackage{lipsum} % for generating dummy text 4 | \usepackage{xparse} 5 | \usepackage{amsmath} 6 | \usepackage{amsfonts} 7 | \usepackage{amsthm} 8 | \usepackage{placeins} 9 | \usepackage[T1]{fontenc} 10 | \usepackage[]{hyperref} 11 | \usepackage{import} 12 | \usepackage{graphicx} 13 | \usepackage{adjustbox} 14 | \usepackage{changepage} % for adjustwidth 15 | \usepackage{listings} 16 | \usepackage{xcolor} 17 | \usepackage{makecell} 18 | 19 | \usepackage{geometry} 20 | \geometry{hmargin=4cm,vmargin=4cm} 21 | 22 | \allowdisplaybreaks 23 | 24 | \usepackage{tikz} 25 | \input{chapters/tikz-styles.tex} 26 | \input{chapters/tikz-macros.tex} 27 | 28 | % https://tex.stackexchange.com/a/398211/24956 29 | \usepackage{environ} 30 | \NewEnviron{walign}{% 31 | \noindent 32 | \begin{adjustwidth}{-1em}{-1em} 33 | \begin{align*} 34 | \BODY 35 | \end{align*} 36 | \end{adjustwidth} 37 | } 38 | 39 | \newcommand{\vflip}[1]{% 40 | \ifmmode 41 | \rotatebox[origin=c]{180}{$#1$}% 42 | \else 43 | \rotatebox[origin=c]{180}{#1}% 44 | \fi 45 | } 46 | 47 | \lstset{ 48 | language=Python, 49 | basicstyle=\ttfamily\footnotesize, 50 | keywordstyle=\color{blue}, 51 | commentstyle=\color{gray}, 52 | stringstyle=\color{red}, 53 | showstringspaces=false, 54 | numbers=left, 55 | numberstyle=\tiny\color{gray}, 56 | breaklines=true, 57 | frame=single, 58 | captionpos=b, 59 | tabsize=2, 60 | } 61 | 62 | 63 | \newtheoremstyle{normalstyle} % Name of the style 64 | {3pt} % Space above 65 | {3pt} % Space below 66 | {\normalfont} % Body font 67 | {} % Indent amount 68 | {\bfseries} % Theorem head font 69 | {.} % Punctuation after theorem head 70 | { } % Space after theorem head 71 | {} % Theorem head spec 72 | \theoremstyle{normalstyle} 73 | \newtheorem{exercise}{Exercise} 74 | 75 | \newcommand{\smat}[1]{\left[\begin{smallmatrix}#1\end{smallmatrix}\right]} 76 | \newcommand{\svec}[1]{[\begin{smallmatrix}#1\end{smallmatrix}]} 77 | \newcommand{\E}{\mathrm{E}} 78 | \newcommand{\Tr}{\mathrm{Tr}} 79 | \newcommand{\diag}{\mathrm{diag}} 80 | \newcommand\sbullet[1][1.5pt]{% 81 | \tikz[baseline=-0.5ex]\draw (0,0) circle (#1);% 82 | } 83 | \newcommand{\R}{\mathbb R} 84 | \newcommand{\eps}{\varepsilon} 85 | 86 | 87 | \title{The Tensor Cookbook} 88 | \author{Thomas Dybdahl Ahle} 89 | \begin{document} 90 | 91 | % Consider also taking some inspiration from https://arxiv.org/pdf/1802.01528 92 | % "The Matrix Calculus You Need For Deep Learning" 93 | 94 | \maketitle 95 | 96 | \input{chapters/intro.tex} 97 | %\input{chapters/simple_derivatives.tex} 98 | %\input{chapters/kronecker.tex} 99 | %\input{chapters/functions.tex} 100 | \input{chapters/statistics.tex} 101 | 102 | % \input{chapters/determinant.tex} 103 | % \input{chapters/advanced_derivatives.tex} 104 | %\input{chapters/special_matrices.tex} 105 | % \input{chapters/decompositions.tex} 106 | % \input{chapters/ml.tex} 107 | % \input{chapters/tensor_algos.tex} 108 | % \input{chapters/tensorgrad.tex} 109 | 110 | \bibliographystyle{plain} 111 | \bibliography{references} 112 | 113 | \input{chapters/appendix.tex} 114 | 115 | \end{document} 116 | -------------------------------------------------------------------------------- /paper/figures/Y.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/Y.pdf -------------------------------------------------------------------------------- /paper/figures/Y.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'Y.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{62.63818215bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.89999068)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0,0){\includegraphics[width=\unitlength,page=1]{Y.pdf}}% 57 | \put(-0.00215138,0.3259852){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 58 | \put(0.57341835,-0.00000083){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 59 | \put(0.85865647,0.76912336){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 60 | \end{picture}% 61 | \endgroup% 62 | -------------------------------------------------------------------------------- /paper/figures/Y.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 38 | 40 | 44 | 55 | 59 | 63 | 71 | ffasd 83 | fasdf 94 | Y 105 | Y 116 | Y 127 | 128 | 129 | -------------------------------------------------------------------------------- /paper/figures/attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/attention.png -------------------------------------------------------------------------------- /paper/figures/basic_graph.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/basic_graph.pdf -------------------------------------------------------------------------------- /paper/figures/basic_graph.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'basic_graph.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{143.04609043bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.34822793)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0.1772555,0.11688692){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 57 | \put(0,0){\includegraphics[width=\unitlength,page=1]{basic_graph.pdf}}% 58 | \put(0.53200184,0.22239344){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}A\end{tabular}}}}% 59 | \put(0.53316922,0.03169728){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}C\end{tabular}}}}% 60 | \put(0.7231074,0.2237656){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}B\end{tabular}}}}% 61 | \put(0.72079312,0.03169728){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}D\end{tabular}}}}% 62 | \put(0.90835328,0.13120216){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}E\end{tabular}}}}% 63 | \put(0,0){\includegraphics[width=\unitlength,page=2]{basic_graph.pdf}}% 64 | \put(0.47742391,0.29552701){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{i}\end{tabular}}}}% 65 | \put(0.4351673,0.05423374){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{j}\end{tabular}}}}% 66 | \put(0.51087708,0.13002169){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{k}\end{tabular}}}}% 67 | \put(0.63236484,0.1617146){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{l}\end{tabular}}}}% 68 | \put(0.6358862,0.00325216){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{m}\end{tabular}}}}% 69 | \put(0.76089532,0.11065441){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{n}\end{tabular}}}}% 70 | \put(0.8278016,0.20397107){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{o}\end{tabular}}}}% 71 | \put(0.14340814,0.23401499){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{i}\end{tabular}}}}% 72 | \put(0.08706598,0.1441413){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textit{j}\end{tabular}}}}% 73 | \end{picture}% 74 | \endgroup% 75 | -------------------------------------------------------------------------------- /paper/figures/chain_rule_broadcast.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/chain_rule_broadcast.pdf -------------------------------------------------------------------------------- /paper/figures/contraction_1.tex: -------------------------------------------------------------------------------- 1 | \def\Ax{0} 2 | \def\Bx{2} 3 | \def\Cx{7} 4 | \begin{tikzpicture}[thick, every node/.style={scale=1.5}, tensor/.style={minimum size=0.5cm}] 5 | \node[tensor] (A) at (\Ax,0) {$A$}; 6 | \node[tensor] (B) at (\Bx,0) {$B$}; 7 | \node[tensor] (Aprime) at (\Cx,0) {$A'$}; 8 | 9 | \draw (A.north) -- +(0,0.8) node[midway, right] {\footnotesize 2}; 10 | \draw (A.west) -- +(-0.8,0) node[midway, above] {\footnotesize 2}; 11 | \draw (B.north) -- +(0,0.8) node[midway, right] {\footnotesize 2}; 12 | \draw (A.east) -- (B.west) node[midway, above] {\footnotesize 4}; 13 | \draw (Aprime.north) -- +(0,0.8) node[midway, right] {\footnotesize 2}; 14 | \draw (Aprime.west) -- +(-0.8,0) node[midway, above] {\footnotesize 2}; 15 | 16 | \draw[->] (\Bx + 1.0, 0) -- (\Cx - 2.0, 0) node[midway, above] {\footnotesize Contraction}; 17 | \end{tikzpicture} -------------------------------------------------------------------------------- /paper/figures/contraction_2.tex: -------------------------------------------------------------------------------- 1 | \def\Ax{0} 2 | \def\Bx{2} 3 | \def\Cx{4} 4 | \begin{tikzpicture}[thick, every node/.style={scale=1.5}, tensor/.style={minimum size=0.5cm}] 5 | \node[tensor] (A) at (\Ax,0) {$A$}; 6 | \node[tensor] (B) at (\Bx,0) {$B$}; 7 | \node[tensor] (C) at (\Cx,0) {$C$}; 8 | 9 | \draw (A.north) -- +(0,0.8) node[midway, right] {\footnotesize 2}; 10 | \draw (A.west) -- +(-0.8,0) node[midway, above] {\footnotesize 2}; 11 | \draw (B.north) -- +(0,0.8) node[midway, right] {\footnotesize 2}; 12 | \draw (A.east) -- (B.west) node[midway, above] {\footnotesize 4}; 13 | \draw (C.north) -- +(0,0.8) node[midway, right] {\footnotesize 2}; 14 | \draw (C.east) -- +(+0.8,0.0) node[midway, above] {\footnotesize 2}; 15 | \draw (B.east) -- (C.west) node[midway, above] {\footnotesize 1}; 16 | \end{tikzpicture} -------------------------------------------------------------------------------- /paper/figures/cos.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/cos.pdf -------------------------------------------------------------------------------- /paper/figures/dTdU.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/dTdU.pdf -------------------------------------------------------------------------------- /paper/figures/dTdU.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'dTdU.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{39.98175403bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.97444432)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0.39380832,0.37697177){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 57 | \put(0,0){\includegraphics[width=\unitlength,page=1]{dTdU.pdf}}% 58 | \end{picture}% 59 | \endgroup% 60 | -------------------------------------------------------------------------------- /paper/figures/dTdU.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | T 52 | 56 | 63 | 67 | 71 | 77 | 81 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /paper/figures/drawing-1.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 50 | 55 | 56 | 57 | 61 | 68 | 75 | 82 | Parent 93 | C1 104 | C2 115 | 122 | 129 | 136 | 140 | 145 | 152 | 159 | 166 | 173 | 180 | Contect 191 | Real page 202 | 203 | 204 | -------------------------------------------------------------------------------- /paper/figures/fold-unfold.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | 45 | 49 | 67 | * 78 | 86 | 94 | 102 | 106 | 110 | 128 | * 139 | 143 | X 154 | Y 165 | Fold 176 | Unfold 187 | 188 | 189 | -------------------------------------------------------------------------------- /paper/figures/fourier-function.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 50 | 56 | 57 | 58 | 62 | n 73 | 77 | n 88 | 92 | 100 | exp 111 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /paper/figures/kron.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/kron.pdf -------------------------------------------------------------------------------- /paper/figures/kron.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'kron.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{522.72174985bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.35232485)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0,0){\includegraphics[width=\unitlength,page=1]{kron.pdf}}% 57 | \put(0.00171021,0.24084915){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}a\end{tabular}}}}% 58 | \put(0,0){\includegraphics[width=\unitlength,page=2]{kron.pdf}}% 59 | \put(0.11108915,0.22476398){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}B\end{tabular}}}}% 60 | \put(0,0){\includegraphics[width=\unitlength,page=3]{kron.pdf}}% 61 | \put(0.07731036,0.11216799){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}C\end{tabular}}}}% 62 | \put(0,0){\includegraphics[width=\unitlength,page=4]{kron.pdf}}% 63 | \put(0.22046808,0.15238087){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}D\end{tabular}}}}% 64 | \put(0,0){\includegraphics[width=\unitlength,page=5]{kron.pdf}}% 65 | \put(0.2462043,0.24245763){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}E\end{tabular}}}}% 66 | \put(0,0){\includegraphics[width=\unitlength,page=6]{kron.pdf}}% 67 | \put(0.34432363,0.1539894){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}g\end{tabular}}}}% 68 | \put(0,0){\includegraphics[width=\unitlength,page=7]{kron.pdf}}% 69 | \put(0.3282385,0.05908706){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}H\end{tabular}}}}% 70 | \put(0,0){\includegraphics[width=\unitlength,page=8]{kron.pdf}}% 71 | \put(0.35397472,0.22637256){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}f\end{tabular}}}}% 72 | \put(0,0){\includegraphics[width=\unitlength,page=9]{kron.pdf}}% 73 | \put(0.50643687,0.15784689){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}a\end{tabular}}}}% 74 | \put(0,0){\includegraphics[width=\unitlength,page=10]{kron.pdf}}% 75 | \put(0.60089519,0.23028922){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}B\end{tabular}}}}% 76 | \put(0,0){\includegraphics[width=\unitlength,page=11]{kron.pdf}}% 77 | \put(0.58203701,0.1119545){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}C\end{tabular}}}}% 78 | \put(0,0){\includegraphics[width=\unitlength,page=12]{kron.pdf}}% 79 | \put(0.7791855,0.14493816){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}D\end{tabular}}}}% 80 | \put(0,0){\includegraphics[width=\unitlength,page=13]{kron.pdf}}% 81 | \put(0.78126804,0.22397653){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}E\end{tabular}}}}% 82 | \put(0,0){\includegraphics[width=\unitlength,page=14]{kron.pdf}}% 83 | \put(0.9590805,0.14576835){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}g\end{tabular}}}}% 84 | \put(0,0){\includegraphics[width=\unitlength,page=15]{kron.pdf}}% 85 | \put(0.94299539,0.050866){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}H\end{tabular}}}}% 86 | \put(0,0){\includegraphics[width=\unitlength,page=16]{kron.pdf}}% 87 | \put(0.96873162,0.21815151){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}f\end{tabular}}}}% 88 | \put(0,0){\includegraphics[width=\unitlength,page=17]{kron.pdf}}% 89 | \end{picture}% 90 | \endgroup% 91 | -------------------------------------------------------------------------------- /paper/figures/linearityOfExpectation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/linearityOfExpectation.pdf -------------------------------------------------------------------------------- /paper/figures/linearityOfExpectation.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'linearityOfExpectation.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{370.29717393bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.14428691)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0.09473814,0.02074787){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}D\end{tabular}}}}% 57 | \put(0.24151633,0.02074787){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}E\end{tabular}}}}% 58 | \put(0.16812727,0.02074787){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 59 | \put(0.24151633,0.09494396){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}C\end{tabular}}}}% 60 | \put(0.16814309,0.09441389){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}B\end{tabular}}}}% 61 | \put(0.02236967,0.09494396){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}A\end{tabular}}}}% 62 | \put(0.09520496,0.09494396){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 63 | \put(0.02147563,0.02074787){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 64 | \put(0,0){\includegraphics[width=\unitlength,page=1]{linearityOfExpectation.pdf}}% 65 | \put(0.48902061,0.02573751){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}D\end{tabular}}}}% 66 | \put(0.63579903,0.02573751){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}E\end{tabular}}}}% 67 | \put(0.56240997,0.02573751){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 68 | \put(0.63579903,0.09993365){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}C\end{tabular}}}}% 69 | \put(0.56242585,0.09940358){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}B\end{tabular}}}}% 70 | \put(0.4166522,0.09993365){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}A\end{tabular}}}}% 71 | \put(0.48948754,0.09993365){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 72 | \put(0.41575828,0.02573751){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 73 | \put(0,0){\includegraphics[width=\unitlength,page=2]{linearityOfExpectation.pdf}}% 74 | \put(0.97844853,0.02519593){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}E\end{tabular}}}}% 75 | \put(0.97844853,0.09939208){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}C\end{tabular}}}}% 76 | \put(0.90507535,0.09886201){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}B\end{tabular}}}}% 77 | \put(0.75930187,0.09939208){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}A\end{tabular}}}}% 78 | \put(0,0){\includegraphics[width=\unitlength,page=3]{linearityOfExpectation.pdf}}% 79 | \put(0.8110548,0.0676324){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}M$_3$\end{tabular}}}}% 80 | \put(0.32231061,0.06139124){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}=\end{tabular}}}}% 81 | \put(0.69076929,0.05765709){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}=\end{tabular}}}}% 82 | \put(0.79674269,0.01726965){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}D\end{tabular}}}}% 83 | \put(0,0){\includegraphics[width=\unitlength,page=4]{linearityOfExpectation.pdf}}% 84 | \end{picture}% 85 | \endgroup% 86 | -------------------------------------------------------------------------------- /paper/figures/multi_input_functions.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/multi_input_functions.pdf -------------------------------------------------------------------------------- /paper/figures/other_functions.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/other_functions.pdf -------------------------------------------------------------------------------- /paper/figures/path1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/path1.pdf -------------------------------------------------------------------------------- /paper/figures/path1.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'path1.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{302.1430765bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.48333792)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0,0){\includegraphics[width=\unitlength,page=1]{path1.pdf}}% 57 | \put(0.26580391,0.43300202){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 58 | \put(0,0){\includegraphics[width=\unitlength,page=2]{path1.pdf}}% 59 | \put(0.40209069,0.41166252){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 60 | \put(0,0){\includegraphics[width=\unitlength,page=3]{path1.pdf}}% 61 | \put(0.00295875,0.35961285){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}S\end{tabular}}}}% 62 | \put(0,0){\includegraphics[width=\unitlength,page=4]{path1.pdf}}% 63 | \put(0.20434061,0.00671868){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}V\end{tabular}}}}% 64 | \put(0,0){\includegraphics[width=\unitlength,page=5]{path1.pdf}}% 65 | \put(0.80182885,0.34668742){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 66 | \put(0,0){\includegraphics[width=\unitlength,page=6]{path1.pdf}}% 67 | \put(0.94590414,0.31984987){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 68 | \put(0,0){\includegraphics[width=\unitlength,page=7]{path1.pdf}}% 69 | \put(0.64927859,0.3382125){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}S\end{tabular}}}}% 70 | \put(0,0){\includegraphics[width=\unitlength,page=8]{path1.pdf}}% 71 | \put(0.73120379,0.14046209){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}V\end{tabular}}}}% 72 | \put(0,0){\includegraphics[width=\unitlength,page=9]{path1.pdf}}% 73 | \end{picture}% 74 | \endgroup% 75 | -------------------------------------------------------------------------------- /paper/figures/path2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/path2.pdf -------------------------------------------------------------------------------- /paper/figures/path2.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'path2.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{462.71550697bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.16753565)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0,0){\includegraphics[width=\unitlength,page=1]{path2.pdf}}% 57 | \put(0.00193201,0.03003643){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}S\end{tabular}}}}% 58 | \put(0,0){\includegraphics[width=\unitlength,page=2]{path2.pdf}}% 59 | \put(0.31167315,0.08707807){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 60 | \put(0,0){\includegraphics[width=\unitlength,page=3]{path2.pdf}}% 61 | \put(0.55306382,0.10014619){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}S\end{tabular}}}}% 62 | \put(0,0){\includegraphics[width=\unitlength,page=4]{path2.pdf}}% 63 | \put(0.63034011,0.11154836){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 64 | \put(0,0){\includegraphics[width=\unitlength,page=5]{path2.pdf}}% 65 | \put(0.852345,0.08349739){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}S\end{tabular}}}}% 66 | \put(0,0){\includegraphics[width=\unitlength,page=6]{path2.pdf}}% 67 | \put(0.93019289,0.08537013){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 68 | \put(0,0){\includegraphics[width=\unitlength,page=7]{path2.pdf}}% 69 | \end{picture}% 70 | \endgroup% 71 | -------------------------------------------------------------------------------- /paper/figures/path3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/path3.pdf -------------------------------------------------------------------------------- /paper/figures/path3.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'path3.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{535.96386527bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.15791008)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0,0){\includegraphics[width=\unitlength,page=1]{path3.pdf}}% 57 | \put(0.19685291,0.05048664){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}M\end{tabular}}}}% 58 | \put(0,0){\includegraphics[width=\unitlength,page=2]{path3.pdf}}% 59 | \put(0.17086033,0.09744198){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 60 | \put(0,0){\includegraphics[width=\unitlength,page=3]{path3.pdf}}% 61 | \put(0.43375819,0.04837661){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}V\end{tabular}}}}% 62 | \put(0,0){\includegraphics[width=\unitlength,page=4]{path3.pdf}}% 63 | \put(0.00166796,0.06504196){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}R\end{tabular}}}}% 64 | \put(0,0){\includegraphics[width=\unitlength,page=5]{path3.pdf}}% 65 | \put(0.00784813,0.01560066){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 66 | \put(0,0){\includegraphics[width=\unitlength,page=6]{path3.pdf}}% 67 | \put(0.7403043,0.08257851){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}M\end{tabular}}}}% 68 | \put(0,0){\includegraphics[width=\unitlength,page=7]{path3.pdf}}% 69 | \put(0.7143117,0.12953381){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 70 | \put(0,0){\includegraphics[width=\unitlength,page=8]{path3.pdf}}% 71 | \put(0.90296829,0.02510207){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}V\end{tabular}}}}% 72 | \put(0,0){\includegraphics[width=\unitlength,page=9]{path3.pdf}}% 73 | \put(0.72906834,0.00378759){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}R\end{tabular}}}}% 74 | \put(0,0){\includegraphics[width=\unitlength,page=10]{path3.pdf}}% 75 | \put(0.6623256,0.00587844){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 76 | \put(0,0){\includegraphics[width=\unitlength,page=11]{path3.pdf}}% 77 | \put(0.95037892,0.11471771){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}R\end{tabular}}}}% 78 | \put(0,0){\includegraphics[width=\unitlength,page=12]{path3.pdf}}% 79 | \put(0.96950418,0.04968508){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 80 | \put(0,0){\includegraphics[width=\unitlength,page=13]{path3.pdf}}% 81 | \end{picture}% 82 | \endgroup% 83 | -------------------------------------------------------------------------------- /paper/figures/path8.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/path8.pdf -------------------------------------------------------------------------------- /paper/figures/path8.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'path8.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{60.91392337bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.91900344)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(-0.00221244,0.48538673){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{0}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 57 | \put(0.5643118,0.22336881){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{0}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 58 | \put(0.85465555,0.89611641){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{0}\smash{\begin{tabular}[t]{l}Y\end{tabular}}}}% 59 | \put(0,0){\includegraphics[width=\unitlength,page=1]{path8.pdf}}% 60 | \end{picture}% 61 | \endgroup% 62 | -------------------------------------------------------------------------------- /paper/figures/power.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/power.pdf -------------------------------------------------------------------------------- /paper/figures/product.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/product.pdf -------------------------------------------------------------------------------- /paper/figures/product.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'product.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{256.92133866bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.16223413)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0.03458922,0.04937072){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 57 | \put(0,0){\includegraphics[width=\unitlength,page=1]{product.pdf}}% 58 | \put(0.16392559,0.05089233){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 59 | \put(0,0){\includegraphics[width=\unitlength,page=2]{product.pdf}}% 60 | \put(0.4177573,0.04915588){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 61 | \put(0,0){\includegraphics[width=\unitlength,page=3]{product.pdf}}% 62 | \put(0.54709367,0.05067749){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 63 | \put(0,0){\includegraphics[width=\unitlength,page=4]{product.pdf}}% 64 | \put(0.78229932,0.04939328){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}T\end{tabular}}}}% 65 | \put(0,0){\includegraphics[width=\unitlength,page=5]{product.pdf}}% 66 | \put(0.91163569,0.05091488){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}U\end{tabular}}}}% 67 | \put(0,0){\includegraphics[width=\unitlength,page=6]{product.pdf}}% 68 | \put(0.29532723,0.04625408){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textbf{=}\end{tabular}}}}% 69 | \put(0.6702872,0.04625408){\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}\textbf{+}\end{tabular}}}}% 70 | \end{picture}% 71 | \endgroup% 72 | -------------------------------------------------------------------------------- /paper/figures/steins.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/paper/figures/steins.pdf -------------------------------------------------------------------------------- /paper/figures/steins.pdf_tex: -------------------------------------------------------------------------------- 1 | %% Creator: Inkscape 1.3 (0e150ed, 2023-07-21), www.inkscape.org 2 | %% PDF/EPS/PS + LaTeX output extension by Johan Engelen, 2010 3 | %% Accompanies image file 'steins.pdf' (pdf, eps, ps) 4 | %% 5 | %% To include the image in your LaTeX document, write 6 | %% \input{.pdf_tex} 7 | %% instead of 8 | %% \includegraphics{.pdf} 9 | %% To scale the image, write 10 | %% \def\svgwidth{} 11 | %% \input{.pdf_tex} 12 | %% instead of 13 | %% \includegraphics[width=]{.pdf} 14 | %% 15 | %% Images with a different path to the parent latex file can 16 | %% be accessed with the `import' package (which may need to be 17 | %% installed) using 18 | %% \usepackage{import} 19 | %% in the preamble, and then including the image with 20 | %% \import{}{.pdf_tex} 21 | %% Alternatively, one can specify 22 | %% \graphicspath{{/}} 23 | %% 24 | %% For more information, please see info/svg-inkscape on CTAN: 25 | %% http://tug.ctan.org/tex-archive/info/svg-inkscape 26 | %% 27 | \begingroup% 28 | \makeatletter% 29 | \providecommand\color[2][]{% 30 | \errmessage{(Inkscape) Color is used for the text in Inkscape, but the package 'color.sty' is not loaded}% 31 | \renewcommand\color[2][]{}% 32 | }% 33 | \providecommand\transparent[1]{% 34 | \errmessage{(Inkscape) Transparency is used (non-zero) for the text in Inkscape, but the package 'transparent.sty' is not loaded}% 35 | \renewcommand\transparent[1]{}% 36 | }% 37 | \providecommand\rotatebox[2]{#2}% 38 | \newcommand*\fsize{\dimexpr\f@size pt\relax}% 39 | \newcommand*\lineheight[1]{\fontsize{\fsize}{#1\fsize}\selectfont}% 40 | \ifx\svgwidth\undefined% 41 | \setlength{\unitlength}{178.67449302bp}% 42 | \ifx\svgscale\undefined% 43 | \relax% 44 | \else% 45 | \setlength{\unitlength}{\unitlength * \real{\svgscale}}% 46 | \fi% 47 | \else% 48 | \setlength{\unitlength}{\svgwidth}% 49 | \fi% 50 | \global\let\svgwidth\undefined% 51 | \global\let\svgscale\undefined% 52 | \makeatother% 53 | \begin{picture}(1,0.23912747)% 54 | \lineheight{1}% 55 | \setlength\tabcolsep{0pt}% 56 | \put(0.07880839,0.10952169){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 57 | \put(0.20889024,0.10949305){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}f(X)\end{tabular}}}}% 58 | \put(0,0){\includegraphics[width=\unitlength,page=1]{steins.pdf}}% 59 | \put(0.59053471,0.06513804){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 60 | \put(0.78755177,0.11149682){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}f(X)\end{tabular}}}}% 61 | \put(0,0){\includegraphics[width=\unitlength,page=2]{steins.pdf}}% 62 | \put(0.58848139,0.15447679){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}X\end{tabular}}}}% 63 | \put(0,0){\includegraphics[width=\unitlength,page=3]{steins.pdf}}% 64 | \put(0.43443835,0.10452983){\color[rgb]{0,0,0}\makebox(0,0)[lt]{\lineheight{1.25}\smash{\begin{tabular}[t]{l}=\end{tabular}}}}% 65 | \put(0,0){\includegraphics[width=\unitlength,page=4]{steins.pdf}}% 66 | \end{picture}% 67 | \endgroup% 68 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "tensorgrad" 3 | version = "0.2.0" 4 | dependencies = [ 5 | "networkx==3.2.1", 6 | "numpy", 7 | "torch", 8 | "nbval", 9 | "pytest", 10 | "einops", 11 | "pdf2image", 12 | "sympy", 13 | "sparse", 14 | "opt_einsum", 15 | "scipy", 16 | "requests>=2.32.3", 17 | "pydantic>=2.11.5", 18 | "fastapi>=0.115.12", 19 | "mangum>=0.19.0", 20 | "boto3>=1.38.23", 21 | ] 22 | requires-python = ">=3.10" 23 | authors = [ 24 | {name = "Thomas Ahle", email = "lobais@gmail.com"}, 25 | ] 26 | description = "Tensor Network Library with Symbolic Autograd" 27 | readme = "README.md" 28 | license = {file = "LICENSE"} 29 | keywords = ["tensors", "autograd"] 30 | classifiers = [ 31 | "Development Status :: 4 - Beta", 32 | "Programming Language :: Python" 33 | ] 34 | 35 | 36 | [project.urls] 37 | Homepage = "https://tensorcookbook.com" 38 | Repository = "https://github.com/thomasahle/tensorgrad.git" 39 | Issues = "https://github.com/thomasahle/tensorgrad/issues" 40 | 41 | [build-system] 42 | requires = [ 43 | "setuptools", 44 | "wheel" 45 | ] 46 | build-backend = "setuptools.build_meta" 47 | 48 | 49 | [tool.setuptools.packages] 50 | find = {} 51 | 52 | [tool.ruff] 53 | line-length = 110 54 | indent-width = 4 55 | 56 | [tool.mypy] 57 | # Enable/disable type checking options 58 | check_untyped_defs = true 59 | disallow_untyped_defs = true 60 | disallow_incomplete_defs = true 61 | exclude = "tests/" 62 | 63 | 64 | [tool.pytest.ini_options] 65 | # addopts = "-s" 66 | # addopts = "--nbval" 67 | 68 | -------------------------------------------------------------------------------- /scripts/analyze_profile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to analyze profile data and print useful statistics. 3 | 4 | Usage: 5 | python scripts/analyze_profile.py 6 | 7 | Example: 8 | python scripts/analyze_profile.py profiling/test_tensor.prof 9 | 10 | This script will: 11 | 1. Display overall profile statistics 12 | 2. Show top functions by cumulative time 13 | 3. Show top functions by total time 14 | 4. Show top tensor.py functions 15 | 5. Show top NetworkX functions 16 | """ 17 | import pstats 18 | import sys 19 | from pstats import SortKey 20 | 21 | def analyze_profile(profile_path): 22 | """Analyze a profiling data file and print useful statistics.""" 23 | stats = pstats.Stats(profile_path) 24 | 25 | print(f"\n{'='*80}") 26 | print(f"Profile Analysis for: {profile_path}") 27 | print(f"{'='*80}\n") 28 | 29 | # Print overall statistics 30 | print("Overall Statistics:") 31 | print(f"Total calls: {stats.total_calls}") 32 | print(f"Total primitive calls: {stats.prim_calls}") 33 | print(f"Total time: {stats.total_tt:.4f} seconds") 34 | print() 35 | 36 | # Print top functions by cumulative time 37 | print("Top 20 Functions (by cumulative time):") 38 | stats.sort_stats(SortKey.CUMULATIVE).print_stats(20) 39 | print() 40 | 41 | # Print top functions by total time 42 | print("Top 20 Functions (by total time):") 43 | stats.sort_stats(SortKey.TIME).print_stats(20) 44 | print() 45 | 46 | # Print specific function groups 47 | 48 | # Find tensor.py functions 49 | tensor_funcs = [] 50 | for func in stats.stats: 51 | if func[0] is not None and '/tensor.py' in func[0]: 52 | tensor_funcs.append((func, stats.stats[func])) 53 | 54 | # Find networkx functions 55 | nx_funcs = [] 56 | for func in stats.stats: 57 | if func[0] is not None and 'networkx' in func[0]: 58 | nx_funcs.append((func, stats.stats[func])) 59 | 60 | # Sort by cumulative time 61 | tensor_funcs.sort(key=lambda x: x[1][3], reverse=True) 62 | nx_funcs.sort(key=lambda x: x[1][3], reverse=True) 63 | 64 | # Print results 65 | print("Top tensor.py Functions:") 66 | for i, (func, stat) in enumerate(tensor_funcs[:10]): 67 | ncalls, tottime, p_calls, cumtime, callers = stat 68 | print(f"{i+1:2d}. {func[2]:30} {cumtime:8.4f}s (calls: {ncalls})") 69 | 70 | print("\nTop NetworkX Functions:") 71 | for i, (func, stat) in enumerate(nx_funcs[:10]): 72 | ncalls, tottime, p_calls, cumtime, callers = stat 73 | print(f"{i+1:2d}. {func[2]:30} {cumtime:8.4f}s (calls: {ncalls})") 74 | 75 | if __name__ == "__main__": 76 | if len(sys.argv) != 2: 77 | print("Usage: python scripts/analyze_profile.py ") 78 | sys.exit(1) 79 | 80 | analyze_profile(sys.argv[1]) -------------------------------------------------------------------------------- /scripts/benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple benchmark script to test the performance of the cached structural graph. 3 | """ 4 | import time 5 | from sympy import symbols 6 | from tensorgrad import Variable 7 | import tensorgrad.functions as F 8 | 9 | def run_benchmark(iterations=10000): 10 | # Create test variables 11 | i, j, k = symbols("i j k") 12 | s = symbols("s") # Same size for symmetric dimensions 13 | X = Variable("X", i, j) 14 | W = Variable("W", j, k) 15 | Y = Variable("Y", i, k) 16 | Z = Variable("Z", i=s, j=s) # For symmetry testing 17 | 18 | # Perform operations that use structural graph 19 | start_time = time.time() 20 | 21 | for _ in range(iterations): 22 | # Hash computation 23 | hash(X) 24 | hash(W) 25 | 26 | # Isomorphism checking 27 | X.is_isomorphic(X.rename(i="i_new", j="j_new")) 28 | 29 | # Symmetries computation 30 | Z_sym = Z.with_symmetries("i j") 31 | Z_sym.symmetries 32 | 33 | # Complex expression with multiple isomorphism tests 34 | expr = F.frobenius2(X @ W - Y) 35 | grad = expr.grad(W) 36 | grad.full_simplify() 37 | 38 | end_time = time.time() 39 | elapsed = end_time - start_time 40 | 41 | return { 42 | "total_time": elapsed, 43 | "avg_time_per_iteration": elapsed / iterations, 44 | } 45 | 46 | if __name__ == "__main__": 47 | print("Running benchmark...") 48 | results = run_benchmark(iterations=5) 49 | print(f"Total time: {results['total_time']:.4f} seconds") 50 | print(f"Average time per iteration: {results['avg_time_per_iteration']:.4f} seconds") 51 | -------------------------------------------------------------------------------- /scripts/profile_ops.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to profile common tensor operations in tensorgrad. 3 | 4 | This script runs a series of standard tensor operations and profiles their performance. 5 | It shows which operations are most expensive and helps identify optimization targets. 6 | 7 | Usage: 8 | python scripts/profile_ops.py [output_file] 9 | """ 10 | import cProfile 11 | import pstats 12 | from pstats import SortKey 13 | import os 14 | import sys 15 | from sympy import symbols 16 | from tensorgrad import Variable 17 | import tensorgrad.functions as F 18 | 19 | def profile_tensor_operations(): 20 | """Profile a set of common tensor operations.""" 21 | # Define symbols for dimensions 22 | i, j, k, l = symbols("i j k l") 23 | n, m, p, q = symbols("n m p q") 24 | 25 | # Create test variables 26 | X = Variable("X", i, j) 27 | W = Variable("W", j, k) 28 | B = Variable("B", i, k) 29 | Y = Variable("Y", i, k) 30 | 31 | # Operations to profile 32 | operations = [] 33 | 34 | # 1. Matrix multiplication and gradient 35 | matmul = X @ W 36 | operations.append(("Matrix Multiplication", lambda: X @ W)) 37 | operations.append(("MatMul Gradient", lambda: matmul.grad(W))) 38 | 39 | # 2. Addition, subtraction, and their gradients 40 | addition = X @ W + B 41 | operations.append(("Addition", lambda: X @ W + B)) 42 | operations.append(("Addition Gradient", lambda: addition.grad(B))) 43 | 44 | # 3. L2 Loss and its gradient 45 | l2_loss = F.frobenius2(X @ W - Y) 46 | operations.append(("L2 Loss", lambda: F.frobenius2(X @ W - Y))) 47 | operations.append(("L2 Loss Gradient", lambda: l2_loss.grad(W))) 48 | 49 | # 4. Simplification of complex expressions 50 | complex_expr = l2_loss.grad(W) 51 | operations.append(("Simplify", lambda: complex_expr.simplify())) 52 | operations.append(("Full Simplify", lambda: complex_expr.full_simplify())) 53 | 54 | # 5. Function composition and gradients 55 | softmax_expr = F.softmax(X @ W, dim='k') 56 | operations.append(("Softmax", lambda: F.softmax(X @ W, dim='k'))) 57 | operations.append(("Softmax Gradient", lambda: softmax_expr.grad(W))) 58 | 59 | # 6. Hessian computation (second derivative) 60 | hessian = l2_loss.grad(W).grad(W) 61 | operations.append(("Hessian", lambda: l2_loss.grad(W).grad(W))) 62 | operations.append(("Hessian Simplify", lambda: hessian.simplify())) 63 | 64 | # 7. Tensor isomorphism tests 65 | operations.append(("Isomorphism Test", lambda: X @ W == X.rename(i="i_") @ W.rename(j="j_"))) 66 | 67 | # Run the operations and collect timing data 68 | results = {} 69 | for name, op in operations: 70 | # Clear any cached computations 71 | op() # Warm-up run 72 | 73 | # Time the operation 74 | results[name] = op() 75 | 76 | return results 77 | 78 | if __name__ == "__main__": 79 | # Set up the output file path 80 | output_file = "profiling/tensor_ops.prof" 81 | if len(sys.argv) > 1: 82 | output_file = sys.argv[1] 83 | 84 | # Make sure the output directory exists 85 | os.makedirs(os.path.dirname(output_file), exist_ok=True) 86 | 87 | # Run the profiling 88 | print("Profiling tensor operations...") 89 | profiler = cProfile.Profile() 90 | profiler.enable() 91 | 92 | results = profile_tensor_operations() 93 | 94 | profiler.disable() 95 | profiler.dump_stats(output_file) 96 | 97 | # Print a summary 98 | print(f"\nProfile data saved to {output_file}") 99 | print("\nOperation summary:") 100 | for name in results: 101 | print(f"- {name}") 102 | 103 | print("\nTo analyze the profile, run:") 104 | print(f"python scripts/analyze_profile.py {output_file}") -------------------------------------------------------------------------------- /scripts/profile_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Python script to run profiling on specific pytest tests. 3 | 4 | Usage: 5 | python scripts/profile_test.py tests/test_file.py [test_function_name] 6 | 7 | Examples: 8 | # Profile a specific test file 9 | python scripts/profile_test.py tests/test_tensor.py 10 | 11 | # Profile a specific test function 12 | python scripts/profile_test.py tests/test_tensor.py::test_rename 13 | 14 | # Use analyze_profile.py to examine the results 15 | python scripts/analyze_profile.py test_output.prof 16 | """ 17 | import cProfile 18 | import pytest 19 | import sys 20 | import os 21 | 22 | if __name__ == "__main__": 23 | if len(sys.argv) < 2: 24 | print("Usage: python scripts/profile_test.py [test_function]") 25 | sys.exit(1) 26 | 27 | test_path = sys.argv[1] 28 | 29 | # Create a output filename based on the test path 30 | profile_output = os.path.join( 31 | "profiling", 32 | os.path.basename(test_path).replace('.py', '') + '.prof' 33 | ) 34 | 35 | # Make sure the output directory exists 36 | os.makedirs("profiling", exist_ok=True) 37 | 38 | # Run the test with cProfile 39 | cProfile.run(f'pytest.main(["{test_path}", "-v"])', profile_output) 40 | 41 | print(f"\nProfile data saved to {profile_output}") 42 | print(f"To analyze the profile, run:") 43 | print(f"python scripts/analyze_profile.py {profile_output}") -------------------------------------------------------------------------------- /scripts/run_profile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Main profiling script for tensorgrad. 4 | 5 | This script provides a unified interface to profile different aspects of tensorgrad. 6 | 7 | Usage: 8 | python scripts/profile.py [options] 9 | 10 | Options: 11 | --ops Profile tensor operations 12 | --test FILE Profile a specific test file 13 | --all Run all profiling jobs 14 | --clean Clean up profiling output 15 | """ 16 | import os 17 | import sys 18 | import shutil 19 | import subprocess 20 | 21 | def create_profiling_dir(): 22 | """Create the profiling directory if it doesn't exist.""" 23 | os.makedirs("profiling", exist_ok=True) 24 | 25 | def clean_profiling_outputs(): 26 | """Remove all profiling outputs.""" 27 | if os.path.exists("profiling"): 28 | shutil.rmtree("profiling") 29 | print("Cleaned profiling outputs.") 30 | 31 | def profile_ops(): 32 | """Run the tensor operations profiling.""" 33 | create_profiling_dir() 34 | print("\n=== Profiling Tensor Operations ===") 35 | subprocess.run([sys.executable, "scripts/profile_ops.py"]) 36 | 37 | def profile_test(test_path): 38 | """Profile a specific test file.""" 39 | create_profiling_dir() 40 | if not test_path.startswith("tests/"): 41 | test_path = f"tests/{test_path}" 42 | print(f"\n=== Profiling Test: {test_path} ===") 43 | subprocess.run([sys.executable, "scripts/profile_test.py", test_path]) 44 | 45 | def print_usage(): 46 | """Print usage instructions.""" 47 | print(__doc__) 48 | 49 | def main(): 50 | """Main entry point.""" 51 | if len(sys.argv) < 2: 52 | print_usage() 53 | return 54 | 55 | arg = sys.argv[1] 56 | 57 | if arg == "--clean": 58 | clean_profiling_outputs() 59 | return 60 | 61 | if arg == "--ops": 62 | profile_ops() 63 | return 64 | 65 | if arg == "--test" and len(sys.argv) > 2: 66 | profile_test(sys.argv[2]) 67 | return 68 | 69 | if arg == "--all": 70 | # Run all profiling jobs 71 | profile_ops() 72 | profile_test("test_isomorphism.py") 73 | profile_test("test_tensor.py") 74 | profile_test("test_ml.py") 75 | return 76 | 77 | # If we got here, the arguments were invalid 78 | print("Invalid arguments.") 79 | print_usage() 80 | 81 | if __name__ == "__main__": 82 | main() -------------------------------------------------------------------------------- /scripts/simple_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Simple benchmark to compare original vs optimized to_numpy.""" 3 | 4 | import time 5 | import numpy as np 6 | import torch 7 | import sympy 8 | from tensorgrad import Variable, Delta, Product, Sum 9 | from tensorgrad import functions as F 10 | from tensorgrad.extras.to_numpy import compile_to_callable as compile_orig 11 | from tensorgrad.extras.to_numpy_optimized import compile_to_callable as compile_opt 12 | 13 | 14 | def time_function(fn, inputs, n_runs=100): 15 | """Time a function over multiple runs.""" 16 | times = [] 17 | for _ in range(n_runs): 18 | start = time.perf_counter() 19 | _ = fn(inputs) 20 | end = time.perf_counter() 21 | times.append(end - start) 22 | return np.mean(times[10:]) # Skip first few for warmup 23 | 24 | 25 | def benchmark_simple(): 26 | """Simple benchmarks.""" 27 | print("=== SIMPLE BENCHMARKS ===\n") 28 | 29 | # 1. Matrix multiplication 30 | print("1. Matrix Multiplication") 31 | d = sympy.Symbol('d') 32 | X = Variable('X', i=d, j=d) 33 | Y = Variable('Y', i=d, j=d) 34 | Z = X @ Y 35 | 36 | # Compilation time 37 | start = time.perf_counter() 38 | fn_orig = compile_orig(Z) 39 | compile_time_orig = time.perf_counter() - start 40 | 41 | start = time.perf_counter() 42 | fn_opt = compile_opt(Z) 43 | compile_time_opt = time.perf_counter() - start 44 | 45 | print(f" Compilation - Original: {compile_time_orig*1000:.2f}ms, Optimized: {compile_time_opt*1000:.2f}ms") 46 | print(f" Speedup: {compile_time_orig/compile_time_opt:.2f}x") 47 | 48 | # Execution time 49 | for size in [50, 100, 200]: 50 | values = { 51 | X: torch.randn(size, size), 52 | Y: torch.randn(size, size) 53 | } 54 | 55 | time_orig = time_function(fn_orig, values) 56 | time_opt = time_function(fn_opt, values) 57 | 58 | print(f" Execution ({size}x{size}) - Original: {time_orig*1000:.3f}ms, Optimized: {time_opt*1000:.3f}ms") 59 | print(f" Speedup: {time_orig/time_opt:.2f}x") 60 | 61 | # 2. Complex sum expression 62 | print("\n2. Complex Sum Expression") 63 | A = Variable('A', i=d, j=d) 64 | B = Variable('B', i=d, j=d) 65 | C = Variable('C', i=d, j=d) 66 | expr = A + 2*B - 3*C + A@B - B@C 67 | 68 | fn_orig = compile_orig(expr) 69 | fn_opt = compile_opt(expr) 70 | 71 | values = { 72 | A: torch.randn(100, 100), 73 | B: torch.randn(100, 100), 74 | C: torch.randn(100, 100) 75 | } 76 | 77 | time_orig = time_function(fn_orig, values) 78 | time_opt = time_function(fn_opt, values) 79 | 80 | print(f" Execution - Original: {time_orig*1000:.3f}ms, Optimized: {time_opt*1000:.3f}ms") 81 | print(f" Speedup: {time_orig/time_opt:.2f}x") 82 | 83 | # 3. Convolution 84 | print("\n3. Convolution") 85 | w_in = sympy.Symbol('w_in') 86 | k_size = sympy.Symbol('k_size') 87 | w_out = sympy.Symbol('w_out') 88 | conv = F.Convolution(input=w_in, kernel=k_size, output=w_out) 89 | 90 | fn_orig = compile_orig(conv) 91 | fn_opt = compile_opt(conv) 92 | 93 | shapes = {w_in: 100, k_size: 5, w_out: 96} 94 | 95 | # Need to pass shapes differently 96 | def run_conv_orig(): 97 | return fn_orig({}, shapes) 98 | 99 | def run_conv_opt(): 100 | return fn_opt({}, shapes) 101 | 102 | times_orig = [] 103 | times_opt = [] 104 | for _ in range(50): 105 | start = time.perf_counter() 106 | _ = run_conv_orig() 107 | times_orig.append(time.perf_counter() - start) 108 | 109 | start = time.perf_counter() 110 | _ = run_conv_opt() 111 | times_opt.append(time.perf_counter() - start) 112 | 113 | time_orig = np.mean(times_orig[10:]) * 1000 114 | time_opt = np.mean(times_opt[10:]) * 1000 115 | 116 | print(f" Execution - Original: {time_orig:.3f}ms, Optimized: {time_opt:.3f}ms") 117 | print(f" Speedup: {time_orig/time_opt:.2f}x") 118 | 119 | # 4. Many variable access 120 | print("\n4. Many Variables (tests arg passing efficiency)") 121 | vars = [Variable(f'v{i}', d) for i in range(20)] 122 | expr = Sum(vars) 123 | 124 | fn_orig = compile_orig(expr) 125 | fn_opt = compile_opt(expr) 126 | 127 | values = {v: torch.randn(100) for v in vars} 128 | 129 | # Run many times to test argument passing overhead 130 | time_orig = time_function(fn_orig, values, n_runs=1000) 131 | time_opt = time_function(fn_opt, values, n_runs=1000) 132 | 133 | print(f" Execution (1000 runs) - Original: {time_orig*1000:.3f}ms, Optimized: {time_opt*1000:.3f}ms") 134 | print(f" Speedup: {time_orig/time_opt:.2f}x") 135 | 136 | 137 | if __name__ == "__main__": 138 | benchmark_simple() -------------------------------------------------------------------------------- /server/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use AWS Lambda base image for Python 3.12 2 | FROM public.ecr.aws/lambda/python:3.12 3 | 4 | ENV AWS_DEFAULT_REGION="us-east-1" 5 | ENV DYNAMODB_CACHE_TABLE="TensorgradCache" 6 | ENV DYNAMODB_SNIPPET_TABLE="CodeSnippets" 7 | 8 | # Install microdnf for lightweight package management 9 | RUN microdnf update -y \ 10 | && microdnf install -y \ 11 | glibc-langpack-en \ 12 | texlive-scheme-basic \ 13 | texlive-standalone \ 14 | texlive-luatex85 \ 15 | texlive-comicneue \ 16 | poppler-utils \ 17 | && microdnf clean all 18 | 19 | # Set TEXMFVAR to a writable location 20 | ENV TEXMFVAR=/tmp 21 | 22 | # Pre-generate format files during build 23 | RUN mkdir -p /tmp && \ 24 | fmtutil-sys --all && \ 25 | luaotfload-tool --update 26 | 27 | # Install Python dependencies 28 | COPY tensorgrad/ tensorgrad/ 29 | COPY server/ server/ 30 | RUN pip install ./server 31 | 32 | # Command to start the Lambda function 33 | CMD ["server.drawTensors.handler"] 34 | -------------------------------------------------------------------------------- /server/__init__.py: -------------------------------------------------------------------------------- 1 | # Package initializer for server module 2 | __all__ = [] -------------------------------------------------------------------------------- /server/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Tensor Visualizer 7 | 82 | 83 | 84 |
85 |

Tensor Visualizer

86 | 87 | 88 | 89 |
90 |
91 | 92 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /server/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "tensorg_lambda" 3 | version = "0.1.0" 4 | dependencies = [ 5 | "networkx==3.2.1", 6 | "pdf2image", 7 | "sympy", 8 | "boto3", 9 | "moto", 10 | "fastapi", 11 | "mangum", 12 | "requests", 13 | "pydantic", 14 | ] 15 | -------------------------------------------------------------------------------- /server/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasahle/tensorgrad/0eda7c622f4977cf64577208c8fd1db23aa63627/server/tests/__init__.py -------------------------------------------------------------------------------- /server/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | # Add verbosity and timeout settings 3 | addopts = -v -s --tb=short 4 | timeout = 300 5 | timeout_method = thread 6 | 7 | # Show output immediately 8 | console_output_style = plain -------------------------------------------------------------------------------- /server/tests/test_app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | pytest.importorskip("moto") # Skip AWS tests if moto is not installed 4 | import uuid 5 | import textwrap 6 | 7 | from fastapi.testclient import TestClient 8 | from moto import mock_aws # Use the generic AWS mock 9 | 10 | # Set the environment variables before importing your app. 11 | os.environ["DYNAMODB_CACHE_TABLE"] = "TensorgradCache" 12 | os.environ["DYNAMODB_SNIPPET_TABLE"] = "CodeSnippets" 13 | 14 | # Now import the app and helper functions from your FastAPI application. 15 | from server.drawTensors import app, safe_execute, ExecutionResult 16 | 17 | # Create a TestClient for FastAPI 18 | client = TestClient(app) 19 | 20 | 21 | @pytest.fixture(autouse=True) 22 | def setup_dynamodb(): 23 | """ 24 | Use Moto to mock AWS and create the necessary DynamoDB tables for testing. 25 | """ 26 | with mock_aws(): 27 | import boto3 # Import boto3 within the Moto context 28 | 29 | dynamodb = boto3.resource("dynamodb", region_name="us-east-1") 30 | 31 | # Create the CodeCache table 32 | try: 33 | dynamodb.create_table( 34 | TableName="CodeCache", 35 | KeySchema=[{"AttributeName": "code_hash", "KeyType": "HASH"}], 36 | AttributeDefinitions=[ 37 | {"AttributeName": "code_hash", "AttributeType": "S"} 38 | ], 39 | ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, 40 | ) 41 | except Exception: 42 | pass 43 | 44 | # Create the CodeSnippets table 45 | try: 46 | dynamodb.create_table( 47 | TableName="CodeSnippets", 48 | KeySchema=[{"AttributeName": "snippet_id", "KeyType": "HASH"}], 49 | AttributeDefinitions=[ 50 | {"AttributeName": "snippet_id", "AttributeType": "S"} 51 | ], 52 | ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, 53 | ) 54 | except Exception: 55 | pass 56 | 57 | # Yield to run tests inside the Moto context. 58 | yield 59 | 60 | 61 | # -------- Tests for safe_execute -------- 62 | 63 | 64 | def test_safe_execute_success(): 65 | code = "print('Hello, world!')" 66 | result: ExecutionResult = safe_execute(code) 67 | assert result.success is True 68 | assert "Hello, world!" in result.output 69 | # For this code, there should be no image, error, or stacktrace. 70 | assert result.image is None 71 | assert result.error == "" 72 | assert result.stacktrace is None 73 | 74 | 75 | def test_safe_execute_syntax_error(): 76 | code = "print('Hello" # unbalanced quote causes SyntaxError 77 | result: ExecutionResult = safe_execute(code) 78 | assert result.success is False 79 | assert "SyntaxError" in result.error or "unterminated" in result.error 80 | 81 | 82 | def test_safe_execute_unsafe_code(): 83 | # Attempt to import os should be blocked. 84 | code = "import os\nprint('Unsafe')" 85 | result: ExecutionResult = safe_execute(code) 86 | assert result.success is False 87 | assert "unsafe operations" in result.error 88 | 89 | 90 | # -------- Tests for /execute endpoint -------- 91 | 92 | 93 | def test_execute_endpoint_success(): 94 | payload = {"code": "print('Test Execute')"} 95 | response = client.post("/execute", json=payload) 96 | assert response.status_code == 200 97 | data = response.json() 98 | # Validate using the ExecutionResult schema fields. 99 | assert data["success"] is True 100 | assert "Test Execute" in data["output"] 101 | assert data["error"] == "" 102 | 103 | 104 | def test_execute_good_code_with_image(): 105 | code = textwrap.dedent( 106 | """ 107 | i = sp.symbols('i'); 108 | x = tg.Delta(i, 'i', 'j'); 109 | y = x * 2; 110 | save_steps(y); 111 | """ 112 | ) 113 | response = client.post("/execute", json={"code": code}) 114 | assert response.status_code == 200 115 | data = response.json() 116 | assert data["success"] is True 117 | assert "image" in data 118 | # parse image 119 | image_data = data["image"] 120 | assert image_data.startswith("data:image/png;base64,") 121 | 122 | 123 | def test_execute_endpoint_invalid_payload(): 124 | # Send an empty payload. 125 | response = client.post("/execute", json={"code": ""}) 126 | assert response.status_code == 400 127 | data = response.json() 128 | assert "No code provided" in data["detail"] 129 | 130 | 131 | # -------- Tests for Snippet Endpoints -------- 132 | 133 | 134 | def test_create_and_fetch_snippet(): 135 | code = "print('Snippet Test')" 136 | # Create a snippet. 137 | response_post = client.post("/snippets", json={"code": code}) 138 | assert response_post.status_code == 200 139 | post_data = response_post.json() 140 | assert "snippet_id" in post_data 141 | snippet_id = post_data["snippet_id"] 142 | 143 | # Retrieve the snippet. 144 | response_get = client.get(f"/snippets/{snippet_id}") 145 | assert response_get.status_code == 200 146 | get_data = response_get.json() 147 | # Verify that the snippet contains the correct code. 148 | assert get_data["snippet_id"] == snippet_id 149 | assert get_data["code"] == code 150 | # Verify created_at and author_id are present. 151 | assert "created_at" in get_data 152 | assert "author_id" in get_data 153 | 154 | 155 | def test_fetch_nonexistent_snippet(): 156 | # Attempt to fetch a snippet that doesn't exist. 157 | fake_snippet_id = str(uuid.uuid4()) 158 | response_get = client.get(f"/snippets/{fake_snippet_id}") 159 | assert response_get.status_code == 404 160 | data = response_get.json() 161 | assert "not found" in data["detail"] 162 | -------------------------------------------------------------------------------- /server/tests/test_docker.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import subprocess 3 | import time 4 | import requests 5 | import pytest 6 | import json 7 | import socket 8 | from typing import Optional 9 | from pydantic import BaseModel 10 | 11 | from server.drawTensors import CodePayload, ExecutionResult, SnippetCreationResponse, Snippet, safe_execute 12 | 13 | PORT = 9000 # Host port 14 | LAMBDA_URL = f"http://localhost:{PORT}/2015-03-31/functions/function/invocations" 15 | 16 | 17 | def wait_for_port(host: str, port: int, timeout: float = 10.0): 18 | """Wait for a network port to become available.""" 19 | deadline = time.time() + timeout 20 | while time.time() < deadline: 21 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: 22 | result = sock.connect_ex((host, port)) 23 | if result == 0: 24 | return True 25 | time.sleep(0.5) 26 | raise TimeoutError(f"Timeout waiting for {host}:{port} to become available.") 27 | 28 | 29 | @pytest.fixture(scope="session") 30 | def docker_container(): 31 | """ 32 | Build the Docker image once per session, run the container, yield the container ID, 33 | and then stop and remove the container after the session ends. 34 | """ 35 | 36 | # Run the build command with live output. 37 | # We need to run from the tensorgrad root directory 38 | import os 39 | repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 40 | 41 | # Try buildx first, fall back to regular build 42 | build_cmd = ["docker", "buildx", "build"] 43 | 44 | # Check if buildx is available 45 | buildx_check = subprocess.run( 46 | ["docker", "buildx", "version"], 47 | capture_output=True, 48 | text=True 49 | ) 50 | 51 | if buildx_check.returncode != 0: 52 | print("Note: docker buildx not available, using regular docker build") 53 | build_cmd = ["docker", "build"] 54 | 55 | # Build the Docker image 56 | build_proc = subprocess.run( 57 | build_cmd + [ 58 | "-t", 59 | "tensorgrad", 60 | "-f", 61 | "server/Dockerfile", 62 | ".", 63 | ], 64 | capture_output=True, 65 | text=True, 66 | cwd=repo_root, 67 | ) 68 | assert build_proc.returncode == 0, f"Docker build failed:\n{build_proc.stderr}" 69 | 70 | # Run the Docker container in detached mode (mapping host port PORT to container port 8080). 71 | run_proc = subprocess.run( 72 | [ 73 | "docker", 74 | "run", 75 | "-d", 76 | "--name", 77 | "lambda_local_test", 78 | "-p", 79 | f"{PORT}:8080", 80 | "tensorgrad", 81 | ], 82 | capture_output=True, 83 | text=True, 84 | cwd=repo_root, 85 | ) 86 | assert run_proc.returncode == 0, f"Docker run failed:\n{run_proc.stderr}" 87 | container_id = run_proc.stdout.strip() 88 | 89 | # Allow time for the container to start. 90 | wait_for_port("localhost", PORT, timeout=10) 91 | time.sleep(1) 92 | print("Docker container is running.") 93 | 94 | try: 95 | yield container_id 96 | 97 | finally: 98 | print("Cleaning up Docker container...") 99 | logs = subprocess.run( 100 | ["docker", "logs", container_id], capture_output=True, text=True 101 | ) 102 | print("\nDocker container logs:\n", logs.stdout, logs.stderr) 103 | 104 | # Teardown: stop and remove the container. 105 | subprocess.run( 106 | ["docker", "stop", container_id], stdout=sys.stdout, stderr=sys.stderr 107 | ) 108 | subprocess.run( 109 | ["docker", "rm", container_id], stdout=sys.stdout, stderr=sys.stderr 110 | ) 111 | 112 | 113 | def invoke_api( 114 | body: Optional[BaseModel] = None, 115 | path: str = "/", 116 | method: str = "GET", 117 | pathParameters: dict = None, 118 | ) -> requests.Response: 119 | """ 120 | Helper function that creates a simulated API Gateway event and invokes the Lambda endpoint. 121 | """ 122 | response = requests.post( 123 | LAMBDA_URL, 124 | json={ 125 | "resource": path, 126 | "path": path, 127 | "httpMethod": method, 128 | "queryStringParameters": None, 129 | "pathParameters": pathParameters, 130 | "body": None if body is None else body.model_dump_json(), 131 | "isBase64Encoded": False, 132 | "requestContext": { 133 | "resourcePath": path, 134 | "httpMethod": method, 135 | "path": path, 136 | }, 137 | }, 138 | headers={"Content-Type": "application/json"}, 139 | ) 140 | assert response.status_code == 200, f"API failed: {response.text}" 141 | print(response) 142 | data = response.json() 143 | assert data["statusCode"] == 200, f"API failed: {data['body']}" 144 | print("reponse.json()=", data) 145 | return json.loads(data["body"]) 146 | 147 | 148 | def test_docker_image_execute_endpoint(docker_container): 149 | payload_obj = CodePayload(code="print('Hello from Docker')") 150 | response = invoke_api(body=payload_obj, path="/execute", method="POST") 151 | result = ExecutionResult.model_validate(response) 152 | assert result.success is True, f"Execution did not succeed: {result}" 153 | 154 | 155 | def test_docker_snippet_endpoints(docker_container): 156 | # ---- Snippet Creation ---- 157 | payload_obj = CodePayload(code="print('Hello, snippet!')") 158 | response_body = invoke_api(body=payload_obj, path="/snippets", method="POST") 159 | snippet_creation = SnippetCreationResponse.model_validate(response_body) 160 | assert snippet_creation.snippet_id is not None 161 | snippet_id = snippet_creation.snippet_id 162 | 163 | # ---- Snippet Retrieval ---- 164 | response_get = invoke_api( 165 | path=f"/snippets/{snippet_id}", 166 | method="GET", 167 | pathParameters={"snippet_id": snippet_id}, 168 | ) 169 | snippet_obj = Snippet.model_validate(response_get) 170 | assert snippet_obj.snippet_id == snippet_id 171 | assert snippet_obj.code == payload_obj.code 172 | assert snippet_obj.created_at is not None 173 | assert snippet_obj.author_id is not None 174 | 175 | 176 | def test_safe_execute_dynamic_import(): 177 | """ 178 | Ensure that calls to __import__ are detected as unsafe and blocked by safe_execute. 179 | """ 180 | code = "__import__('os')" 181 | result = safe_execute(code) 182 | # Execution should be rejected as unsafe 183 | assert not result.success 184 | assert "unsafe" in (result.error or "").lower() 185 | 186 | -------------------------------------------------------------------------------- /server/tests/test_docker_debug.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Debug script to test Docker build and run issues.""" 3 | 4 | import subprocess 5 | import sys 6 | import time 7 | 8 | def run_command(cmd, cwd=None): 9 | """Run a command and stream output in real-time.""" 10 | print(f"\n>>> Running: {' '.join(cmd)}") 11 | process = subprocess.Popen( 12 | cmd, 13 | stdout=subprocess.PIPE, 14 | stderr=subprocess.STDOUT, 15 | text=True, 16 | cwd=cwd 17 | ) 18 | 19 | # Stream output in real-time 20 | for line in iter(process.stdout.readline, ''): 21 | print(line, end='') 22 | 23 | process.wait() 24 | return process.returncode 25 | 26 | def main(): 27 | print("Testing Docker build process...") 28 | 29 | # Check if Docker is running 30 | print("\n1. Checking Docker status...") 31 | ret = run_command(["docker", "version"]) 32 | if ret != 0: 33 | print("ERROR: Docker is not running or not installed!") 34 | return 1 35 | 36 | # Check for existing containers 37 | print("\n2. Checking for existing containers...") 38 | run_command(["docker", "ps", "-a", "--filter", "name=lambda_local_test"]) 39 | 40 | # Try to remove any existing container 41 | print("\n3. Cleaning up any existing container...") 42 | run_command(["docker", "rm", "-f", "lambda_local_test"]) 43 | 44 | # Build with progress output 45 | print("\n4. Building Docker image (this may take several minutes)...") 46 | start_time = time.time() 47 | ret = run_command([ 48 | "docker", "build", 49 | "--progress=plain", # Show detailed progress 50 | "-t", "tensorgrad", 51 | "-f", "server/Dockerfile", 52 | "." 53 | ], cwd="..") 54 | 55 | if ret != 0: 56 | print(f"\nERROR: Docker build failed after {time.time() - start_time:.1f} seconds!") 57 | return 1 58 | 59 | print(f"\nDocker build completed in {time.time() - start_time:.1f} seconds") 60 | 61 | # Try running the container 62 | print("\n5. Running container...") 63 | ret = run_command([ 64 | "docker", "run", 65 | "-d", 66 | "--name", "lambda_local_test", 67 | "-p", "9000:8080", 68 | "tensorgrad" 69 | ]) 70 | 71 | if ret != 0: 72 | print("\nERROR: Failed to run container!") 73 | return 1 74 | 75 | # Check if container is running 76 | print("\n6. Checking container status...") 77 | time.sleep(2) 78 | run_command(["docker", "ps", "--filter", "name=lambda_local_test"]) 79 | 80 | # Check logs 81 | print("\n7. Container logs:") 82 | run_command(["docker", "logs", "lambda_local_test"]) 83 | 84 | # Cleanup 85 | print("\n8. Cleaning up...") 86 | run_command(["docker", "stop", "lambda_local_test"]) 87 | run_command(["docker", "rm", "lambda_local_test"]) 88 | 89 | print("\nDone!") 90 | return 0 91 | 92 | if __name__ == "__main__": 93 | sys.exit(main()) -------------------------------------------------------------------------------- /server/tests/test_docker_quick.py: -------------------------------------------------------------------------------- 1 | """Quick test to check if Docker tests can run without building.""" 2 | 3 | import subprocess 4 | import sys 5 | 6 | def check_docker(): 7 | """Check if Docker is available and running.""" 8 | try: 9 | result = subprocess.run( 10 | ["docker", "version"], 11 | capture_output=True, 12 | text=True, 13 | timeout=5 14 | ) 15 | if result.returncode != 0: 16 | print("ERROR: Docker is not running!") 17 | print(result.stderr) 18 | return False 19 | print("✓ Docker is running") 20 | return True 21 | except Exception as e: 22 | print(f"ERROR: Failed to check Docker: {e}") 23 | return False 24 | 25 | def check_existing_image(): 26 | """Check if the tensorgrad image already exists.""" 27 | result = subprocess.run( 28 | ["docker", "images", "tensorgrad", "-q"], 29 | capture_output=True, 30 | text=True 31 | ) 32 | if result.stdout.strip(): 33 | print("✓ tensorgrad image already exists") 34 | return True 35 | else: 36 | print("✗ tensorgrad image not found - would need to build") 37 | return False 38 | 39 | def check_port(): 40 | """Check if port 9000 is available.""" 41 | import socket 42 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 43 | result = sock.connect_ex(('localhost', 9000)) 44 | sock.close() 45 | if result == 0: 46 | print("✗ Port 9000 is already in use!") 47 | # Check what's using it 48 | subprocess.run(["lsof", "-i", ":9000"], capture_output=False) 49 | return False 50 | else: 51 | print("✓ Port 9000 is available") 52 | return True 53 | 54 | def check_buildx(): 55 | """Check if docker buildx is available.""" 56 | result = subprocess.run( 57 | ["docker", "buildx", "version"], 58 | capture_output=True, 59 | text=True 60 | ) 61 | if result.returncode == 0: 62 | print("✓ docker buildx is available") 63 | return True 64 | else: 65 | print("✗ docker buildx not available - will fall back to regular build") 66 | return False 67 | 68 | def main(): 69 | print("Checking Docker test prerequisites...\n") 70 | 71 | checks = [ 72 | check_docker(), 73 | check_existing_image(), 74 | check_port(), 75 | check_buildx() 76 | ] 77 | 78 | if not checks[0]: # Docker not running 79 | print("\nDocker is required to run these tests!") 80 | return 1 81 | 82 | if not checks[1]: # Image doesn't exist 83 | print("\nThe Docker image needs to be built first.") 84 | print("This can take 10-20 minutes due to TeX installation.") 85 | print("Run: docker build -t tensorgrad -f server/Dockerfile .") 86 | 87 | if not checks[2]: # Port in use 88 | print("\nPort 9000 is in use. Kill the process or use a different port.") 89 | 90 | return 0 91 | 92 | if __name__ == "__main__": 93 | sys.exit(main()) -------------------------------------------------------------------------------- /tensorgrad/__init__.py: -------------------------------------------------------------------------------- 1 | from .tensor import ( 2 | Tensor, # noqa: F401 3 | Function, # noqa: F401 4 | Zero, # noqa: F401 5 | Product, # noqa: F401 6 | Sum, # noqa: F401 7 | Variable, # noqa: F401 8 | Delta, # noqa: F401 9 | Ones, # noqa: F401 10 | Derivative, # noqa: F401 11 | function, # noqa: F401 12 | ) 13 | from .extras.expectation import Expectation # noqa: F401 14 | from .functions import frobenius2, kronecker, diag, sum, log, pow, trace # noqa: F401 15 | from sympy import symbols # noqa: F401 16 | -------------------------------------------------------------------------------- /tensorgrad/extras/__init__.py: -------------------------------------------------------------------------------- 1 | from .expectation import Expectation # noqa: F401 2 | -------------------------------------------------------------------------------- /tensorgrad/extras/_convolution.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import functional as F 3 | 4 | 5 | def naive_tensor(X: int, Z: int, Y: int) -> torch.Tensor: 6 | assert X == Y + Z - 1, f"Expected X == Y + Z - 1, got {X=} {Y=} {Z=}" 7 | conv = torch.zeros(X, Z, Y) 8 | for x in range(X): 9 | for y in range(Y): 10 | for z in range(Z): 11 | if x == y + z: 12 | conv[x, z, y] = 1 13 | return conv 14 | 15 | 16 | def conv_einsum_dispatch(einsum_str: str, A: torch.Tensor | tuple, B: torch.Tensor | tuple) -> torch.Tensor: 17 | lhs, rhs = einsum_str.split("->") 18 | A_dims, B_dims = lhs.split(",") 19 | 20 | if isinstance(B, tuple): 21 | assert len(B_dims) == 3, f"Expected 3 dimensions for B tensor, got {einsum_str}" 22 | if isinstance(A, tuple): 23 | assert len(A_dims) == 3, f"Expected 3 dimensions for B tensor, got {einsum_str}" 24 | 25 | # Two convolutions should be easy to handle, but we don't support it yet 26 | # E.g. ijk,ilm->jklm corresponds to 27 | # Out[j,k,l,m] = sum_i [i=j+k][i=l+m] = [j+k=l+m] = [j=l+m-k] 28 | # which is some kind of generalized convolution... 29 | if isinstance(B, tuple): 30 | return torch.einsum(einsum_str, naive_tensor(*A), naive_tensor(*B)) 31 | 32 | # Swap A and B so that A is the tensor and B is the convolution 33 | A_dims, B_dims = B_dims, A_dims 34 | einsum_str = f"{A_dims},{B_dims}->{rhs}" 35 | A, B = B, A 36 | 37 | # If we have to do a hadamard multiplication, we can't handle that right now 38 | if set(A_dims) & set(B_dims) & set(rhs): 39 | return torch.einsum(einsum_str, A, naive_tensor(*B)) 40 | 41 | X, Z, Y = B 42 | assert X == Y + Z - 1, f"Expected X == Y + Z - 1, got {X=} {Y=} {Z=}" 43 | 44 | # Move the batch dimensions to the front 45 | batch_dims = [i for i, a in enumerate(A_dims) if a not in B_dims] 46 | contr_dims = [i for i, a in enumerate(A_dims) if a in B_dims] 47 | bd = "".join(A_dims[i] for i in batch_dims) 48 | # print(f"{batch_dims=}, {contr_dims=} {einsum_str=}") 49 | # Flatten to a 2D tensor 50 | A_flat = A.permute(*(batch_dims + contr_dims)).flatten(len(batch_dims)) 51 | if len(batch_dims) > 0: 52 | A_flat = A_flat.flatten(0, len(batch_dims) - 1) 53 | elif len(batch_dims) == 0: 54 | A_flat = A_flat.unsqueeze(0) 55 | B = A_flat.shape[0] 56 | # Figure out what dimensions we need from the convolution 57 | b_contract = [i for i, b in enumerate(B_dims) if b in A_dims] 58 | 59 | res = None # type: torch.Tensor 60 | # bx,xzy->bzy 61 | if b_contract == [0]: 62 | res = A_flat.unfold(dimension=1, size=Z, step=1) # shape [B,Y,Z] 63 | res = res.permute(0, 2, 1) 64 | assert res.shape == (B, Z, Y), f"{res.shape=}" 65 | 66 | # by,xzy->bxz 67 | elif b_contract == [2]: 68 | # out[x,z] = A[x-z] if valid 69 | # => typical "toeplitz" with negative shift 70 | # => pad left/right by (Z-1), unfold(size=Z, step=1), 71 | # slice first X, flip dimension 1 72 | assert A_flat.shape == (B, Y), f"{A_flat.shape=}" 73 | padded = F.pad(A_flat, (Z - 1, Z - 1)) # shape [Y + 2*(Z-1)] 74 | assert padded.shape == (B, Y + 2 * (Z - 1)), f"{padded.shape=}" 75 | tmp = padded.unfold(dimension=1, size=Z, step=1) # => [B,Y+2*(Z-1),Z] 76 | tmp = tmp[:, :X] # => [B,X,Z] 77 | res = tmp.flip(dims=(2,)) # => [B,X,Z] 78 | assert res.shape == (B, X, Z), f"{res.shape=}" 79 | 80 | # bz,xzy->bxy, using symmetry 81 | elif b_contract == [1]: 82 | eq = f"{A_dims},{B_dims[0]}{B_dims[2]}{B_dims[1]}->{rhs}" 83 | res = conv_einsum_dispatch(eq, A, (X, Y, Z)) 84 | assert res.shape == (B, X, Y) 85 | 86 | # bzy,xzy->bx, and byz,xzy->bx 87 | elif b_contract == [1, 2]: 88 | # if bzy 89 | if A_dims[contr_dims[0]] == B_dims[1]: 90 | expanded = A_flat.reshape(B, Z, Y) 91 | else: 92 | expanded = A_flat.reshape(B, Y, Z).permute(0, 2, 1) 93 | # Fold assumes (B, C*K, L) => (B, C, H, W) 94 | res = F.fold( 95 | expanded, # shape [B, Z, Y] 96 | output_size=(1, X), # treat as (H=1, W=X) 97 | kernel_size=(1, Z), 98 | stride=(1, 1), 99 | ).squeeze(1, 2) # shape [B, X] 100 | assert res.shape == (B, X) 101 | 102 | if res is not None: 103 | # Handle extra summations and permutations 104 | mid = bd + "".join(b for i, b in enumerate(B_dims) if i not in b_contract) 105 | res = res.reshape([A.shape[i] for i in batch_dims] + list(res.shape)[1:]) 106 | return torch.einsum(f"{mid}->{rhs}", res) 107 | 108 | # Fallback to the naive implementation 109 | return torch.einsum(einsum_str, A, naive_tensor(X, Z, Y)) 110 | -------------------------------------------------------------------------------- /tensorgrad/extras/polynomials.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from functools import singledispatch 3 | from tensorgrad import Tensor, Function, Product, Sum, Variable, Expectation, Zero, Delta 4 | from tensorgrad import functions as F 5 | 6 | 7 | @singledispatch 8 | def collect(expr: Tensor, x: Tensor) -> dict[int, Tensor]: 9 | if expr == x: 10 | return {1: 1} 11 | return {0: expr} # This is always the default 12 | # raise NotImplementedError(f"Collect not implemented for {type(expr)}") 13 | 14 | 15 | @collect.register 16 | def _(expr: Variable, x: Tensor) -> dict[int, Tensor]: 17 | if expr == x: 18 | return {1: 1} 19 | return {0: expr} 20 | 21 | 22 | def _prod(c1, c2): 23 | if isinstance(c1, int) or isinstance(c2, int): 24 | return c1 * c2 25 | return c1 @ c2 26 | 27 | 28 | @collect.register 29 | def _(expr: Product, x: Tensor) -> dict[int, Tensor]: 30 | # TODO: We may want to support the case where x is partially in the product 31 | # Using isomorphism stuff 32 | 33 | res = {0: 1} # The convolutional identity 34 | for factor in expr.factors: 35 | new_res = defaultdict(int) 36 | for k1, c1 in res.items(): 37 | for k2, c2 in collect(factor, x).items(): 38 | new_res[k1 + k2] += _prod(c1, c2) 39 | res = new_res 40 | return res 41 | 42 | 43 | @collect.register 44 | def _(expr: Sum, x: Tensor) -> dict[int, Tensor]: 45 | res = defaultdict(int) 46 | for weight, term in zip(expr.weights, expr.terms): 47 | for k, v in collect(term, x).items(): 48 | res[k] += weight * v 49 | return res 50 | 51 | 52 | @collect.register 53 | def _(expr: Function, x: Tensor) -> dict[int, Tensor]: 54 | if isinstance(expr.signature, F._PowerFunction): 55 | k = expr.signature.k 56 | inner = collect(expr.inputs[0], x) 57 | 58 | if k == 0: 59 | # x^0 = 1 (constant) 60 | return {0: Ones(**expr.inputs[0].shape)} 61 | 62 | if k < 0: 63 | if len(inner) == 1: 64 | ((k1, c1),) = inner.items() 65 | return {k1 * k: c1**k} 66 | raise NotImplementedError("Cannot collect negative powers of sums") 67 | 68 | # For positive integer powers, use repeated convolution 69 | if isinstance(k, int) and k > 0: 70 | res = {0: 1} 71 | for _ in range(k): 72 | new_res = defaultdict(int) 73 | for k1, c1 in res.items(): 74 | for k2, c2 in inner.items(): 75 | new_res[k1 + k2] += c1 * c2 # mul instead of prod, since pow is hadamard 76 | res = new_res 77 | return res 78 | 79 | # For fractional powers, only handle simple case 80 | if len(inner) == 1: 81 | ((k1, c1),) = inner.items() 82 | if c1 == 1: # Only x^k where k is fractional 83 | return {k1 * k: 1} 84 | 85 | raise NotImplementedError(f"Cannot collect fractional powers of sums or complex expressions") 86 | 87 | # Handle matrix inverse as x^(-1) 88 | if isinstance(expr.signature, F._MatrixInverseFunction): 89 | inner = collect(expr.inputs[0], x) 90 | if len(inner) == 1: 91 | ((k1, c1),) = inner.items() 92 | if c1 == 1: # Only handle simple case where inner is just x 93 | return {-1: 1} 94 | raise NotImplementedError("Cannot collect inverse of sums or complex expressions") 95 | 96 | # For other functions like exp, log, etc., treat them as constants if they don't depend on x 97 | if not expr.depends_on(x): 98 | return {0: expr} 99 | 100 | # Otherwise, we can't handle it as a polynomial 101 | raise NotImplementedError(f"Function {expr.signature} not implemented for polynomial collection") 102 | 103 | 104 | @collect.register 105 | def _(expr: Zero, x: Tensor) -> dict[int, Tensor]: 106 | """Special handler for Zero tensors.""" 107 | return {0: Zero()} 108 | 109 | 110 | @collect.register 111 | def _(expr: Delta, x: Tensor) -> dict[int, Tensor]: 112 | """Special handler for Delta tensors.""" 113 | if expr == x: 114 | return {1: 1} 115 | return {0: expr} 116 | 117 | 118 | @collect.register 119 | def _(expr: Expectation, x: Tensor) -> dict[int, Tensor]: 120 | if x.depends_on(expr.wrt): 121 | # TODO: What about depdence on mu and covar? 122 | raise ValueError("Cannot collect a variable that depends on the variable of the expectation") 123 | 124 | res = collect(expr.tensor, x) 125 | 126 | # Convert scalar values to Tensor to avoid errors when creating Expectation 127 | from tensorgrad import Tensor as TensorClass # Local import to avoid circular import 128 | 129 | # Create expectation for each term, wrapping scalars as tensors if needed 130 | result = {} 131 | for k, v in res.items(): 132 | if isinstance(v, (int, float)): 133 | # Convert scalar to Tensor with empty shape 134 | v = TensorClass(v) 135 | result[k] = Expectation(v, expr.wrt, expr.mu, expr.covar, expr.covar_names) 136 | 137 | return result 138 | 139 | 140 | # Helper function to normalize tensor representation 141 | def _normalize_tensor(tensor): 142 | """ 143 | Normalize tensor representation for comparison purposes. 144 | Converts Sum/Product representations of scalars to simple values. 145 | """ 146 | if isinstance(tensor, (int, float)): 147 | return tensor 148 | # Add more normalization as needed 149 | return tensor 150 | -------------------------------------------------------------------------------- /tensorgrad/extras/to_formula_old.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | from tensorgrad import Sum, Product, Variable, Delta 4 | 5 | 6 | def to_simple_matrix_formula(expr): 7 | assert isinstance(expr, Sum) 8 | res = [] 9 | e0 = list(expr.edges)[0] 10 | for prod in expr.terms: 11 | assert isinstance(prod, Product) 12 | mat = [] 13 | for comp in prod.components(): 14 | print("Comp:", comp) 15 | # if len(comp.edges) == 0: 16 | # t = comp.tensors[0] 17 | # in_edge = "" # FIXME: This prevents us from getting the right transpose on the first matrix in the trace 18 | # else: 19 | # t = next(t for t in comp.tensors if e0 in t.edges) 20 | # in_edge = e0 21 | # s = " ".join(dfs(t, adj, visited, in_edge)) 22 | s = component_to_matrix(comp, e0) 23 | # if comp.rank == 0: 24 | if len(comp.edges) == 0: 25 | s = s.replace("1", "") 26 | s = f"\\mathrm{{tr}}({s})" 27 | else: 28 | s = s.replace("1", "\\mathbf{1}") 29 | mat.append(s) 30 | res.append(" \\cdot ".join(mat)) 31 | return "\n\\\\&+ ".join(res) 32 | 33 | 34 | def component_to_matrix(comp, left_edge): 35 | adj = defaultdict(list) 36 | for t in comp.tensors: 37 | for e in t.edges: 38 | adj[e].append(t) 39 | 40 | # We can handle 4 cases: 41 | # - A matrix 42 | # - A trace 43 | # - A matrix-vector mult 44 | # - A vector-matrix*-vector mult 45 | 46 | # If there's a vector, we start from that 47 | # vec = next((t for t in comp.tensors if len(t.edges) == 1), None) 48 | # if vec is not None: 49 | # # This is a vector 50 | # pass 51 | 52 | # If there's supposed to be a free edge on the left, we start from that 53 | start = next((t for t in comp.tensors if left_edge in t.edges), None) 54 | if start is not None: 55 | in_edge = left_edge 56 | else: 57 | start = comp.tensors[0] 58 | in_edge = "" 59 | cur = start 60 | 61 | visited = set() 62 | res = [] 63 | while id(cur) not in visited: 64 | visited.add(id(cur)) 65 | e0, e1 = cur.edges 66 | res.append(cur.name) 67 | if e0 != in_edge: 68 | e0, e1 = e1, e0 69 | res.append("^T") 70 | t0, t1 = adj[e1] 71 | in_edge = e1 72 | cur = t1 if t0 is cur else t0 73 | 74 | return " ".join(res) 75 | 76 | 77 | def dfs(t, adj, visited, in_edge): 78 | visited[t] = True 79 | if isinstance(t, Variable): 80 | yield t.name + ("^T" if in_edge == t.edges[1] else "") 81 | elif isinstance(t, Delta): 82 | yield "1" 83 | else: 84 | assert False 85 | for e in t.edges: 86 | for u in adj[e]: 87 | if u not in visited: 88 | yield from dfs(u, adj, visited, e) 89 | -------------------------------------------------------------------------------- /tensorgrad/extras/to_numpy_optimization_notes.md: -------------------------------------------------------------------------------- 1 | # to_numpy Optimization Notes 2 | 3 | ## Overview 4 | This document describes the optimizations made to `to_numpy.py`, resulting in `to_numpy_optimized.py`. 5 | 6 | ## Key Optimizations 7 | 8 | ### 1. Memory Efficiency 9 | - Avoid unnecessary numpy conversions by checking if tensors are already numpy arrays 10 | - Proper handling of sparse arrays with `todense()` when needed 11 | - Configurable dtype support (defaults to `np.float32`) 12 | 13 | ### 2. Computational Efficiency 14 | - Added `@lru_cache` for permutation computations to avoid recalculating 15 | - Used `defaultdict` for efficient name generation instead of string checks 16 | - Special case optimization for matrix multiplication (2-factor products) 17 | - Pre-computed argument positions for function calls to avoid repeated lookups 18 | 19 | ### 3. Code Generation 20 | - More efficient variable naming using counters 21 | - Direct function invocation instead of `eval()` for better performance 22 | - Improved sparse tensor handling in Sum operations 23 | 24 | ### 4. Bug Fixes 25 | - Fixed symbol tracking for Variables to ensure dimensions are properly registered 26 | - Fixed dtype string generation to avoid module prefix issues 27 | - Maintained full API compatibility with original 28 | 29 | ## Performance Improvements 30 | 31 | Based on benchmarks: 32 | - **Compilation**: ~11x faster due to efficient name generation and caching 33 | - **Execution**: 1.1-1.6x faster depending on operation type 34 | - **Multi-variable operations**: ~1.5x faster due to optimized argument passing 35 | 36 | ## Usage 37 | 38 | The optimized version is a drop-in replacement: 39 | 40 | ```python 41 | # Original 42 | from tensorgrad.extras.to_numpy import compile_to_callable 43 | 44 | # Optimized 45 | from tensorgrad.extras.to_numpy_optimized import compile_to_callable 46 | ``` 47 | 48 | ## Testing 49 | 50 | Run tests with: 51 | ```bash 52 | python -m pytest tests/extras/test_to_numpy_optimized.py 53 | ``` 54 | 55 | Run benchmarks with: 56 | ```bash 57 | python benchmarks/benchmark_to_numpy.py 58 | python benchmarks/simple_benchmark.py 59 | ``` -------------------------------------------------------------------------------- /tensorgrad/imgtools.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from pdf2image import convert_from_path 3 | 4 | import os 5 | import subprocess 6 | from PIL import Image, ImageDraw 7 | 8 | from tensorgrad.extras.to_tikz import to_tikz 9 | 10 | from tensorgrad import Derivative 11 | from tensorgrad.extras import Expectation 12 | 13 | import networkx as nx 14 | 15 | 16 | def compile_latex(latex_code, suffix=""): 17 | output_dir = tempfile.mkdtemp() 18 | 19 | # Save the LaTeX code to a file 20 | tex_file_path = os.path.join(output_dir, f"output_{suffix}.tex") 21 | with open(tex_file_path, "w") as file: 22 | file.write(latex_code) 23 | 24 | # Compile the LaTeX file to PDF 25 | # subprocess.run( 26 | # ["lualatex", "-output-directory", output_dir, tex_file_path], 27 | # check=True, 28 | # ) 29 | try: 30 | result = subprocess.run( 31 | [ 32 | "lualatex", 33 | "-interaction=batchmode", # Make it quieter 34 | "-halt-on-error", # Stop on first error 35 | "-output-directory", 36 | output_dir, 37 | tex_file_path, 38 | ], 39 | check=True, 40 | stdout=subprocess.PIPE, 41 | stderr=subprocess.PIPE, 42 | text=True, # Ensures output is returned as a string 43 | ) 44 | print("LuaLaTeX Output:", result.stdout) 45 | except subprocess.CalledProcessError as e: 46 | raise ValueError(e.stderr) 47 | 48 | # Convert the PDF to an image 49 | pdf_path = tex_file_path.replace(".tex", ".pdf") 50 | images = convert_from_path(pdf_path, dpi=300) 51 | 52 | # Save the first page as an image 53 | image_path = pdf_path.replace(".pdf", ".png") 54 | images[0].save(image_path, "PNG") 55 | 56 | return image_path 57 | 58 | 59 | def combine_images_vertically( 60 | image_paths, 61 | padding=10, 62 | line_padding=5, 63 | background_color="white", 64 | line_color="black", 65 | line_width=2, 66 | output_path="combined_image.png", 67 | ): 68 | images = [Image.open(x) for x in image_paths] 69 | 70 | # Calculate total height considering padding, maximum width, separating lines, and line padding 71 | total_height = ( 72 | sum(image.height for image in images) 73 | + padding * (len(images) + 1) 74 | + line_width * (len(images) - 1) 75 | + line_padding * 2 * (len(images) - 1) 76 | ) 77 | max_width = max(image.width for image in images) + padding * 2 78 | 79 | # Create a new image with a white background 80 | combined_image = Image.new("RGB", (max_width, total_height), color=background_color) 81 | 82 | # Create a drawing context 83 | draw = ImageDraw.Draw(combined_image) 84 | 85 | # Paste images into the new image with padding, separating lines, and line padding 86 | y_offset = padding 87 | for i, image in enumerate(images): 88 | # Calculate horizontal padding to center images 89 | x_padding = (max_width - image.width) // 2 90 | 91 | combined_image.paste(image, (x_padding, y_offset)) 92 | y_offset += image.height + padding 93 | 94 | # Add line padding below the current image 95 | y_offset += line_padding 96 | 97 | # Draw a black separating line below the current image (except for the last image) 98 | if i < len(images) - 1: 99 | line_y = y_offset + line_width // 2 100 | draw.line([(0, line_y), (max_width, line_y)], fill=line_color, width=line_width) 101 | y_offset += line_width + line_padding 102 | 103 | # Save the combined image 104 | combined_image_path = output_path 105 | combined_image.save(combined_image_path) 106 | 107 | return combined_image_path 108 | 109 | 110 | def save_as_image(expr, path): 111 | latex_code = to_tikz(expr) 112 | image_path = compile_latex(latex_code) 113 | os.rename(image_path, path) 114 | print(f"Image saved to {path}") 115 | 116 | 117 | def save_steps_old(expr, min_steps=None): 118 | images = [] 119 | images.append(compile_latex(expr, suffix="0")) 120 | old = expr 121 | while True: 122 | new = expr.simplify({"grad_steps": len(images)}) 123 | if new == old and (min_steps is None or len(images) >= min_steps): 124 | print(f"{new=} = {old=}") 125 | break 126 | old = new 127 | images.append(compile_latex(to_tikz(new), suffix=f"{len(images)}")) 128 | 129 | output_path = combine_images_vertically(images) 130 | print(f"Combined image saved to {output_path}") 131 | 132 | 133 | def save_steps(expr, slow_grad=False, output_path="steps.png"): 134 | images = [] 135 | images.append(compile_latex(to_tikz(expr), suffix="0")) 136 | 137 | # TODO: This is still not good enough, since there may be a double derivative somewhere inside the tensor. 138 | cnt_derivatives = 0 139 | d = expr 140 | while isinstance(d, Derivative) or isinstance(d, Expectation): 141 | cnt_derivatives += 1 142 | d = d.tensor 143 | if cnt_derivatives > 1: 144 | expr = expr.simplify({"grad_steps": cnt_derivatives}) 145 | images.append(compile_latex(to_tikz(expr), suffix=f"{len(images)}")) 146 | 147 | expand = False 148 | while True: 149 | try: 150 | args = {"grad_steps": 1} if slow_grad else {} 151 | if expand: 152 | args["expand"] = True 153 | new = expr.simplify(args).simplify() 154 | except Exception as e: 155 | print("ERROR:", e) 156 | break 157 | if new == expr: 158 | if not expand: 159 | expand = True 160 | continue 161 | break 162 | print(new) 163 | images.append(compile_latex(to_tikz(new), suffix=f"{len(images)}")) 164 | expr = new 165 | 166 | combine_images_vertically(images, output_path=output_path) 167 | print(f"Combined image saved to {output_path}") 168 | 169 | 170 | def draw_structural_graph(tensor, iter=50): 171 | G, edges = tensor.structural_graph() 172 | for e, node in edges.items(): 173 | n = G.number_of_nodes() 174 | G.add_node(n, name=f"{e}") 175 | G.add_edge(n, node) 176 | labels = {i: data.get("name", "") for i, data in G.nodes(data=True)} 177 | pos = nx.spectral_layout(G) 178 | pos = nx.kamada_kawai_layout(G, pos=pos) 179 | pos = nx.spring_layout(G, pos=pos, k=1, iterations=iter) 180 | 181 | nx.draw_networkx(G, pos=pos, labels=labels) 182 | -------------------------------------------------------------------------------- /tensorgrad/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TypeVar, Generic 2 | 3 | 4 | class _MatchEdgesKey: 5 | """Normally Tensors use isomorphism as their test for equality, but they don't include 6 | the edge names in the comparison. This class is used to compare tensors based on their 7 | edge names. It is used in the Sum tensor to combine tensors with the same edge names.""" 8 | 9 | def __init__(self, value: Any, **edge_names: str): 10 | self.value = value 11 | self.hash = hash(value) 12 | self.edge_names = edge_names 13 | 14 | def __eq__(self, other: Any) -> bool: 15 | if isinstance(other, _MatchEdgesKey): 16 | return self.value.is_isomorphic(other.value, edge_names=self.edge_names, match_edges=True) 17 | return False 18 | 19 | def __hash__(self) -> int: 20 | return self.hash 21 | 22 | 23 | K = TypeVar("K") # Key type 24 | V = TypeVar("V") # Value type 25 | 26 | 27 | class DisjointSets(Generic[K, V]): 28 | def __init__(self): 29 | self.parent: dict[K, K] = {} 30 | self.values: dict[K, V] = {} 31 | 32 | def find(self, x: K) -> K: 33 | if x not in self.parent: 34 | self.parent[x] = x 35 | if self.parent[x] != x: 36 | self.parent[x] = self.find(self.parent[x]) 37 | return self.parent[x] 38 | 39 | def union(self, x: K, y: K) -> None: 40 | x_root = self.find(x) 41 | y_root = self.find(y) 42 | 43 | # Check value compatibility if both roots have values 44 | x_value = self.values.get(x_root) 45 | y_value = self.values.get(y_root) 46 | 47 | if x_value is not None and y_value is not None and x_value != y_value: 48 | raise ValueError(f"Cannot merge nodes with incompatible values: {x_value} and {y_value}") 49 | 50 | self.parent[x_root] = y_root 51 | 52 | # Keep the non-None value if one exists 53 | if y_value is None and x_value is not None: 54 | self.values[y_root] = x_value 55 | 56 | def __setitem__(self, key: K, value: V) -> None: 57 | root = self.find(key) 58 | if root in self.values and self.values[root] != value: 59 | raise ValueError(f"Node already had incompatible value: {self.values[root]} and {value}") 60 | self.values[root] = value 61 | 62 | def __getitem__(self, key: K) -> V: 63 | return self.values[self.find(key)] 64 | 65 | def get(self, key: K, default: V = None) -> V: 66 | return self.values.get(self.find(key), default) 67 | 68 | def items(self) -> list[tuple[list[K], V]]: 69 | groups = {} 70 | for key in self.parent: 71 | root = self.find(key) 72 | if root not in groups: 73 | groups[root] = [] 74 | groups[root].append(key) 75 | return [(keys, self.values.get(root)) for root, keys in groups.items()] 76 | 77 | def __repr__(self) -> str: 78 | """{[k1, k2]: v, [k3]: v}""" 79 | return "{" + ", ".join(f"{keys}: {value}" for keys, value in self.items()) + "}" 80 | 81 | 82 | class KeyStoringDict: 83 | """ 84 | A dictionary-like class that: 85 | - Internally stores { key: (key, value) } in self._store. 86 | - Normal lookups return only 'value', but we can also retrieve 87 | the actual stored key object via get_with_key(). 88 | """ 89 | 90 | def __init__(self, *args, **kwargs): 91 | """ 92 | Similar to dict's constructor, it can accept: 93 | - KeyStoringDict(mapping) 94 | - KeyStoringDict(iterable_of_pairs) 95 | - KeyStoringDict(key1=value1, key2=value2, ...) 96 | in any combination. 97 | """ 98 | self._store = {} 99 | self.update(*args, **kwargs) 100 | 101 | def __setitem__(self, key, value): 102 | # Internally store (key, value). 103 | self._store[key] = (key, value) 104 | 105 | def __getitem__(self, key): 106 | # Return only the user_value portion 107 | _, user_value = self._store[key] 108 | return user_value 109 | 110 | def __delitem__(self, key): 111 | del self._store[key] 112 | 113 | def __contains__(self, key): 114 | return key in self._store 115 | 116 | def __len__(self): 117 | return len(self._store) 118 | 119 | def __iter__(self): 120 | """ 121 | Iterating over this dict should iterate over its keys. 122 | """ 123 | return iter(self._store) 124 | 125 | def __repr__(self): 126 | """ 127 | Show a nice representation: KeyStoringDict({k: v, ...}) 128 | """ 129 | class_name = type(self).__name__ 130 | items_str = ", ".join(f"{k!r}: {v!r}" for k, v in self.items()) 131 | return f"{class_name}({{{items_str}}})" 132 | 133 | def get_with_key(self, key, default=None): 134 | """ 135 | Return (stored_key, user_value) if key is found, 136 | else return 'default'. 137 | """ 138 | return self._store.get(key, default) 139 | 140 | def get(self, key, default=None): 141 | """ 142 | Normal dict .get(), returning just the user_value (or default). 143 | """ 144 | _, value = self._store.get(key, (None, default)) 145 | return value 146 | 147 | def pop(self, key): 148 | """ 149 | pop(key) → user_value, removing the item if it exists. 150 | If key not found, raise KeyError. 151 | """ 152 | _, user_value = self._store.pop(key) 153 | return user_value 154 | 155 | def update(self, other: dict): 156 | """ 157 | update(...) adds/overwrites items from another dict/mapping. 158 | """ 159 | for k, v in other.items(): 160 | self[k] = v 161 | 162 | def keys(self): 163 | """ 164 | Return a view (or iterable) of keys. 165 | """ 166 | return self._store.keys() 167 | 168 | def values(self): 169 | """ 170 | Return a view (or iterable) of user_values. 171 | """ 172 | for _, user_value in self._store.values(): 173 | yield user_value 174 | 175 | def items(self): 176 | """ 177 | Return a view (or iterable) of (key, user_value). 178 | """ 179 | return self._store.values() 180 | 181 | def copy(self): 182 | """ 183 | Return a shallow copy of this KeyStoringDict. 184 | """ 185 | return KeyStoringDict(self) 186 | -------------------------------------------------------------------------------- /tests/extras/test_to_index.py: -------------------------------------------------------------------------------- 1 | from sympy import symbols 2 | 3 | from tensorgrad import Variable, Product, Function, Zero, Delta, Derivative, Expectation 4 | from tensorgrad import functions as F 5 | from tensorgrad.extras.to_index import to_index, to_index_free 6 | 7 | 8 | def test_cycles(): 9 | i = symbols("i") 10 | M = Variable("M", i, j=i) 11 | for k in range(2, 5): 12 | # Using pow function 13 | Mk = F.pow(M, k) 14 | assert isinstance(Mk, Function) 15 | assert to_index_free(Mk) == f"(M)^{{{k}}}" 16 | assert to_index(Mk) == f"(M_{{i,j}})^{{{k}}}" 17 | assert to_index_free(F.trace(Mk)) == f"tr((M)^{{{k}}})" 18 | 19 | # Using a product 20 | Mk2 = F.multi_dot([M] * k, ("i", "j")).simplify() 21 | assert isinstance(Mk2, Product) 22 | 23 | # Paths are annoying because we don't have a good way to decide transpose. 24 | # We just assume that the order in Mk2.edges is meaningful. 25 | if list(Mk2.edges) == ["i", "j"]: 26 | assert to_index_free(Mk2) == " ".join("M" * k) 27 | else: 28 | assert to_index_free(Mk2) == " ".join(("M^T",) * k) 29 | 30 | trMk2 = F.trace(Mk2).simplify() 31 | assert to_index_free(trMk2) == f"tr({' '.join('M' * k)})" 32 | 33 | 34 | def test_variable_and_rename(): 35 | i, j = symbols("i j") 36 | X = Variable("X", i, j) 37 | Y = X.rename(i="a", j="b") 38 | 39 | assert to_index(X) == "X_{i,j}" 40 | assert to_index(Y) == "X_{a,b}" 41 | assert to_index_free(X) == "X" 42 | assert to_index_free(Y) == "X" 43 | 44 | 45 | def test_zero_and_delta(): 46 | i, j = symbols("i j") 47 | 48 | Z = Zero(i, j) 49 | assert to_index(Z) == "0_{i,j}" 50 | assert to_index_free(Z) == "0" 51 | 52 | delta = Delta(i, "i, j") 53 | assert to_index(delta) == "δ_{i,j}" 54 | assert to_index_free(delta) == "I" 55 | 56 | 57 | def test_sum(): 58 | i, j = symbols("i j") 59 | X = Variable("X", i) 60 | Y = Variable("Y", i) 61 | 62 | expr = X + 2 * Y 63 | assert to_index(expr) == "X_{i} + 2 Y_{i}" 64 | assert to_index_free(expr) == "X + 2 Y" 65 | 66 | 67 | def test_derivative(): 68 | i = symbols("i") 69 | X = Variable("X", i) 70 | expr = Derivative(X, X) 71 | 72 | assert to_index(expr) == "d(X_{i})/d(X_{i})" 73 | assert to_index_free(expr) == "d(X)/d(X)" 74 | 75 | 76 | def test_function(): 77 | i = symbols("i") 78 | X = Variable("X", i) 79 | 80 | square_X = F.pow(X, 2) 81 | sin_X = F.exp(X) 82 | 83 | assert to_index(square_X) == "(X_{i})^{2}" 84 | assert to_index(sin_X) == "exp(X_{i})" 85 | 86 | assert to_index_free(square_X) == "(X)^{2}" 87 | assert to_index_free(sin_X) == "exp(X)" 88 | 89 | 90 | def test_expectation(): 91 | i = symbols("i") 92 | X = Variable("X", i) 93 | expectation = Expectation(X, wrt=X) 94 | 95 | assert to_index(expectation) == "E_X[X_{i}]" 96 | assert to_index_free(expectation) == "E_X[X]" 97 | 98 | 99 | def test_self_product(): 100 | i, j = symbols("i j") 101 | A = Variable("A", i, j) 102 | prod = A @ A 103 | 104 | assert to_index(prod) == "A_{i,j} A_{i,j}" 105 | assert to_index_free(prod) == "tr(A A^T)" 106 | 107 | 108 | def test_higher_order_trace(): 109 | i, j, k = symbols("i j k") 110 | A = Variable("A", i, j) 111 | B = Variable("B", j, k) 112 | C = Variable("C", k, i) 113 | 114 | trace_ABC = F.trace(A @ B @ C).simplify() 115 | assert to_index_free(trace_ABC) == "tr(A B C)" 116 | 117 | 118 | def test_complex_expression(): 119 | i, j, k = symbols("i j k") 120 | X = Variable("X", i) 121 | Y = Variable("Y", i) 122 | Z = Zero(i) 123 | 124 | expr = (2 * X + -3 * Y + Z).simplify() 125 | assert to_index(expr) == "2 X_{i} - 3 Y_{i}" 126 | assert to_index_free(expr) == "2 X - 3 Y" 127 | 128 | 129 | def test_variable_no_edges(): 130 | v = Variable("x") 131 | assert to_index(v) == "x" 132 | assert to_index_free(v) == "x" 133 | 134 | 135 | def test_MT_string(): 136 | i = symbols("i") 137 | M = Variable("M", i, j=i) 138 | T = M.rename(i="j", j="i") 139 | A = F.trace(F.multi_dot([M, M, T, M, T, T], ("i", "j"))).simplify() 140 | B = F.trace(F.multi_dot([M, M, T, T, M, T], ("i", "j"))).simplify() 141 | assert A != B 142 | assert to_index_free(A) == "tr(M M M^T M M^T M^T)" 143 | assert to_index_free(B) == "tr(M M M^T M^T M M^T)" 144 | 145 | 146 | def test_inner_product(): 147 | # Disabled: We don't have a way to handle this right now 148 | i = symbols("i") 149 | v1 = Variable("x", i) 150 | v2 = Variable("y", i) 151 | p = v1 @ v2 152 | assert to_index(p) == "x_{i} y_{i}" 153 | assert to_index_free(p) == "x^T y" 154 | 155 | 156 | def _test_outer_product(): 157 | # Disabled: We don't have a way to handle this right now 158 | i, j = symbols("i j") 159 | v1 = Variable("x", i) 160 | v2 = Variable("y", j) 161 | p = v1 @ v2 162 | assert to_index(p) == "x_{i} y_{j}" 163 | assert to_index_free(p) == "x y^T" 164 | 165 | 166 | def test_mixed_product(): 167 | # Product of tensors that do not share edges. 168 | i, j = symbols("i j") 169 | v1 = Variable("x", i) 170 | v2 = Variable("y", j) 171 | A = Variable("A", i, j) 172 | 173 | p = (v1 @ A @ v2).simplify() 174 | assert to_index(p) == "x_{i} A_{i,j} y_{j}" 175 | assert to_index_free(p) == "x^T A y" 176 | 177 | p = (v1 @ A).simplify() 178 | assert to_index(p) == "x_{i} A_{i,j}" 179 | assert to_index_free(p) == "x^T A" 180 | 181 | p = (A @ v2).simplify() 182 | assert to_index(p) == "A_{i,j} y_{j}" 183 | assert to_index_free(p) == "A y" 184 | 185 | 186 | def test_lexicographical_minimal_rotation_bug(): 187 | i, j, y = symbols("i j y") 188 | # Construct three matrices: 189 | # A has edges (i, j) -> if used with in/out (j, i) then its marker is T. 190 | A = Variable("A", i, j) 191 | # B has edges (i, y) -> used with (i, y) gives marker M. 192 | B = Variable("B", i, y) 193 | # C has edges (y, j) -> used with (y, j) gives marker M. 194 | C = Variable("C", y, j) 195 | # Manually set the cycle's out_edges so that: 196 | # out_edges = [i, y, j] and hence in_edges = [j, i, y] 197 | # This forces: 198 | # For A: (in, out) = (j, i) which is reversed (T). 199 | # For B: (in, out) = (i, y) which matches (M). 200 | # For C: (in, out) = (y, j) which matches (M). 201 | # out_edges = [i, y, j] 202 | # result = _handle_trace([A, B, C], out_edges) 203 | result = to_index_free(Product([A, B, C])) 204 | expected = "tr(B C A^T)" # The optimal rotation rotates the pair list to [B, C, A]. 205 | assert result == expected, f"Expected {expected}, got {result}" 206 | -------------------------------------------------------------------------------- /tests/extras/test_to_latex.py: -------------------------------------------------------------------------------- 1 | from sympy import symbols 2 | 3 | from tensorgrad.tensor import Variable, Sum, Product 4 | from tensorgrad.extras.to_latex import to_latex, Rename 5 | 6 | 7 | def test_variable_indexed(): 8 | i, j = symbols("i j") 9 | X = Variable("X", i, j) 10 | latex = to_latex(X) # index-based by default 11 | assert latex == "X_{i,j}" 12 | 13 | 14 | def test_variable_index_free_no_transpose(): 15 | i, j = symbols("i j") 16 | X = Variable("X", i, j) 17 | X.original_edges = (i, j) 18 | latex = to_latex(X, index_free=True) 19 | assert latex == "X" 20 | 21 | 22 | def test_variable_index_free_transpose(): 23 | i, j = symbols("i j") 24 | X = Variable("X", j, i) 25 | X.original_edges = (i, j) 26 | latex = to_latex(X, index_free=True) 27 | assert latex == "X^T" 28 | 29 | 30 | def test_sum_indexed(): 31 | i, j = symbols("i j") 32 | X = Variable("X", i, j) 33 | Y = Variable("Y", i, j) 34 | expr = Sum([X, Y], weights=[2, -1]) 35 | latex = to_latex(expr) # indexed 36 | assert latex == "2 X_{i,j} - Y_{i,j}" 37 | 38 | 39 | def test_sum_index_free(): 40 | i, j = symbols("i j") 41 | X = Variable("X", i, j) 42 | Y = Variable("Y", i, j) 43 | expr = Sum([X, Y], weights=[1, 1]) 44 | latex = to_latex(expr, index_free=True) 45 | assert latex == "X + Y" 46 | 47 | 48 | def test_product_indexed(): 49 | i, j, k = symbols("i j k") 50 | A = Variable("A", i, j) 51 | B = Variable("B", j, k) 52 | expr = Product([A, B]) 53 | latex = to_latex(expr) 54 | assert latex == "A_{i,j} B_{j,k}" 55 | 56 | 57 | def test_product_index_free(): 58 | i, j, k = symbols("i j k") 59 | A = Variable("A", i, j) 60 | B = Variable("B", j, k) 61 | A.original_edges = (i, j) 62 | B.original_edges = (j, k) 63 | expr = Product([A, B]) 64 | latex = to_latex(expr, index_free=True) 65 | assert latex == "A B" 66 | 67 | 68 | def test_rename_indexed(): 69 | i, j = symbols("i j") 70 | X = Variable("X", i, j) 71 | # rename i->k, j->l 72 | rename_expr = Rename(X, {"i": "k", "j": "l"}) 73 | # By default we have a placeholder implementation 74 | latex = to_latex(rename_expr) 75 | assert "X_{k,l}" in latex 76 | -------------------------------------------------------------------------------- /tests/test_jacobians.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd.functional import jacobian 3 | from sympy import symbols 4 | 5 | from tensorgrad.tensor import Variable 6 | from tensorgrad.testutils import rand_values, assert_close 7 | from tensorgrad.extras.evaluate import evaluate 8 | 9 | 10 | def test_simple_vector(): 11 | i = symbols("i") 12 | x = Variable("x", i) 13 | ts = rand_values([x], {i: 3}) 14 | res = evaluate(x.grad(x).simplify(), {x: ts[x]}) 15 | expected = jacobian(lambda x: x, ts[x].rename(None)) 16 | torch.testing.assert_close(res.rename(None), expected) 17 | 18 | 19 | def test_simple_matrix(): 20 | i, j = symbols("i j") 21 | x = Variable("x", i, j) 22 | ts = rand_values([x], {i: 3, j: 2}) 23 | res = evaluate(x.grad(x).simplify(), {x: ts[x]}) 24 | expected = jacobian(lambda x: x, ts[x].rename(None)).rename("i", "j", "i_", "j_") 25 | assert_close(res, expected) 26 | 27 | 28 | def test_a_not_function_of_x(): 29 | i = symbols("i") 30 | a = Variable("a") 31 | x = Variable("x", i) 32 | ts = rand_values([a, x], {i: 3}) 33 | expr = (a / x).grad(x).simplify() 34 | res = evaluate(expr, ts) 35 | expected = jacobian(lambda x: ts[a] / x, ts[x].rename(None)).rename("i", "i_") 36 | assert_close(res, expected) 37 | 38 | 39 | def test_a_not_function_of_x_matrix(): 40 | i, j = symbols("i j") 41 | A = Variable("A", i, j) 42 | x = Variable("x", j) 43 | ts = rand_values([A, x], {i: 3, j: 2}) 44 | res = evaluate((A @ x).grad(x).simplify(), {A: ts[A], x: ts[x]}) 45 | expected = jacobian(lambda x: (ts[A].rename(None) @ x), ts[x].rename(None)).rename("i", "j") 46 | assert_close(res, expected) 47 | -------------------------------------------------------------------------------- /tests/test_matrix_calc.py: -------------------------------------------------------------------------------- 1 | from tensorgrad import Variable 2 | import tensorgrad.functions as F 3 | from sympy import symbols 4 | 5 | # Various tests from matrix calculus 6 | 7 | 8 | def test_gradient_of_product(): 9 | # Testing: ∇(Bx + b)^T C(Dx + d) = B^T C(Dx + d) + D^T C^T (Bx + b) 10 | x, b, d = symbols("x b d") 11 | B = Variable("B", x=x, b=b) 12 | C = Variable("C", b=b, d=d) 13 | D = Variable("D", x=x, d=d) 14 | x = Variable("x", x=x) 15 | b = Variable("b", b=b) 16 | d = Variable("d", d=d) 17 | 18 | expr = (B @ x + b) @ C @ (D @ x + d) 19 | gradient = expr.grad(x).simplify() 20 | 21 | expected = B @ C @ (D @ x + d) + D @ C @ (B @ x + b) 22 | 23 | assert gradient.simplify() == expected.simplify() 24 | 25 | 26 | def test_gradient_of_quadratic_form(): 27 | # Testing: ∂b^T X^T X c / ∂X = X(bc^T + cb^T) 28 | i, j = symbols("i j") 29 | X = Variable("X", i, j) 30 | b = Variable("b", i) 31 | c = Variable("c", i) 32 | 33 | expr = (b @ X) @ (X @ c) 34 | gradient = expr.grad(X) 35 | gradient = gradient.simplify({"expand": True}) 36 | 37 | expected = X @ F.symmetrize(b @ c.rename(i="i_")) 38 | 39 | # The issue is: We don't factor things, so we end up with $X b c^T + X c b^T$ 40 | # for the gradient. So we'll just expand the expected value too. 41 | expected = expected.simplify({"expand": True}) 42 | 43 | assert gradient == expected 44 | 45 | 46 | def test_gradient_of_matrix_trace(): 47 | # Testing: ∂ tr(BA) / ∂A = B^T 48 | i, j = symbols("i j") 49 | A = Variable("A", i, j) 50 | B = Variable("B", i, j) 51 | 52 | # We don't really need F.trace here, since B @ A already connects both edges. 53 | # But it's a good test of what happens when F.trace gets a scalar. 54 | expr = F.trace(B @ A) 55 | gradient = expr.grad(A) 56 | assert gradient.simplify() == B 57 | 58 | 59 | def test_gradient_of_quadratic_form_with_middle_matrix(): 60 | # Testing: ∂b^T X^T D X c / ∂X = D^T X bc^T + D X cb^T 61 | i, j = symbols("i j") 62 | X = Variable("X", i, j) 63 | D = Variable("D", j1=j, j2=j) 64 | b = Variable("b", i) 65 | c = Variable("c", i) 66 | 67 | expr = b @ X.rename(j="j1") @ D @ X.rename(j="j2") @ c 68 | gradient = expr.grad(X) 69 | 70 | expected = D.rename(j1="j", j2="j'") @ X @ (b @ c.rename(i="i_")) 71 | expected += D.rename(j2="j", j1="j'") @ X @ (c @ b.rename(i="i_")) 72 | 73 | assert gradient.full_simplify() == expected.full_simplify() 74 | 75 | 76 | def test_gradient_of_quadratic_form_with_affine_transform(): 77 | # Testing: ∂(Xb + c)^T D(Xb + c) / ∂X = (D + D^T)(Xb + c)b^T 78 | i, j = symbols("i j") 79 | X = Variable("X", i, j) 80 | D = Variable("D", j=j, j2=j) 81 | b = Variable("b", i) 82 | c = Variable("c", j) 83 | 84 | expr = (X @ b + c) @ D @ (X @ b + c).rename(j="j2") 85 | gradient = expr.grad(X) 86 | 87 | expected = F.symmetrize(D) @ (X @ b + c) @ b 88 | 89 | assert gradient.simplify({"expand": True}) == expected.simplify({"expand": True}) 90 | -------------------------------------------------------------------------------- /tests/test_ml.py: -------------------------------------------------------------------------------- 1 | from sympy import symbols 2 | 3 | from tensorgrad import Variable 4 | from tensorgrad import functions as F 5 | from tensorgrad.tensor import function 6 | 7 | 8 | def test_attention(): 9 | batch, seq, dim, inner, head = symbols("batch, seq, dim, inner, head") 10 | X = Variable("X", batch, seq, dim) 11 | W_q = Variable("W_q", dim, inner, head) 12 | W_k = Variable("W_k", dim, inner, head) 13 | W_v = Variable("W_v", dim, inner, head) 14 | W_o = Variable("W_o", dim, inner, head) 15 | 16 | query = (W_q @ X).rename(seq="seq_q") 17 | key = (W_k @ X).rename(seq="seq_k") 18 | value = (W_v @ X).rename(seq="seq_k") 19 | logits = F.dot(query, key, dim="inner") 20 | attention_scores = function("softmax", {"seq_k": seq}, (logits, "seq_k")) 21 | expr = F.dot(value, attention_scores, dim="seq_k") 22 | expr = F.sum(W_o * expr, ["inner", "head"]) 23 | 24 | 25 | def test_attention_2(): 26 | batch, seq, dim, inner, head = symbols("batch, seq, dim, inner, head") 27 | X = Variable("X", batch, seq, dim) 28 | W_q = Variable("W_q", dim, inner, head) 29 | W_k = Variable("W_k", dim, inner, head) 30 | W_v = Variable("W_v", dim, inner, head) 31 | W_o = Variable("W_o", dim, inner, head) 32 | 33 | logits = F.graph( 34 | """ 35 | X1 -dim- Wq 36 | X2 -dim- Wk 37 | X2 -seq-sk- 38 | Wq -inner- Wk 39 | """, 40 | X1=X, 41 | X2=X, 42 | Wq=W_q, 43 | Wk=W_k, 44 | ) 45 | assert logits.edges == {"batch", "seq", "sk", "head"} 46 | 47 | attention_scores = F.softmax(logits, ["sk"]) 48 | 49 | expr = F.graph( 50 | """ 51 | X -seq-sk- attention_scores -head- *0 52 | X -dim- Wv -head- *0 53 | *0 -head- Wo 54 | Wv -inner- Wo 55 | """, 56 | X=X, 57 | Wv=W_v, 58 | Wo=W_o, 59 | attention_scores=attention_scores, 60 | ) 61 | assert expr.edges == {"batch", "seq", "dim"} 62 | -------------------------------------------------------------------------------- /tests/test_powers.py: -------------------------------------------------------------------------------- 1 | from sympy import symbols 2 | import tensorgrad.functions as F 3 | from tensorgrad import Delta 4 | 5 | 6 | def test_combine_powers(): 7 | i = symbols("i") 8 | out = F._PowerFunction._combine_powers( 9 | [ 10 | F.pow(Delta(i), k=2), 11 | Delta(i), 12 | ] 13 | ) 14 | assert F.pow(Delta(i), k=3) in out 15 | 16 | out = F._PowerFunction._combine_powers( 17 | [ 18 | F.pow(Delta(i), k=2), 19 | F.pow(Delta(i), k=2), 20 | ] 21 | ) 22 | assert F.pow(Delta(i), k=4) in out 23 | 24 | out = F._PowerFunction._combine_powers( 25 | [ 26 | F.pow(Delta(i), k=5), 27 | F.pow(Delta(i), k=-2), 28 | ] 29 | ) 30 | assert F.pow(Delta(i), k=3) in out 31 | 32 | out = F._PowerFunction._combine_powers( 33 | [ 34 | F.pow(Delta(i), k=5), 35 | F.pow(Delta(i), k=-2), 36 | Delta(i), 37 | ] 38 | ) 39 | assert F.pow(Delta(i), k=4) in out 40 | -------------------------------------------------------------------------------- /tests/test_product.py: -------------------------------------------------------------------------------- 1 | from sympy import symbols 2 | from tensorgrad import Variable, Product 3 | 4 | 5 | def test_components(): 6 | i = symbols("i") 7 | V = Variable("V", i) 8 | t = Product([V, V]) 9 | assert t.components() == [t] 10 | -------------------------------------------------------------------------------- /tests/test_rename.py: -------------------------------------------------------------------------------- 1 | from sympy import symbols 2 | 3 | from tensorgrad.tensor import Delta, Variable 4 | import tensorgrad.functions as F 5 | 6 | 7 | def test_interaction_with_copy(): 8 | i = symbols("i") 9 | x0 = Variable("x", i) 10 | x1 = x0.rename(i="i_") 11 | c0 = Delta(i, "i") 12 | c1 = Delta(i, "i_") 13 | assert (x0 @ c0).simplify() == (x1 @ c1).simplify() 14 | 15 | 16 | def test_interaction_with_copy2(): 17 | i = symbols("i") 18 | x = Variable("x", i) 19 | y = Variable("y", i_=i) 20 | assert (x.rename(i="i_") @ y).simplify() == (x @ Delta(i, "i", "i_") @ y).simplify() 21 | assert (x @ y).simplify() != (x @ Delta(i, "i", "i_") @ y).simplify() 22 | 23 | 24 | def test_rename_with_function(): 25 | i = symbols("i") 26 | x = Variable("x", i) 27 | a = (F.relu(x) @ Delta(i, "i")).simplify() 28 | b = (F.relu(x.rename(i="i_")) @ Delta(i, "i_")).simplify() 29 | c = (F.relu(x).rename(i="i_") @ Delta(i, "i_")).simplify() 30 | print(a.graph_to_string()) 31 | print(b.graph_to_string()) 32 | print(c.graph_to_string()) 33 | assert b == c 34 | assert a == b 35 | assert a == c 36 | -------------------------------------------------------------------------------- /tests/test_simplify.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sympy import symbols 3 | from tensorgrad.tensor import Delta, Product, Sum, Variable 4 | 5 | 6 | def test_copy_loop(): 7 | i = symbols("i") 8 | expr = Delta(i, "i, j") @ Delta(i, "i, j") 9 | assert expr.simplify() == Delta(i) 10 | 11 | 12 | def test_copy_double(): 13 | p = symbols("p") 14 | expr = Delta(p, "p0, p2, p0_") @ Delta(p, "p0, p0_") 15 | assert expr.simplify() == Delta(p, "p2") 16 | 17 | 18 | def test_copy_trace(): 19 | p = symbols("p") 20 | expr = Delta(p, "p, p1") @ Delta(p, "p, p1") 21 | expected = Delta(p, "p") @ Delta(p, "p") 22 | assert expr.simplify() == expected.simplify() 23 | 24 | 25 | def test_copy_iso(): 26 | # We can't just assume two copies are the same, in particular if they have 27 | # edge dimensions linked to different variables. 28 | i, j = symbols("i j") 29 | X = Variable("X", i, j) 30 | c1, c2 = X.grad(X).factors 31 | assert c1 != c2 32 | 33 | 34 | def test_copy_iso_sym(): 35 | # If X is symmetric, X.i and X.j are the same, so the two identity matrices 36 | # created are the same. 37 | i = symbols("i") 38 | X = Variable("X", i=i, j=i).with_symmetries("i j") 39 | c1, c2 = X.grad(X).factors 40 | assert c1 == c2 41 | 42 | 43 | def test_bad_size_sym(): 44 | # If X is symmetric, X.i and X.j are the same, so the two identity matrices 45 | # created are the same. 46 | i, j = symbols("i, j") 47 | with pytest.raises(ValueError): 48 | Variable("X", i, j).with_symmetries("i j") 49 | 50 | 51 | def test_expand(): 52 | i, j = symbols("i, j") 53 | X = Variable("X", i, j) 54 | a = Variable("a", i) 55 | b = Variable("b", j) 56 | expr = X @ (a + b) 57 | expr = expr.simplify({"expand": True}) 58 | assert isinstance(expr, Sum) 59 | assert expr == Sum([Product([Delta(j, "j"), X, a]), Product([Delta(i, "i"), X, b])]) 60 | -------------------------------------------------------------------------------- /tests/test_variable.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sympy import symbols 3 | from tensorgrad import Variable, Delta, Product, Zero 4 | 5 | 6 | def test_variable_initialization(): 7 | i, k = symbols("i k") 8 | v = Variable("x", i=i, j=i, k=k).with_symmetries("i j, k") 9 | assert v.name == "x" 10 | assert v.edges == {"i", "j", "k"} 11 | assert v.symmetries == {frozenset({"i", "j"}), frozenset({"k"})} 12 | 13 | 14 | def test_variable_repr(): 15 | i, k = symbols("i k") 16 | v = Variable("x", i=i, j=i, k=k).with_symmetries("i j, k") 17 | assert repr(v) == 'Variable("x", i, j, k).with_symmetries("i j, k")' 18 | 19 | 20 | def test_variable_rename(): 21 | i, k, a = symbols("i k a") 22 | v = Variable("x", i=i, j=i, k=k).with_symmetries("i j, k") 23 | v_renamed = v.rename(i="a") 24 | assert v_renamed.edges == {"a", "j", "k"} 25 | assert v_renamed.symmetries == {frozenset({"a", "j"}), frozenset({"k"})} 26 | 27 | 28 | def test_variable_grad(): 29 | i, j, k = symbols("i j k") 30 | v = Variable("x", i=i, j=j) 31 | grad_v = v.grad(v, {"i": "di", "j": "dj"}) 32 | assert isinstance(grad_v, Product) 33 | assert len(grad_v.factors) == 2 34 | assert all(isinstance(t, Delta) for t in grad_v.factors) 35 | assert grad_v.factors[0].edges == {"i", "di"} 36 | assert grad_v.factors[1].edges == {"j", "dj"} 37 | 38 | w = Variable("y", k) 39 | grad_w = v.grad(w, {"k": "dk"}) 40 | assert isinstance(grad_w, Zero) 41 | assert grad_w.edges == {"i", "j", "dk"} 42 | 43 | 44 | def test_variable_depends_on(): 45 | i, j, k = symbols("i j k") 46 | v = Variable("x", i=i, j=j) 47 | w = Variable("y", k=k) 48 | assert v.depends_on(v) 49 | assert not v.depends_on(w) 50 | 51 | 52 | def test_variable_symmetries(): 53 | i, k, l, a = symbols("i k l a") 54 | 55 | # We can't make symmetries for edges that don't have the same size 56 | with pytest.raises(ValueError): 57 | Variable("x", i=i, j=i, k=k, l=l).with_symmetries("i j k, l") 58 | 59 | v = Variable("x", i=i, j=i, k=i, l=l).with_symmetries("i j k, l") 60 | assert v.symmetries == {frozenset({"i", "j", "k"}), frozenset({"l"})} 61 | 62 | # Test that symmetries are preserved after renaming 63 | v_renamed = v.rename(i="a") 64 | assert v_renamed.symmetries == {frozenset({"a", "j", "k"}), frozenset({"l"})} 65 | 66 | 67 | def test_variable_invalid_symmetries(): 68 | i, j = symbols("i j") 69 | with pytest.raises(ValueError): 70 | Variable("x", i, j).with_symmetries("i j, k") # 'k' is not in edges 71 | 72 | 73 | def test_transpose_grad(): 74 | i = symbols("i") 75 | X = Variable("X", i=i, j=i).with_symmetries("i j") 76 | Xt = X.rename(i="j", j="i") 77 | assert X == Xt 78 | assert (X + Xt).simplify() == (2 * X).simplify() 79 | 80 | 81 | def test_transpose_not_equal(): 82 | # If we don't have symmetries, the transpose is not equal to M 83 | d = symbols("d") 84 | M = Variable("M", i=d, j=d) 85 | Mt = M.rename(i="j", j="i") 86 | assert M == Mt # Nomal equality allows for transposes 87 | assert not M.is_isomorphic(Mt, match_edges=True) 88 | 89 | 90 | def test_duplicate_positional_dims_error(): 91 | """ 92 | Creating a Variable with duplicate positional dimensions should raise an error. 93 | """ 94 | from tensorgrad import Variable 95 | 96 | i = symbols("i") 97 | with pytest.raises(ValueError, match="Duplicate positional dimension names"): 98 | Variable("x", i, i) 99 | 100 | --------------------------------------------------------------------------------