├── tests ├── __init__.py ├── strategies.py ├── test_scalar.py ├── test_autodiff.py ├── test_module.py └── test_operators.py ├── setup.py ├── files_to_sync.txt ├── requirements.extra.txt ├── requirements.txt ├── minitorch ├── __init__.py ├── optim.py ├── datasets.py ├── autodiff.py ├── module.py ├── testing.py ├── operators.py ├── scalar_functions.py └── scalar.py ├── README.md ├── project ├── show_expression_interface.py ├── interface │ ├── streamlit_utils.py │ ├── train.py │ └── plots.py ├── module_interface.py ├── run_manual.py ├── show_expression.py ├── graph_builder.py ├── run_torch.py ├── run_scalar.py ├── app.py └── math_interface.py ├── setup.cfg ├── .pre-commit-config.yaml ├── sync_previous_module.py └── .gitignore /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup(py_modules=[]) 4 | -------------------------------------------------------------------------------- /files_to_sync.txt: -------------------------------------------------------------------------------- 1 | minitorch/operators.py 2 | minitorch/module.py 3 | tests/test_module.py 4 | tests/test_operators.py 5 | project/run_manual.py -------------------------------------------------------------------------------- /requirements.extra.txt: -------------------------------------------------------------------------------- 1 | datasets==2.4.0 2 | embeddings==0.0.8 3 | networkx==2.4 4 | plotly==4.14.3 5 | pydot==1.4.1 6 | python-mnist 7 | streamlit==1.12.0 8 | streamlit-ace 9 | torch 10 | watchdog==1.0.2 11 | altair<5 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | colorama==0.4.3 2 | hypothesis == 6.54 3 | mypy == 0.971 4 | numba == 0.56 5 | numpy == 1.22 6 | pre-commit == 2.20.0 7 | pytest == 7.1.2 8 | pytest-env 9 | pytest-runner == 5.2 10 | typing_extensions 11 | -------------------------------------------------------------------------------- /minitorch/__init__.py: -------------------------------------------------------------------------------- 1 | from .autodiff import * # noqa: F401,F403 2 | from .datasets import * # noqa: F401,F403 3 | from .module import * # noqa: F401,F403 4 | from .optim import * # noqa: F401,F403 5 | from .scalar import * # noqa: F401,F403 6 | from .scalar_functions import * # noqa: F401,F403 7 | from .testing import * # noqa: F401,F403 8 | from .testing import MathTest, MathTestVariable # type: ignore # noqa: F401,F403 9 | -------------------------------------------------------------------------------- /tests/strategies.py: -------------------------------------------------------------------------------- 1 | from hypothesis import settings 2 | from hypothesis.strategies import floats, integers 3 | 4 | import minitorch 5 | 6 | settings.register_profile("ci", deadline=None) 7 | settings.load_profile("ci") 8 | 9 | 10 | small_ints = integers(min_value=1, max_value=3) 11 | small_floats = floats(min_value=-100, max_value=100, allow_nan=False) 12 | med_ints = integers(min_value=1, max_value=20) 13 | 14 | 15 | def assert_close(a: float, b: float) -> None: 16 | assert minitorch.operators.is_close(a, b), "Failure x=%f y=%f" % (a, b) 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MiniTorch Module 1 2 | 3 | 4 | 5 | * Docs: https://minitorch.github.io/ 6 | 7 | * Overview: https://minitorch.github.io/module1/module1/ 8 | 9 | This assignment requires the following files from the previous assignments. You can get these by running 10 | 11 | ```bash 12 | python sync_previous_module.py previous-module-dir current-module-dir 13 | ``` 14 | 15 | The files that will be synced are: 16 | 17 | minitorch/operators.py minitorch/module.py tests/test_module.py tests/test_operators.py project/run_manual.py 18 | -------------------------------------------------------------------------------- /project/show_expression_interface.py: -------------------------------------------------------------------------------- 1 | import graph_builder 2 | import networkx as nx 3 | import streamlit as st 4 | from streamlit_ace import st_ace 5 | 6 | 7 | def render_show_expression(tensor=False): 8 | 9 | if tensor: 10 | st.text("Build an expression of tensors x, y, and z. (All the same shape)") 11 | code = st_ace( 12 | language="python", height=300, value="(x * x) * y + 10.0 * x.sum()" 13 | ) 14 | out = graph_builder.build_tensor_expression(code) 15 | else: 16 | code = st_ace(language="python", height=300, value="(x * x) * y + 10.0 * x") 17 | out = graph_builder.build_expression(code) 18 | 19 | G = graph_builder.GraphBuilder().run(out) 20 | G.graph["graph"] = {"rankdir": "LR"} 21 | st.graphviz_chart(nx.nx_pydot.to_pydot(G).to_string()) 22 | -------------------------------------------------------------------------------- /project/interface/streamlit_utils.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | import streamlit as st 4 | 5 | img_id_counter = 0 6 | 7 | 8 | def get_image_id(): 9 | global img_id_counter 10 | img_id_counter += 1 11 | return img_id_counter 12 | 13 | 14 | def get_img_tag(src, width=None): 15 | img_id = get_image_id() 16 | if width is not None: 17 | style = """ 18 | 23 | """.format( 24 | img_id, width 25 | ) 26 | else: 27 | style = "" 28 | return """ 29 | img-{} 30 | {} 31 | """.format( 32 | src, img_id, img_id, style 33 | ) 34 | 35 | 36 | def render_function(fn): 37 | st.markdown( 38 | """ 39 | ```python 40 | %s 41 | 42 | ```""" 43 | % inspect.getsource(fn) 44 | ) 45 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name=minitorch 3 | version=0.4 4 | 5 | [files] 6 | packages = 7 | minitorch 8 | 9 | [darglint] 10 | ignore_regex=((^_(.*))|(.*map)|(.*zip)|(.*reduce)|(test.*)|(tensor_.*)) 11 | docstring_style=google 12 | strictness=long 13 | 14 | [flake8] 15 | ignore = N801, E203, E266, E501, W503, F812, E741, N803, N802, N806 16 | exclude = .git,__pycache__,docs/*,old,build,dist 17 | 18 | [isort] 19 | profile=black 20 | src_paths=minitorch,test 21 | 22 | [mypy] 23 | strict = True 24 | ignore_missing_imports = True 25 | exclude=^(docs/)|(project/)|(assignments/) 26 | implicit_reexport = True 27 | 28 | [mypy-tests.*] 29 | disallow_untyped_decorators = False 30 | implicit_reexport = True 31 | 32 | [black] 33 | exclude=^(docs/)|(project/)|(assignments/) 34 | 35 | [tool:pytest] 36 | markers = 37 | task0_0 38 | task0_1 39 | task0_2 40 | task0_3 41 | task0_4 42 | task1_0 43 | task1_1 44 | task1_2 45 | task1_3 46 | task1_4 47 | task2_0 48 | task2_1 49 | task2_2 50 | task2_3 51 | task2_4 52 | task3_0 53 | task3_1 54 | task3_2 55 | task3_3 56 | task3_4 57 | task4_0 58 | task4_1 59 | task4_2 60 | task4_3 61 | task4_4 62 | -------------------------------------------------------------------------------- /minitorch/optim.py: -------------------------------------------------------------------------------- 1 | from typing import Sequence 2 | 3 | from .module import Parameter 4 | from .scalar import Scalar 5 | 6 | 7 | class Optimizer: 8 | def __init__(self, parameters: Sequence[Parameter]): 9 | self.parameters = parameters 10 | 11 | 12 | class SGD(Optimizer): 13 | def __init__(self, parameters: Sequence[Parameter], lr: float = 1.0): 14 | super().__init__(parameters) 15 | self.lr = lr 16 | 17 | def zero_grad(self) -> None: 18 | for p in self.parameters: 19 | if p.value is None: 20 | continue 21 | if hasattr(p.value, "derivative"): 22 | if p.value.derivative is not None: 23 | p.value.derivative = None 24 | if hasattr(p.value, "grad"): 25 | if p.value.grad is not None: 26 | p.value.grad = None 27 | 28 | def step(self) -> None: 29 | for p in self.parameters: 30 | if p.value is None: 31 | continue 32 | if hasattr(p.value, "derivative"): 33 | if p.value.derivative is not None: 34 | p.update(Scalar(p.value.data - self.lr * p.value.derivative)) 35 | elif hasattr(p.value, "grad"): 36 | if p.value.grad is not None: 37 | p.update(p.value - self.lr * p.value.grad) 38 | -------------------------------------------------------------------------------- /project/module_interface.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import streamlit as st 3 | from streamlit_ace import st_ace 4 | 5 | import minitorch 6 | 7 | MyModule = None 8 | minitorch 9 | 10 | 11 | def render_module_sandbox(): 12 | st.write("## Sandbox for Module Trees") 13 | 14 | st.write( 15 | "Visual debugging checks showing the module tree that your code constructs." 16 | ) 17 | 18 | code = st_ace( 19 | language="python", 20 | height=300, 21 | value=""" 22 | class MyModule(minitorch.Module): 23 | def __init__(self): 24 | super().__init__() 25 | self.parameter1 = minitorch.Parameter(15) 26 | """, 27 | ) 28 | out = exec(code, globals()) 29 | out = MyModule() 30 | st.write(dict(out.named_parameters())) 31 | G = nx.MultiDiGraph() 32 | G.add_node("base") 33 | stack = [(out, "base")] 34 | 35 | while stack: 36 | n, name = stack[0] 37 | stack = stack[1:] 38 | for pname, p in n.__dict__["_parameters"].items(): 39 | G.add_node(name + "." + pname, shape="rect", penwidth=0.5) 40 | G.add_edge(name, name + "." + pname) 41 | 42 | for cname, m in n.__dict__["_modules"].items(): 43 | G.add_edge(name, name + "." + cname) 44 | stack.append((m, name + "." + cname)) 45 | 46 | G.graph["graph"] = {"rankdir": "TB"} 47 | st.graphviz_chart(nx.nx_pydot.to_pydot(G).to_string()) 48 | -------------------------------------------------------------------------------- /project/run_manual.py: -------------------------------------------------------------------------------- 1 | """ 2 | Be sure you have minitorch installed in you Virtual Env. 3 | >>> pip install -Ue . 4 | """ 5 | import random 6 | 7 | import minitorch 8 | 9 | 10 | class Network(minitorch.Module): 11 | def __init__(self): 12 | super().__init__() 13 | self.linear = Linear(2, 1) 14 | 15 | def forward(self, x): 16 | y = self.linear(x) 17 | return minitorch.operators.sigmoid(y[0]) 18 | 19 | 20 | class Linear(minitorch.Module): 21 | def __init__(self, in_size, out_size): 22 | super().__init__() 23 | random.seed(100) 24 | self.weights = [] 25 | self.bias = [] 26 | for i in range(in_size): 27 | weights = [] 28 | for j in range(out_size): 29 | w = self.add_parameter(f"weight_{i}_{j}", 2 * (random.random() - 0.5)) 30 | weights.append(w) 31 | self.weights.append(weights) 32 | for j in range(out_size): 33 | b = self.add_parameter(f"bias_{j}", 2 * (random.random() - 0.5)) 34 | self.bias.append(b) 35 | 36 | def forward(self, inputs): 37 | y = [b.value for b in self.bias] 38 | for i, x in enumerate(inputs): 39 | for j in range(len(y)): 40 | y[j] = y[j] + x * self.weights[i][j].value 41 | return y 42 | 43 | 44 | class ManualTrain: 45 | def __init__(self, hidden_layers): 46 | self.model = Network() 47 | 48 | def run_one(self, x): 49 | return self.model.forward((x[0], x[1])) 50 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # To use: 2 | # 3 | # pre-commit run -a 4 | # 5 | # Or: 6 | # 7 | # pre-commit install # (runs every time you commit in git) 8 | # 9 | # To update this file: 10 | # 11 | # pre-commit autoupdate 12 | # 13 | # See https://github.com/pre-commit/pre-commit 14 | 15 | repos: 16 | # Standard hooks 17 | - repo: https://github.com/pre-commit/pre-commit-hooks 18 | rev: v4.3.0 19 | hooks: 20 | - id: check-added-large-files 21 | - id: check-case-conflict 22 | - id: check-docstring-first 23 | - id: check-merge-conflict 24 | - id: check-symlinks 25 | - id: check-toml 26 | - id: debug-statements 27 | - id: mixed-line-ending 28 | - id: requirements-txt-fixer 29 | - id: trailing-whitespace 30 | 31 | - repo: https://github.com/timothycrosley/isort 32 | rev: 5.10.1 33 | hooks: 34 | - id: isort 35 | 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v0.971 38 | hooks: 39 | - id: mypy 40 | exclude: ^(docs/)|(project/)|(assignments/) 41 | 42 | 43 | # Black, the code formatter, natively supports pre-commit 44 | - repo: https://github.com/psf/black 45 | rev: 22.6.0 46 | hooks: 47 | - id: black 48 | 49 | # Flake8 also supports pre-commit natively (same author) 50 | - repo: https://github.com/PyCQA/flake8 51 | rev: 5.0.4 52 | hooks: 53 | - id: flake8 54 | additional_dependencies: 55 | - pep8-naming 56 | exclude: ^(docs/)|(assignments/) 57 | 58 | # Doc linters 59 | - repo: https://github.com/terrencepreilly/darglint 60 | rev: v1.8.1 61 | hooks: 62 | - id: darglint 63 | -------------------------------------------------------------------------------- /sync_previous_module.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: 3 | Note: Make sure that both the new and old module files are in same directory! 4 | 5 | This script helps you sync your previous module works with current modules. 6 | It takes 2 arguments, source_dir_name and destination_dir_name. 7 | All the files which will be moved are specified in files_to_sync.txt as newline separated strings 8 | 9 | Usage: python sync_previous_module.py 10 | 11 | Ex: python sync_previous_module.py mle-module-0-sauravpanda24 mle-module-1-sauravpanda24 12 | """ 13 | import os 14 | import shutil 15 | import sys 16 | 17 | if len(sys.argv) != 3: 18 | print( 19 | "Invalid argument count! Please pass source directory and destination directory after the file name" 20 | ) 21 | sys.exit() 22 | 23 | # Get the users path to evaluate the username and root directory 24 | current_path = os.getcwd() 25 | grandparent_path = "/".join(current_path.split("/")[:-1]) 26 | 27 | print("Looking for modules in : ", grandparent_path) 28 | 29 | # List of files which we want to move 30 | f = open("files_to_sync.txt", "r+") 31 | files_to_move = f.read().splitlines() 32 | f.close() 33 | 34 | # get the source and destination from arguments 35 | source = sys.argv[1] 36 | dest = sys.argv[2] 37 | 38 | # copy the files from source to destination 39 | try: 40 | for file in files_to_move: 41 | print(f"Moving file : ", file) 42 | shutil.copy( 43 | os.path.join(grandparent_path, source, file), 44 | os.path.join(grandparent_path, dest, file), 45 | ) 46 | print(f"Finished moving {len(files_to_move)} files") 47 | except Exception as e: 48 | print( 49 | "Something went wrong! please check if the source and destination folders are present in same folder" 50 | ) 51 | -------------------------------------------------------------------------------- /minitorch/datasets.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | from dataclasses import dataclass 4 | from typing import List, Tuple 5 | 6 | 7 | def make_pts(N: int) -> List[Tuple[float, float]]: 8 | X = [] 9 | for i in range(N): 10 | x_1 = random.random() 11 | x_2 = random.random() 12 | X.append((x_1, x_2)) 13 | return X 14 | 15 | 16 | @dataclass 17 | class Graph: 18 | N: int 19 | X: List[Tuple[float, float]] 20 | y: List[int] 21 | 22 | 23 | def simple(N: int) -> Graph: 24 | X = make_pts(N) 25 | y = [] 26 | for x_1, x_2 in X: 27 | y1 = 1 if x_1 < 0.5 else 0 28 | y.append(y1) 29 | return Graph(N, X, y) 30 | 31 | 32 | def diag(N: int) -> Graph: 33 | X = make_pts(N) 34 | y = [] 35 | for x_1, x_2 in X: 36 | y1 = 1 if x_1 + x_2 < 0.5 else 0 37 | y.append(y1) 38 | return Graph(N, X, y) 39 | 40 | 41 | def split(N: int) -> Graph: 42 | X = make_pts(N) 43 | y = [] 44 | for x_1, x_2 in X: 45 | y1 = 1 if x_1 < 0.2 or x_1 > 0.8 else 0 46 | y.append(y1) 47 | return Graph(N, X, y) 48 | 49 | 50 | def xor(N: int) -> Graph: 51 | X = make_pts(N) 52 | y = [] 53 | for x_1, x_2 in X: 54 | y1 = 1 if ((x_1 < 0.5 and x_2 > 0.5) or (x_1 > 0.5 and x_2 < 0.5)) else 0 55 | y.append(y1) 56 | return Graph(N, X, y) 57 | 58 | 59 | def circle(N: int) -> Graph: 60 | X = make_pts(N) 61 | y = [] 62 | for x_1, x_2 in X: 63 | x1, x2 = (x_1 - 0.5, x_2 - 0.5) 64 | y1 = 1 if x1 * x1 + x2 * x2 > 0.1 else 0 65 | y.append(y1) 66 | return Graph(N, X, y) 67 | 68 | 69 | def spiral(N: int) -> Graph: 70 | def x(t: float) -> float: 71 | return t * math.cos(t) / 20.0 72 | 73 | def y(t: float) -> float: 74 | return t * math.sin(t) / 20.0 75 | 76 | X = [ 77 | (x(10.0 * (float(i) / (N // 2))) + 0.5, y(10.0 * (float(i) / (N // 2))) + 0.5) 78 | for i in range(5 + 0, 5 + N // 2) 79 | ] 80 | X = X + [ 81 | (y(-10.0 * (float(i) / (N // 2))) + 0.5, x(-10.0 * (float(i) / (N // 2))) + 0.5) 82 | for i in range(5 + 0, 5 + N // 2) 83 | ] 84 | y2 = [0] * (N // 2) + [1] * (N // 2) 85 | return Graph(N, X, y2) 86 | 87 | 88 | datasets = { 89 | "Simple": simple, 90 | "Diag": diag, 91 | "Split": split, 92 | "Xor": xor, 93 | "Circle": circle, 94 | "Spiral": spiral, 95 | } 96 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | *.\#* 131 | data/ 132 | pyodide -------------------------------------------------------------------------------- /project/show_expression.py: -------------------------------------------------------------------------------- 1 | """ 2 | Be sure you have the extra requirements installed. 3 | 4 | >>> pip install -r requirements.extra.txt 5 | """ 6 | 7 | import networkx as nx 8 | 9 | import minitorch 10 | 11 | 12 | ## Create an autodiff expression here. 13 | def expression(): 14 | x = minitorch.Scalar(1.0, name="x") 15 | y = minitorch.Scalar(1.0, name="y") 16 | z = (x * x) * y + 10.0 * x 17 | z.name = "z" 18 | return z 19 | 20 | 21 | class GraphBuilder: 22 | def __init__(self): 23 | self.op_id = 0 24 | self.hid = 0 25 | self.intermediates = {} 26 | 27 | def get_name(self, x): 28 | if not isinstance(x, minitorch.Scalar): 29 | return "constant %s" % (x,) 30 | elif len(x.name) > 15: 31 | if x.name in self.intermediates: 32 | return "v%d" % (self.intermediates[x.name],) 33 | else: 34 | self.hid = self.hid + 1 35 | self.intermediates[x.name] = self.hid 36 | return "v%d" % (self.hid,) 37 | else: 38 | return x.name 39 | 40 | def run(self, final): 41 | queue = [[final]] 42 | 43 | G = nx.MultiDiGraph() 44 | G.add_node(self.get_name(final)) 45 | 46 | while queue: 47 | (cur,) = queue[0] 48 | queue = queue[1:] 49 | 50 | if cur.history is None: 51 | continue 52 | elif cur.is_leaf(): 53 | continue 54 | else: 55 | op = "%s (Op %d)" % (cur.history.last_fn.__name__, self.op_id) 56 | G.add_node(op, shape="square", penwidth=3) 57 | G.add_edge(op, self.get_name(cur)) 58 | self.op_id += 1 59 | for i, input in enumerate(cur.history.inputs): 60 | G.add_edge(self.get_name(input), op, f"{i}") 61 | 62 | for input in cur.history.inputs: 63 | if not isinstance(input, minitorch.Scalar): 64 | continue 65 | 66 | seen = False 67 | for s in queue: 68 | if s[0] == input: 69 | seen = True 70 | if not seen: 71 | queue.append([input]) 72 | return G 73 | 74 | 75 | def make_graph(y, lr=False): 76 | G = GraphBuilder().run(y) 77 | if lr: 78 | G.graph["graph"] = {"rankdir": "LR"} 79 | output_graphviz_svg = nx.nx_pydot.to_pydot(G).create_svg() 80 | return output_graphviz_svg 81 | -------------------------------------------------------------------------------- /project/graph_builder.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | 3 | import minitorch 4 | 5 | 6 | def build_expression(code): 7 | out = eval( 8 | code, 9 | { 10 | "x": minitorch.Scalar(1.0, name="x"), 11 | "y": minitorch.Scalar(1.0, name="y"), 12 | "z": minitorch.Scalar(1.0, name="z"), 13 | }, 14 | ) 15 | out.name = "out" 16 | return out 17 | 18 | 19 | def build_tensor_expression(code): 20 | 21 | variables = { 22 | "x": minitorch.tensor([[1.0, 2.0, 3.0]], requires_grad=True), 23 | "y": minitorch.tensor([[1.0, 2.0, 3.0]], requires_grad=True), 24 | "z": minitorch.tensor([[1.0, 2.0, 3.0]], requires_grad=True), 25 | } 26 | variables["x"].name = "x" 27 | variables["y"].name = "y" 28 | variables["z"].name = "z" 29 | 30 | out = eval(code, variables) 31 | out.name = "out" 32 | return out 33 | 34 | 35 | class GraphBuilder: 36 | def __init__(self): 37 | self.op_id = 0 38 | self.hid = 0 39 | self.intermediates = {} 40 | 41 | def get_name(self, x): 42 | if not isinstance(x, minitorch.Scalar) and not isinstance(x, minitorch.Tensor): 43 | return "constant %s" % (x,) 44 | elif len(x.name) > 15: 45 | if x.name in self.intermediates: 46 | return "v%d" % (self.intermediates[x.name],) 47 | else: 48 | self.hid = self.hid + 1 49 | self.intermediates[x.name] = self.hid 50 | return "v%d" % (self.hid,) 51 | else: 52 | return x.name 53 | 54 | def run(self, final): 55 | queue = [[final]] 56 | 57 | G = nx.MultiDiGraph() 58 | G.add_node(self.get_name(final)) 59 | 60 | while queue: 61 | (cur,) = queue[0] 62 | queue = queue[1:] 63 | 64 | if cur.is_constant() or cur.is_leaf(): 65 | continue 66 | else: 67 | op = "%s (Op %d)" % (cur.history.last_fn.__name__, self.op_id) 68 | G.add_node(op, shape="square", penwidth=3) 69 | G.add_edge(op, self.get_name(cur)) 70 | self.op_id += 1 71 | for i, input in enumerate(cur.history.inputs): 72 | G.add_edge(self.get_name(input), op, f"{i}") 73 | 74 | for input in cur.history.inputs: 75 | if not isinstance(input, minitorch.Scalar) and not isinstance( 76 | input, minitorch.Tensor 77 | ): 78 | continue 79 | queue.append([input]) 80 | return G 81 | -------------------------------------------------------------------------------- /project/run_torch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | import minitorch 4 | 5 | 6 | def default_log_fn(epoch, total_loss, correct, losses): 7 | print("Epoch ", epoch, " loss ", total_loss, "correct", correct) 8 | 9 | 10 | class Network(torch.nn.Module): 11 | def __init__(self, hidden_layers): 12 | super().__init__() 13 | 14 | # Submodules 15 | self.layer1 = Linear(2, hidden_layers) 16 | self.layer2 = Linear(hidden_layers, hidden_layers) 17 | self.layer3 = Linear(hidden_layers, 1) 18 | 19 | def forward(self, x): 20 | h = self.layer1.forward(x).relu() 21 | h = self.layer2.forward(h).relu() 22 | return self.layer3.forward(h).sigmoid() 23 | 24 | 25 | class Linear(torch.nn.Module): 26 | def __init__(self, in_size, out_size): 27 | super().__init__() 28 | self.weight = torch.nn.Parameter(2 * (torch.rand((in_size, out_size)) - 0.5)) 29 | self.bias = torch.nn.Parameter(2 * (torch.rand((out_size,)) - 0.5)) 30 | 31 | def forward(self, x): 32 | return x @ self.weight + self.bias 33 | 34 | 35 | class TorchTrain: 36 | def __init__(self, hidden_layers): 37 | self.hidden_layers = hidden_layers 38 | self.model = Network(hidden_layers) 39 | 40 | def run_one(self, x): 41 | return self.model.forward(torch.tensor([x])) 42 | 43 | def run_many(self, X): 44 | return self.model.forward(torch.tensor(X)).detach() 45 | 46 | def train( 47 | self, 48 | data, 49 | learning_rate, 50 | max_epochs=500, 51 | log_fn=default_log_fn, 52 | ): 53 | self.model = Network(self.hidden_layers) 54 | self.max_epochs = max_epochs 55 | model = self.model 56 | 57 | losses = [] 58 | for epoch in range(1, max_epochs + 1): 59 | 60 | # Forward 61 | out = model.forward(torch.tensor(data.X, requires_grad=True)).view(data.N) 62 | y = torch.tensor(data.y) 63 | probs = (out * y) + (out - 1.0) * (y - 1.0) 64 | loss = -probs.log().sum() 65 | 66 | # Update 67 | loss.view(1).backward() 68 | 69 | for p in model.parameters(): 70 | if p.grad is not None: 71 | p.data = p.data - learning_rate * (p.grad / float(data.N)) 72 | p.grad.zero_() 73 | 74 | # Logging 75 | pred = out > 0.5 76 | correct = ((y == 1) * (pred)).sum() + ((y == 0) * (~pred)).sum() 77 | loss_num = loss.reshape(-1).item() 78 | losses.append(loss_num) 79 | 80 | if epoch % 10 == 0 or epoch == max_epochs: 81 | log_fn(epoch, loss_num, correct.item(), losses) 82 | 83 | 84 | if __name__ == "__main__": 85 | PTS = 250 86 | HIDDEN = 10 87 | RATE = 0.5 88 | TorchTrain(HIDDEN).train(minitorch.datasets["Xor"](PTS), RATE) 89 | -------------------------------------------------------------------------------- /minitorch/autodiff.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Any, Iterable, List, Tuple 3 | 4 | from typing_extensions import Protocol 5 | 6 | # ## Task 1.1 7 | # Central Difference calculation 8 | 9 | 10 | def central_difference(f: Any, *vals: Any, arg: int = 0, epsilon: float = 1e-6) -> Any: 11 | r""" 12 | Computes an approximation to the derivative of `f` with respect to one arg. 13 | 14 | See :doc:`derivative` or https://en.wikipedia.org/wiki/Finite_difference for more details. 15 | 16 | Args: 17 | f : arbitrary function from n-scalar args to one value 18 | *vals : n-float values $x_0 \ldots x_{n-1}$ 19 | arg : the number $i$ of the arg to compute the derivative 20 | epsilon : a small constant 21 | 22 | Returns: 23 | An approximation of $f'_i(x_0, \ldots, x_{n-1})$ 24 | """ 25 | # TODO: Implement for Task 1.1. 26 | raise NotImplementedError("Need to implement for Task 1.1") 27 | 28 | 29 | variable_count = 1 30 | 31 | 32 | class Variable(Protocol): 33 | def accumulate_derivative(self, x: Any) -> None: 34 | pass 35 | 36 | @property 37 | def unique_id(self) -> int: 38 | pass 39 | 40 | def is_leaf(self) -> bool: 41 | pass 42 | 43 | def is_constant(self) -> bool: 44 | pass 45 | 46 | @property 47 | def parents(self) -> Iterable["Variable"]: 48 | pass 49 | 50 | def chain_rule(self, d_output: Any) -> Iterable[Tuple["Variable", Any]]: 51 | pass 52 | 53 | 54 | def topological_sort(variable: Variable) -> Iterable[Variable]: 55 | """ 56 | Computes the topological order of the computation graph. 57 | 58 | Args: 59 | variable: The right-most variable 60 | 61 | Returns: 62 | Non-constant Variables in topological order starting from the right. 63 | """ 64 | # TODO: Implement for Task 1.4. 65 | raise NotImplementedError("Need to implement for Task 1.4") 66 | 67 | 68 | def backpropagate(variable: Variable, deriv: Any) -> None: 69 | """ 70 | Runs backpropagation on the computation graph in order to 71 | compute derivatives for the leave nodes. 72 | 73 | Args: 74 | variable: The right-most variable 75 | deriv : Its derivative that we want to propagate backward to the leaves. 76 | 77 | No return. Should write to its results to the derivative values of each leaf through `accumulate_derivative`. 78 | """ 79 | # TODO: Implement for Task 1.4. 80 | raise NotImplementedError("Need to implement for Task 1.4") 81 | 82 | 83 | @dataclass 84 | class Context: 85 | """ 86 | Context class is used by `Function` to store information during the forward pass. 87 | """ 88 | 89 | no_grad: bool = False 90 | saved_values: Tuple[Any, ...] = () 91 | 92 | def save_for_backward(self, *values: Any) -> None: 93 | "Store the given `values` if they need to be used during backpropagation." 94 | if self.no_grad: 95 | return 96 | self.saved_values = values 97 | 98 | @property 99 | def saved_tensors(self) -> Tuple[Any, ...]: 100 | return self.saved_values 101 | -------------------------------------------------------------------------------- /tests/test_scalar.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Tuple 2 | 3 | import pytest 4 | from hypothesis import given 5 | from hypothesis.strategies import DrawFn, composite, floats 6 | 7 | import minitorch 8 | from minitorch import ( 9 | MathTestVariable, 10 | Scalar, 11 | central_difference, 12 | derivative_check, 13 | operators, 14 | ) 15 | 16 | from .strategies import assert_close, small_floats 17 | 18 | 19 | @composite 20 | def scalars( 21 | draw: DrawFn, min_value: float = -100000, max_value: float = 100000 22 | ) -> Scalar: 23 | val = draw(floats(min_value=min_value, max_value=max_value)) 24 | return minitorch.Scalar(val) 25 | 26 | 27 | small_scalars = scalars(min_value=-100, max_value=100) 28 | 29 | 30 | # ## Task 1.1 - Test central difference 31 | 32 | 33 | @pytest.mark.task1_1 34 | def test_central_diff() -> None: 35 | d = central_difference(operators.id, 5, arg=0) 36 | assert_close(d, 1.0) 37 | 38 | d = central_difference(operators.add, 5, 10, arg=0) 39 | assert_close(d, 1.0) 40 | 41 | d = central_difference(operators.mul, 5, 10, arg=0) 42 | assert_close(d, 10.0) 43 | 44 | d = central_difference(operators.mul, 5, 10, arg=1) 45 | assert_close(d, 5.0) 46 | 47 | d = central_difference(operators.exp, 2, arg=0) 48 | assert_close(d, operators.exp(2.0)) 49 | 50 | 51 | # ## Task 1.2 - Test each of the different function types 52 | 53 | 54 | @given(small_floats, small_floats) 55 | def test_simple(a: float, b: float) -> None: 56 | # Simple add 57 | c = Scalar(a) + Scalar(b) 58 | assert_close(c.data, a + b) 59 | 60 | # Simple mul 61 | c = Scalar(a) * Scalar(b) 62 | assert_close(c.data, a * b) 63 | 64 | # Simple relu 65 | c = Scalar(a).relu() + Scalar(b).relu() 66 | assert_close(c.data, minitorch.operators.relu(a) + minitorch.operators.relu(b)) 67 | 68 | # Add others if you would like... 69 | 70 | 71 | one_arg, two_arg, _ = MathTestVariable._comp_testing() 72 | 73 | 74 | @given(small_scalars) 75 | @pytest.mark.task1_2 76 | @pytest.mark.parametrize("fn", one_arg) 77 | def test_one_args( 78 | fn: Tuple[str, Callable[[float], float], Callable[[Scalar], Scalar]], t1: Scalar 79 | ) -> None: 80 | name, base_fn, scalar_fn = fn 81 | assert_close(scalar_fn(t1).data, base_fn(t1.data)) 82 | 83 | 84 | @given(small_scalars, small_scalars) 85 | @pytest.mark.task1_2 86 | @pytest.mark.parametrize("fn", two_arg) 87 | def test_two_args( 88 | fn: Tuple[str, Callable[[float, float], float], Callable[[Scalar, Scalar], Scalar]], 89 | t1: Scalar, 90 | t2: Scalar, 91 | ) -> None: 92 | name, base_fn, scalar_fn = fn 93 | assert_close(scalar_fn(t1, t2).data, base_fn(t1.data, t2.data)) 94 | 95 | 96 | # ## Task 1.4 - Computes checks on each of the derivatives. 97 | 98 | # See minitorch.testing for all of the functions checked. 99 | 100 | 101 | @given(small_scalars) 102 | @pytest.mark.task1_4 103 | @pytest.mark.parametrize("fn", one_arg) 104 | def test_one_derivative( 105 | fn: Tuple[str, Callable[[float], float], Callable[[Scalar], Scalar]], t1: Scalar 106 | ) -> None: 107 | name, _, scalar_fn = fn 108 | derivative_check(scalar_fn, t1) 109 | 110 | 111 | @given(small_scalars, small_scalars) 112 | @pytest.mark.task1_4 113 | @pytest.mark.parametrize("fn", two_arg) 114 | def test_two_derivative( 115 | fn: Tuple[str, Callable[[float, float], float], Callable[[Scalar, Scalar], Scalar]], 116 | t1: Scalar, 117 | t2: Scalar, 118 | ) -> None: 119 | name, _, scalar_fn = fn 120 | derivative_check(scalar_fn, t1, t2) 121 | -------------------------------------------------------------------------------- /project/run_scalar.py: -------------------------------------------------------------------------------- 1 | """ 2 | Be sure you have minitorch installed in you Virtual Env. 3 | >>> pip install -Ue . 4 | """ 5 | import random 6 | 7 | import minitorch 8 | 9 | 10 | class Network(minitorch.Module): 11 | def __init__(self, hidden_layers): 12 | super().__init__() 13 | # TODO: Implement for Task 1.5. 14 | raise NotImplementedError("Need to implement for Task 1.5") 15 | 16 | def forward(self, x): 17 | middle = [h.relu() for h in self.layer1.forward(x)] 18 | end = [h.relu() for h in self.layer2.forward(middle)] 19 | return self.layer3.forward(end)[0].sigmoid() 20 | 21 | 22 | class Linear(minitorch.Module): 23 | def __init__(self, in_size, out_size): 24 | super().__init__() 25 | self.weights = [] 26 | self.bias = [] 27 | for i in range(in_size): 28 | self.weights.append([]) 29 | for j in range(out_size): 30 | self.weights[i].append( 31 | self.add_parameter( 32 | f"weight_{i}_{j}", minitorch.Scalar(2 * (random.random() - 0.5)) 33 | ) 34 | ) 35 | for j in range(out_size): 36 | self.bias.append( 37 | self.add_parameter( 38 | f"bias_{j}", minitorch.Scalar(2 * (random.random() - 0.5)) 39 | ) 40 | ) 41 | 42 | def forward(self, inputs): 43 | # TODO: Implement for Task 1.5. 44 | raise NotImplementedError("Need to implement for Task 1.5") 45 | 46 | 47 | def default_log_fn(epoch, total_loss, correct, losses): 48 | print("Epoch ", epoch, " loss ", total_loss, "correct", correct) 49 | 50 | 51 | class ScalarTrain: 52 | def __init__(self, hidden_layers): 53 | self.hidden_layers = hidden_layers 54 | self.model = Network(self.hidden_layers) 55 | 56 | def run_one(self, x): 57 | return self.model.forward( 58 | (minitorch.Scalar(x[0], name="x_1"), minitorch.Scalar(x[1], name="x_2")) 59 | ) 60 | 61 | def train(self, data, learning_rate, max_epochs=500, log_fn=default_log_fn): 62 | self.learning_rate = learning_rate 63 | self.max_epochs = max_epochs 64 | self.model = Network(self.hidden_layers) 65 | optim = minitorch.SGD(self.model.parameters(), learning_rate) 66 | 67 | losses = [] 68 | for epoch in range(1, self.max_epochs + 1): 69 | total_loss = 0.0 70 | correct = 0 71 | optim.zero_grad() 72 | 73 | # Forward 74 | loss = 0 75 | for i in range(data.N): 76 | x_1, x_2 = data.X[i] 77 | y = data.y[i] 78 | x_1 = minitorch.Scalar(x_1) 79 | x_2 = minitorch.Scalar(x_2) 80 | out = self.model.forward((x_1, x_2)) 81 | 82 | if y == 1: 83 | prob = out 84 | correct += 1 if out.data > 0.5 else 0 85 | else: 86 | prob = -out + 1.0 87 | correct += 1 if out.data < 0.5 else 0 88 | loss = -prob.log() 89 | (loss / data.N).backward() 90 | total_loss += loss.data 91 | 92 | losses.append(total_loss) 93 | 94 | # Update 95 | optim.step() 96 | 97 | # Logging 98 | if epoch % 10 == 0 or epoch == max_epochs: 99 | log_fn(epoch, total_loss, correct, losses) 100 | 101 | 102 | if __name__ == "__main__": 103 | PTS = 50 104 | HIDDEN = 2 105 | RATE = 0.5 106 | data = minitorch.datasets["Simple"](PTS) 107 | ScalarTrain(HIDDEN).train(data, RATE) 108 | -------------------------------------------------------------------------------- /project/app.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | import streamlit as st 4 | from interface.streamlit_utils import get_img_tag 5 | from interface.train import render_train_interface 6 | from math_interface import render_math_sandbox 7 | from run_torch import TorchTrain 8 | 9 | parser = ArgumentParser() 10 | parser.add_argument("module_num", type=int) 11 | parser.add_argument( 12 | "--hide_function_defs", action="store_true", dest="hide_function_defs" 13 | ) 14 | args = parser.parse_args() 15 | module_num = args.module_num 16 | hide_function_defs = args.hide_function_defs 17 | 18 | st.set_page_config(page_title="interactive minitorch") 19 | st.sidebar.markdown( 20 | """ 21 |

MiniTorch

{} 22 | """.format( 23 | get_img_tag("https://minitorch.github.io/_images/match.png", width="40") 24 | ), 25 | unsafe_allow_html=True, 26 | ) 27 | 28 | st.sidebar.markdown( 29 | """ 30 | [Documentation](https://minitorch.github.io/) 31 | """ 32 | ) 33 | 34 | module_selection = st.sidebar.radio( 35 | "Module", 36 | ["Module 0", "Module 1", "Module 2", "Module 3", "Module 4"][: module_num + 1], 37 | index=module_num, 38 | ) 39 | 40 | 41 | PAGES = {} 42 | 43 | if module_selection == "Module 0": 44 | from module_interface import render_module_sandbox 45 | from run_manual import ManualTrain 46 | 47 | def render_run_manual_interface(): 48 | st.header("Module 0 - Manual") 49 | render_train_interface(ManualTrain, False, False, True) 50 | 51 | def render_m0_sandbox(): 52 | return render_math_sandbox(False) 53 | 54 | PAGES["Math Sandbox"] = render_m0_sandbox 55 | PAGES["Module Sandbox"] = render_module_sandbox 56 | 57 | def render_run_torch_interface(): 58 | st.header("Demo - Torch") 59 | render_train_interface(TorchTrain, False) 60 | 61 | PAGES["Torch Example"] = render_run_torch_interface 62 | PAGES["Module 0: Manual"] = render_run_manual_interface 63 | 64 | if module_selection == "Module 1": 65 | from run_scalar import ScalarTrain 66 | from show_expression_interface import render_show_expression 67 | 68 | def render_m1_sandbox(): 69 | return render_math_sandbox(True) 70 | 71 | def render_run_scalar_interface(): 72 | st.header("Module 1 - Scalars") 73 | render_train_interface(ScalarTrain) 74 | 75 | PAGES["Scalar Sandbox"] = render_m1_sandbox 76 | PAGES["Autodiff Sandbox"] = render_show_expression 77 | PAGES["Module 1: Scalar"] = render_run_scalar_interface 78 | 79 | if module_selection == "Module 2": 80 | from run_tensor import TensorTrain 81 | from show_expression_interface import render_show_expression 82 | from tensor_interface import render_tensor_sandbox 83 | 84 | def render_run_tensor_interface(): 85 | st.header("Module 2 - Tensors") 86 | render_train_interface(TensorTrain) 87 | 88 | def render_m2_sandbox(): 89 | return render_math_sandbox(True, True) 90 | 91 | PAGES["Tensor Sandbox"] = lambda: render_tensor_sandbox(hide_function_defs) 92 | PAGES["Tensor Math Sandbox"] = render_m2_sandbox 93 | PAGES["Autograd Sandbox"] = lambda: render_show_expression(True) 94 | PAGES["Module 2: Tensor"] = render_run_tensor_interface 95 | 96 | 97 | if module_selection == "Module 3": 98 | from run_fast_tensor import FastTrain 99 | 100 | def render_run_fast_interface(): 101 | st.header("Module 3 - Efficient") 102 | render_train_interface(FastTrain, False) 103 | 104 | PAGES["Module 3: Efficient"] = render_run_fast_interface 105 | 106 | if module_selection == "Module 4": 107 | from run_mnist_interface import render_run_image_interface 108 | from sentiment_interface import render_run_sentiment_interface 109 | 110 | PAGES["Module 4: Images"] = render_run_image_interface 111 | PAGES["Module 4: Sentiment"] = render_run_sentiment_interface 112 | 113 | 114 | PAGE_OPTIONS = list(PAGES.keys()) 115 | 116 | page_selection = st.sidebar.radio("Pages", PAGE_OPTIONS) 117 | page = PAGES[page_selection] 118 | page() 119 | -------------------------------------------------------------------------------- /tests/test_autodiff.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import pytest 4 | 5 | import minitorch 6 | from minitorch import Context, ScalarFunction, ScalarHistory 7 | 8 | # ## Task 1.3 - Tests for the autodifferentiation machinery. 9 | 10 | # Simple sanity check and debugging tests. 11 | 12 | 13 | class Function1(ScalarFunction): 14 | @staticmethod 15 | def forward(ctx: Context, x: float, y: float) -> float: 16 | "$f(x, y) = x + y + 10$" 17 | return x + y + 10 18 | 19 | @staticmethod 20 | def backward(ctx: Context, d_output: float) -> Tuple[float, float]: 21 | "Derivatives are $f'_x(x, y) = 1$ and $f'_y(x, y) = 1$" 22 | return d_output, d_output 23 | 24 | 25 | class Function2(ScalarFunction): 26 | @staticmethod 27 | def forward(ctx: Context, x: float, y: float) -> float: 28 | "$f(x, y) = x \times y + x$" 29 | ctx.save_for_backward(x, y) 30 | return x * y + x 31 | 32 | @staticmethod 33 | def backward(ctx: Context, d_output: float) -> Tuple[float, float]: 34 | "Derivatives are $f'_x(x, y) = y + 1$ and $f'_y(x, y) = x$" 35 | x, y = ctx.saved_values 36 | return d_output * (y + 1), d_output * x 37 | 38 | 39 | # Checks for the chain rule function. 40 | 41 | 42 | @pytest.mark.task1_3 43 | def test_chain_rule1() -> None: 44 | x = minitorch.Scalar(0.0) 45 | constant = minitorch.Scalar( 46 | 0.0, ScalarHistory(Function1, ctx=Context(), inputs=[x, x]) 47 | ) 48 | back = constant.chain_rule(d_output=5) 49 | assert len(list(back)) == 2 50 | 51 | 52 | @pytest.mark.task1_3 53 | def test_chain_rule2() -> None: 54 | var = minitorch.Scalar(0.0, ScalarHistory()) 55 | constant = minitorch.Scalar( 56 | 0.0, ScalarHistory(Function1, ctx=Context(), inputs=[var, var]) 57 | ) 58 | back = constant.chain_rule(d_output=5) 59 | back = list(back) 60 | assert len(back) == 2 61 | variable, deriv = back[0] 62 | assert deriv == 5 63 | 64 | 65 | @pytest.mark.task1_3 66 | def test_chain_rule3() -> None: 67 | "Check that constrants are ignored and variables get derivatives." 68 | constant = 10 69 | var = minitorch.Scalar(5) 70 | 71 | y = Function2.apply(constant, var) 72 | 73 | back = y.chain_rule(d_output=5) 74 | back = list(back) 75 | assert len(back) == 2 76 | variable, deriv = back[1] 77 | # assert variable.name == var.name 78 | assert deriv == 5 * 10 79 | 80 | 81 | @pytest.mark.task1_3 82 | def test_chain_rule4() -> None: 83 | var1 = minitorch.Scalar(5) 84 | var2 = minitorch.Scalar(10) 85 | 86 | y = Function2.apply(var1, var2) 87 | 88 | back = y.chain_rule(d_output=5) 89 | back = list(back) 90 | assert len(back) == 2 91 | variable, deriv = back[0] 92 | # assert variable.name == var1.name 93 | assert deriv == 5 * (10 + 1) 94 | variable, deriv = back[1] 95 | # assert variable.name == var2.name 96 | assert deriv == 5 * 5 97 | 98 | 99 | # ## Task 1.4 - Run some simple backprop tests 100 | 101 | # Main tests are in test_scalar.py 102 | 103 | 104 | @pytest.mark.task1_4 105 | def test_backprop1() -> None: 106 | # Example 1: F1(0, v) 107 | var = minitorch.Scalar(0) 108 | var2 = Function1.apply(0, var) 109 | var2.backward(d_output=5) 110 | assert var.derivative == 5 111 | 112 | 113 | @pytest.mark.task1_4 114 | def test_backprop2() -> None: 115 | # Example 2: F1(0, 0) 116 | var = minitorch.Scalar(0) 117 | var2 = Function1.apply(0, var) 118 | var3 = Function1.apply(0, var2) 119 | var3.backward(d_output=5) 120 | assert var.derivative == 5 121 | 122 | 123 | @pytest.mark.task1_4 124 | def test_backprop3() -> None: 125 | # Example 3: F1(F1(0, v1), F1(0, v1)) 126 | var1 = minitorch.Scalar(0) 127 | var2 = Function1.apply(0, var1) 128 | var3 = Function1.apply(0, var1) 129 | var4 = Function1.apply(var2, var3) 130 | var4.backward(d_output=5) 131 | assert var1.derivative == 10 132 | 133 | 134 | @pytest.mark.task1_4 135 | def test_backprop4() -> None: 136 | # Example 4: F1(F1(0, v1), F1(0, v1)) 137 | var0 = minitorch.Scalar(0) 138 | var1 = Function1.apply(0, var0) 139 | var2 = Function1.apply(0, var1) 140 | var3 = Function1.apply(0, var1) 141 | var4 = Function1.apply(var2, var3) 142 | var4.backward(d_output=5) 143 | assert var0.derivative == 10 144 | -------------------------------------------------------------------------------- /project/interface/train.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import graph_builder 4 | import interface.plots as plots 5 | import networkx as nx 6 | import pandas as pd 7 | import plotly.graph_objects as go 8 | import streamlit as st 9 | 10 | import minitorch 11 | 12 | 13 | def render_train_interface( 14 | TrainCls, graph=True, hidden_layer=True, parameter_control=False 15 | ): 16 | datasets_map = minitorch.datasets 17 | st.write("## Sandbox for Model Training") 18 | 19 | st.markdown("### Dataset") 20 | col1, col2 = st.columns(2) 21 | points = col2.slider("Number of points", min_value=1, max_value=150, value=50) 22 | selected_dataset = col1.selectbox("Select dataset", list(datasets_map.keys())) 23 | 24 | @st.cache 25 | def get_dataset(selected_dataset, points): 26 | return datasets_map[selected_dataset](points) 27 | 28 | dataset = get_dataset(selected_dataset, points) 29 | 30 | fig = plots.plot_out(dataset) 31 | fig.update_layout(width=600, height=600) 32 | st.plotly_chart(fig) 33 | 34 | st.markdown("### Model") 35 | if hidden_layer: 36 | hidden_layers = st.number_input( 37 | "Size of hidden layer", min_value=1, max_value=200, step=1, value=2 38 | ) 39 | else: 40 | hidden_layers = 0 41 | 42 | @st.cache 43 | def get_train(hidden_layers): 44 | train = TrainCls(hidden_layers) 45 | one_output = train.run_one(dataset.X[0]) 46 | G = graph_builder.GraphBuilder().run(one_output) 47 | return nx.nx_pydot.to_pydot(G).to_string() 48 | 49 | train = TrainCls(hidden_layers) 50 | if graph: 51 | graph = get_train(hidden_layers) 52 | if st.checkbox("Show Graph"): 53 | st.graphviz_chart(graph) 54 | 55 | if parameter_control: 56 | st.markdown("### Parameters") 57 | for n, p in train.model.named_parameters(): 58 | value = st.slider( 59 | f"Parameter: {n}", min_value=-10.0, max_value=10.0, value=p.value 60 | ) 61 | p.update(value) 62 | 63 | oned = st.checkbox("Show X-Axis Only (For Simple)", False) 64 | 65 | def plot(): 66 | if hasattr(train, "run_many"): 67 | 68 | def contour(ls): 69 | t = train.run_many(ls) 70 | return [t[i, 0] for i in range(len(ls))] 71 | 72 | else: 73 | 74 | def contour(ls): 75 | out = [train.run_one(x) for x in ls] 76 | out = [(x.data if hasattr(x, "data") else x) for x in out] 77 | return out 78 | 79 | fig = plots.plot_out(dataset, contour, size=15, oned=oned) 80 | fig.update_layout(width=600, height=600) 81 | return fig 82 | 83 | st.markdown("### Initial setting") 84 | st.write(plot()) 85 | 86 | if hasattr(train, "train"): 87 | st.markdown("### Hyperparameters") 88 | col1, col2 = st.columns(2) 89 | learning_rate = col1.selectbox( 90 | "Learning rate", [0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0], index=2 91 | ) 92 | 93 | max_epochs = col2.number_input( 94 | "Number of epochs", min_value=1, step=25, value=500 95 | ) 96 | 97 | col1, col2 = st.columns(2) 98 | st_train_button = col1.empty() 99 | col2.button("Stop Model") 100 | 101 | st_progress = st.empty() 102 | st_epoch_timer = st.empty() 103 | st_epoch_image = st.empty() 104 | st_epoch_plot = st.empty() 105 | st_epoch_stats = st.empty() 106 | 107 | start_time = time.time() 108 | 109 | df = [] 110 | 111 | def log_fn(epoch, total_loss, correct, losses): 112 | time_elapsed = time.time() - start_time 113 | if hasattr(train, "train"): 114 | st_progress.progress(epoch / max_epochs) 115 | time_per_epoch = time_elapsed / (epoch + 1) 116 | st_epoch_timer.markdown( 117 | "Epoch {}/{}. Time per epoch: {:,.3f}s. Time left: {:,.2f}s.".format( 118 | epoch, 119 | max_epochs, 120 | time_per_epoch, 121 | (max_epochs - epoch) * time_per_epoch, 122 | ) 123 | ) 124 | df.append({"epoch": epoch, "loss": total_loss, "correct": correct}) 125 | st_epoch_stats.write(pd.DataFrame(reversed(df))) 126 | 127 | st_epoch_image.plotly_chart(plot()) 128 | if hasattr(train, "train"): 129 | loss_graph = go.Scatter(mode="lines", x=list(range(len(losses))), y=losses) 130 | fig = go.Figure(loss_graph) 131 | fig.update_layout( 132 | title="Loss Graph", 133 | xaxis=dict(range=[0, max_epochs]), 134 | yaxis=dict(range=[0, max(losses)]), 135 | ) 136 | st_epoch_plot.plotly_chart(fig) 137 | 138 | print( 139 | f"Epoch: {epoch}/{max_epochs}, loss: {total_loss}, correct: {correct}" 140 | ) 141 | 142 | if hasattr(train, "train") and st_train_button.button("Train Model"): 143 | train.train(dataset, learning_rate, max_epochs, log_fn) 144 | else: 145 | log_fn(0, 0, 0, [0]) 146 | -------------------------------------------------------------------------------- /minitorch/module.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any, Dict, Optional, Sequence, Tuple 4 | 5 | 6 | class Module: 7 | """ 8 | Modules form a tree that store parameters and other 9 | submodules. They make up the basis of neural network stacks. 10 | 11 | Attributes: 12 | _modules : Storage of the child modules 13 | _parameters : Storage of the module's parameters 14 | training : Whether the module is in training mode or evaluation mode 15 | 16 | """ 17 | 18 | _modules: Dict[str, Module] 19 | _parameters: Dict[str, Parameter] 20 | training: bool 21 | 22 | def __init__(self) -> None: 23 | self._modules = {} 24 | self._parameters = {} 25 | self.training = True 26 | 27 | def modules(self) -> Sequence[Module]: 28 | "Return the direct child modules of this module." 29 | m: Dict[str, Module] = self.__dict__["_modules"] 30 | return list(m.values()) 31 | 32 | def train(self) -> None: 33 | "Set the mode of this module and all descendent modules to `train`." 34 | raise NotImplementedError("Need to include this file from past assignment.") 35 | 36 | def eval(self) -> None: 37 | "Set the mode of this module and all descendent modules to `eval`." 38 | raise NotImplementedError("Need to include this file from past assignment.") 39 | 40 | def named_parameters(self) -> Sequence[Tuple[str, Parameter]]: 41 | """ 42 | Collect all the parameters of this module and its descendents. 43 | 44 | 45 | Returns: 46 | The name and `Parameter` of each ancestor parameter. 47 | """ 48 | raise NotImplementedError("Need to include this file from past assignment.") 49 | 50 | def parameters(self) -> Sequence[Parameter]: 51 | "Enumerate over all the parameters of this module and its descendents." 52 | raise NotImplementedError("Need to include this file from past assignment.") 53 | 54 | def add_parameter(self, k: str, v: Any) -> Parameter: 55 | """ 56 | Manually add a parameter. Useful helper for scalar parameters. 57 | 58 | Args: 59 | k: Local name of the parameter. 60 | v: Value for the parameter. 61 | 62 | Returns: 63 | Newly created parameter. 64 | """ 65 | val = Parameter(v, k) 66 | self.__dict__["_parameters"][k] = val 67 | return val 68 | 69 | def __setattr__(self, key: str, val: Parameter) -> None: 70 | if isinstance(val, Parameter): 71 | self.__dict__["_parameters"][key] = val 72 | elif isinstance(val, Module): 73 | self.__dict__["_modules"][key] = val 74 | else: 75 | super().__setattr__(key, val) 76 | 77 | def __getattr__(self, key: str) -> Any: 78 | if key in self.__dict__["_parameters"]: 79 | return self.__dict__["_parameters"][key] 80 | 81 | if key in self.__dict__["_modules"]: 82 | return self.__dict__["_modules"][key] 83 | return None 84 | 85 | def __call__(self, *args: Any, **kwargs: Any) -> Any: 86 | return self.forward(*args, **kwargs) 87 | 88 | def __repr__(self) -> str: 89 | def _addindent(s_: str, numSpaces: int) -> str: 90 | s2 = s_.split("\n") 91 | if len(s2) == 1: 92 | return s_ 93 | first = s2.pop(0) 94 | s2 = [(numSpaces * " ") + line for line in s2] 95 | s = "\n".join(s2) 96 | s = first + "\n" + s 97 | return s 98 | 99 | child_lines = [] 100 | 101 | for key, module in self._modules.items(): 102 | mod_str = repr(module) 103 | mod_str = _addindent(mod_str, 2) 104 | child_lines.append("(" + key + "): " + mod_str) 105 | lines = child_lines 106 | 107 | main_str = self.__class__.__name__ + "(" 108 | if lines: 109 | # simple one-liner info, which most builtin Modules will use 110 | main_str += "\n " + "\n ".join(lines) + "\n" 111 | 112 | main_str += ")" 113 | return main_str 114 | 115 | 116 | class Parameter: 117 | """ 118 | A Parameter is a special container stored in a `Module`. 119 | 120 | It is designed to hold a `Variable`, but we allow it to hold 121 | any value for testing. 122 | """ 123 | 124 | def __init__(self, x: Any, name: Optional[str] = None) -> None: 125 | self.value = x 126 | self.name = name 127 | if hasattr(x, "requires_grad_"): 128 | self.value.requires_grad_(True) 129 | if self.name: 130 | self.value.name = self.name 131 | 132 | def update(self, x: Any) -> None: 133 | "Update the parameter value." 134 | self.value = x 135 | if hasattr(x, "requires_grad_"): 136 | self.value.requires_grad_(True) 137 | if self.name: 138 | self.value.name = self.name 139 | 140 | def __repr__(self) -> str: 141 | return repr(self.value) 142 | 143 | def __str__(self) -> str: 144 | return str(self.value) 145 | -------------------------------------------------------------------------------- /tests/test_module.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from hypothesis import given 3 | 4 | import minitorch 5 | 6 | from .strategies import med_ints, small_floats 7 | 8 | # # Tests for module.py 9 | 10 | 11 | # ## Website example 12 | 13 | # This example builds a module 14 | # as shown at https://minitorch.github.io/modules.html 15 | # and checks that its properties work. 16 | 17 | 18 | class ModuleA1(minitorch.Module): 19 | def __init__(self) -> None: 20 | super().__init__() 21 | self.p1 = minitorch.Parameter(5) 22 | self.non_param = 10 23 | self.a = ModuleA2() 24 | self.b = ModuleA3() 25 | 26 | 27 | class ModuleA2(minitorch.Module): 28 | def __init__(self) -> None: 29 | super().__init__() 30 | self.p2 = minitorch.Parameter(10) 31 | 32 | 33 | class ModuleA3(minitorch.Module): 34 | def __init__(self) -> None: 35 | super().__init__() 36 | self.c = ModuleA4() 37 | 38 | 39 | class ModuleA4(minitorch.Module): 40 | def __init__(self) -> None: 41 | super().__init__() 42 | self.p3 = minitorch.Parameter(15) 43 | 44 | 45 | @pytest.mark.task0_4 46 | def test_stacked_demo() -> None: 47 | "Check that each of the properties match" 48 | mod = ModuleA1() 49 | np = dict(mod.named_parameters()) 50 | 51 | x = str(mod) 52 | print(x) 53 | assert mod.p1.value == 5 54 | assert mod.non_param == 10 55 | 56 | assert np["p1"].value == 5 57 | assert np["a.p2"].value == 10 58 | assert np["b.c.p3"].value == 15 59 | 60 | 61 | # ## Advanced Tests 62 | 63 | # These tests generate a stack of modules of varying sizes to check 64 | # properties. 65 | 66 | VAL_A = 50.0 67 | VAL_B = 100.0 68 | 69 | 70 | class Module1(minitorch.Module): 71 | def __init__(self, size_a: int, size_b: int, val: float) -> None: 72 | super().__init__() 73 | self.module_a = Module2(size_a) 74 | self.module_b = Module2(size_b) 75 | self.parameter_a = minitorch.Parameter(val) 76 | 77 | 78 | class Module2(minitorch.Module): 79 | def __init__(self, extra: int = 0) -> None: 80 | super().__init__() 81 | self.parameter_a = minitorch.Parameter(VAL_A) 82 | self.parameter_b = minitorch.Parameter(VAL_B) 83 | self.non_parameter = 10 84 | self.module_c = Module3() 85 | for i in range(extra): 86 | self.add_parameter(f"extra_parameter_{i}", 0) 87 | 88 | 89 | class Module3(minitorch.Module): 90 | def __init__(self) -> None: 91 | super().__init__() 92 | self.parameter_a = minitorch.Parameter(VAL_A) 93 | 94 | 95 | @pytest.mark.task0_4 96 | @given(med_ints, med_ints) 97 | def test_module(size_a: int, size_b: int) -> None: 98 | "Check the properties of a single module" 99 | module = Module2() 100 | module.eval() 101 | assert not module.training 102 | module.train() 103 | assert module.training 104 | assert len(module.parameters()) == 3 105 | 106 | module = Module2(size_b) 107 | assert len(module.parameters()) == size_b + 3 108 | 109 | module = Module2(size_a) 110 | named_parameters = dict(module.named_parameters()) 111 | assert named_parameters["parameter_a"].value == VAL_A 112 | assert named_parameters["parameter_b"].value == VAL_B 113 | assert named_parameters["extra_parameter_0"].value == 0 114 | 115 | 116 | @pytest.mark.task0_4 117 | @given(med_ints, med_ints, small_floats) 118 | def test_stacked_module(size_a: int, size_b: int, val: float) -> None: 119 | "Check the properties of a stacked module" 120 | module = Module1(size_a, size_b, val) 121 | module.eval() 122 | assert not module.training 123 | assert not module.module_a.training 124 | assert not module.module_b.training 125 | module.train() 126 | assert module.training 127 | assert module.module_a.training 128 | assert module.module_b.training 129 | 130 | assert len(module.parameters()) == 1 + (size_a + 3) + (size_b + 3) 131 | 132 | named_parameters = dict(module.named_parameters()) 133 | assert named_parameters["parameter_a"].value == val 134 | assert named_parameters["module_a.parameter_a"].value == VAL_A 135 | assert named_parameters["module_a.parameter_b"].value == VAL_B 136 | assert named_parameters["module_b.parameter_a"].value == VAL_A 137 | assert named_parameters["module_b.parameter_b"].value == VAL_B 138 | 139 | 140 | # ## Misc Tests 141 | 142 | # Check that the module runs forward correctly. 143 | 144 | 145 | class ModuleRun(minitorch.Module): 146 | def forward(self) -> int: 147 | return 10 148 | 149 | 150 | @pytest.mark.task0_4 151 | @pytest.mark.xfail 152 | def test_module_fail_forward() -> None: 153 | mod = minitorch.Module() 154 | mod() 155 | 156 | 157 | @pytest.mark.task0_4 158 | def test_module_forward() -> None: 159 | mod = ModuleRun() 160 | assert mod.forward() == 10 161 | 162 | # Calling directly should call forward 163 | assert mod() == 10 164 | 165 | 166 | # Internal check for the system. 167 | 168 | 169 | class MockParam: 170 | def __init__(self) -> None: 171 | self.x = False 172 | 173 | def requires_grad_(self, x: bool) -> None: 174 | self.x = x 175 | 176 | 177 | def test_parameter() -> None: 178 | t = MockParam() 179 | q = minitorch.Parameter(t) 180 | print(q) 181 | assert t.x 182 | t2 = MockParam() 183 | q.update(t2) 184 | assert t2.x 185 | -------------------------------------------------------------------------------- /project/interface/plots.py: -------------------------------------------------------------------------------- 1 | import plotly.graph_objects as go 2 | 3 | 4 | def make_scatters(graph, model=None, size=50): 5 | color_map = ["#69bac9", "#ea8484"] 6 | symbol_map = ["circle-dot", "x"] 7 | colors = [color_map[y] for y in graph.y] 8 | symbols = [symbol_map[y] for y in graph.y] 9 | scatters = [] 10 | 11 | if model is not None: 12 | colorscale = [[0, "#69bac9"], [1.0, "#ea8484"]] 13 | z = [ 14 | model([[j / (size + 1.0), k / (size + 1.0)] for j in range(size + 1)]) 15 | for k in range(size + 1) 16 | ] 17 | scatters.append( 18 | go.Contour( 19 | z=z, 20 | dx=1 / size, 21 | x0=0, 22 | dy=1 / size, 23 | y0=0, 24 | zmin=0.2, 25 | zmax=0.8, 26 | line_smoothing=0.5, 27 | colorscale=colorscale, 28 | opacity=0.6, 29 | showscale=False, 30 | ) 31 | ) 32 | scatters.append( 33 | go.Scatter( 34 | mode="markers", 35 | x=[p[0] for p in graph.X], 36 | y=[p[1] for p in graph.X], 37 | marker_symbol=symbols, 38 | marker_color=colors, 39 | marker=dict(size=15, line=dict(width=3, color="Black")), 40 | ) 41 | ) 42 | return scatters 43 | 44 | 45 | def animate(self, models, names): 46 | import plotly.graph_objects as go 47 | 48 | scatters = [make_scatters(self, m) for m in models] 49 | background = [s[0] for s in scatters] 50 | for i, b in enumerate(background): 51 | b["visible"] = i == 0 52 | points = scatters[0][1] 53 | steps = [] 54 | for i in range(len(background)): 55 | step = dict( 56 | method="update", 57 | args=[ 58 | {"visible": [False] * len(background) + [True]}, 59 | {}, 60 | ], # layout attribute 61 | label="%1.3f" % names[i], 62 | ) 63 | step["args"][0]["visible"][i] = True # Toggle i'th trace to "visible" 64 | steps.append(step) 65 | 66 | sliders = [ 67 | dict(active=0, currentvalue={"prefix": "b="}, pad={"t": 50}, steps=steps) 68 | ] 69 | 70 | fig = go.Figure( 71 | data=background + [points], 72 | ) 73 | fig.update_layout(sliders=sliders) 74 | 75 | fig.update_layout( 76 | template="simple_white", 77 | xaxis={ 78 | "showgrid": False, # thin lines in the background 79 | "zeroline": False, # thick line at x=0 80 | "visible": False, # numbers below 81 | }, 82 | yaxis={ 83 | "showgrid": False, # thin lines in the background 84 | "zeroline": False, # thick line at x=0 85 | "visible": False, # numbers below 86 | }, 87 | ) 88 | fig.show() 89 | 90 | 91 | def make_oned(graph, model=None, size=50): 92 | scatters = [] 93 | color_map = ["#69bac9", "#ea8484"] 94 | symbol_map = ["circle-dot", "x"] 95 | colors = [color_map[y] for y in graph.y] 96 | symbols = [symbol_map[y] for y in graph.y] 97 | 98 | if model is not None: 99 | # colorscale = [[0, "#69bac9"], [1.0, "#ea8484"]] 100 | y = model([[j / (size + 1.0), 0.0] for j in range(size + 1)]) 101 | 102 | x = [j / (size + 1.0) for j in range(size + 1)] 103 | scatters.append( 104 | go.Scatter( 105 | mode="lines", 106 | x=[j / (size + 1.0) for j in range(size + 1)], 107 | y=y, 108 | marker=dict(size=15, line=dict(width=3, color="Black")), 109 | ) 110 | ) 111 | print(x, y) 112 | scatters.append( 113 | go.Scatter( 114 | mode="markers", 115 | x=[p[0] for p in graph.X], 116 | y=graph.y, 117 | marker_symbol=symbols, 118 | marker_color=colors, 119 | marker=dict(size=15, line=dict(width=3, color="Black")), 120 | ) 121 | ) 122 | return scatters 123 | 124 | 125 | def plot_out(graph, model=None, name="", size=50, oned=False): 126 | if oned: 127 | scatters = make_oned(graph, model, size=size) 128 | else: 129 | scatters = make_scatters(graph, model, size=size) 130 | 131 | fig = go.Figure(scatters) 132 | fig.update_layout( 133 | xaxis={ 134 | "showgrid": False, # thin lines in the background 135 | "visible": False, # numbers below 136 | "range": [0, 1], 137 | }, 138 | yaxis={ 139 | "showgrid": False, # thin lines in the background 140 | "visible": False, # numbers below 141 | "range": [0, 1], 142 | }, 143 | ) 144 | return fig 145 | 146 | 147 | def plot(graph, model=None, name=""): 148 | plot_out(graph, model, name).show() 149 | 150 | 151 | def plot_function(title, fn, arange=[(i / 10.0) - 5 for i in range(0, 100)], fn2=None): 152 | ys = [fn(x) for x in arange] 153 | scatters = [] 154 | scatter = go.Scatter(x=arange, y=ys) 155 | scatters.append(scatter) 156 | if fn2 is not None: 157 | ys = [fn2(x) for x in arange] 158 | scatter2 = go.Scatter(x=arange, y=ys) 159 | scatters.append(scatter2) 160 | fig = go.Figure(scatters) 161 | fig.update_layout(template="simple_white", title=title) 162 | 163 | return fig.show() 164 | 165 | 166 | def plot_function3D(title, fn, arange=[(i / 5.0) - 4.0 for i in range(0, 40)]): 167 | 168 | xs = [((x / 10.0) - 5.0 + 1e-5) for x in range(1, 100)] 169 | ys = [((x / 10.0) - 5.0 + 1e-5) for x in range(1, 100)] 170 | zs = [[fn(x, y) for x in xs] for y in ys] 171 | 172 | scatter = go.Surface(x=xs, y=ys, z=zs) 173 | 174 | fig = go.Figure(scatter) 175 | fig.update_layout(template="simple_white", title=title) 176 | 177 | return fig.show() 178 | -------------------------------------------------------------------------------- /minitorch/testing.py: -------------------------------------------------------------------------------- 1 | # type: ignore 2 | 3 | from typing import Callable, Generic, Iterable, Tuple, TypeVar 4 | 5 | import minitorch.operators as operators 6 | 7 | A = TypeVar("A") 8 | 9 | 10 | class MathTest(Generic[A]): 11 | @staticmethod 12 | def neg(a: A) -> A: 13 | "Negate the argument" 14 | return -a 15 | 16 | @staticmethod 17 | def addConstant(a: A) -> A: 18 | "Add contant to the argument" 19 | return 5 + a 20 | 21 | @staticmethod 22 | def square(a: A) -> A: 23 | "Manual square" 24 | return a * a 25 | 26 | @staticmethod 27 | def cube(a: A) -> A: 28 | "Manual cube" 29 | return a * a * a 30 | 31 | @staticmethod 32 | def subConstant(a: A) -> A: 33 | "Subtract a constant from the argument" 34 | return a - 5 35 | 36 | @staticmethod 37 | def multConstant(a: A) -> A: 38 | "Multiply a constant to the argument" 39 | return 5 * a 40 | 41 | @staticmethod 42 | def div(a: A) -> A: 43 | "Divide by a constant" 44 | return a / 5 45 | 46 | @staticmethod 47 | def inv(a: A) -> A: 48 | "Invert after adding" 49 | return operators.inv(a + 3.5) 50 | 51 | @staticmethod 52 | def sig(a: A) -> A: 53 | "Apply sigmoid" 54 | return operators.sigmoid(a) 55 | 56 | @staticmethod 57 | def log(a: A) -> A: 58 | "Apply log to a large value" 59 | return operators.log(a + 100000) 60 | 61 | @staticmethod 62 | def relu(a: A) -> A: 63 | "Apply relu" 64 | return operators.relu(a + 5.5) 65 | 66 | @staticmethod 67 | def exp(a: A) -> A: 68 | "Apply exp to a smaller value" 69 | return operators.exp(a - 200) 70 | 71 | @staticmethod 72 | def explog(a: A) -> A: 73 | return operators.log(a + 100000) + operators.exp(a - 200) 74 | 75 | @staticmethod 76 | def add2(a: A, b: A) -> A: 77 | "Add two arguments" 78 | return a + b 79 | 80 | @staticmethod 81 | def mul2(a: A, b: A) -> A: 82 | "Mul two arguments" 83 | return a * b 84 | 85 | @staticmethod 86 | def div2(a: A, b: A) -> A: 87 | "Divide two arguments" 88 | return a / (b + 5.5) 89 | 90 | @staticmethod 91 | def gt2(a: A, b: A) -> A: 92 | return operators.lt(b, a + 1.2) 93 | 94 | @staticmethod 95 | def lt2(a: A, b: A) -> A: 96 | return operators.lt(a + 1.2, b) 97 | 98 | @staticmethod 99 | def eq2(a: A, b: A) -> A: 100 | return operators.eq(a, (b + 5.5)) 101 | 102 | @staticmethod 103 | def sum_red(a: Iterable[A]) -> A: 104 | return operators.sum(a) 105 | 106 | @staticmethod 107 | def mean_red(a: Iterable[A]) -> A: 108 | return operators.sum(a) / float(len(a)) 109 | 110 | @staticmethod 111 | def mean_full_red(a: Iterable[A]) -> A: 112 | return operators.sum(a) / float(len(a)) 113 | 114 | @staticmethod 115 | def complex(a: A) -> A: 116 | return ( 117 | operators.log( 118 | operators.sigmoid( 119 | operators.relu(operators.relu(a * 10 + 7) * 6 + 5) * 10 120 | ) 121 | ) 122 | / 50 123 | ) 124 | 125 | @classmethod 126 | def _tests( 127 | cls, 128 | ) -> Tuple[ 129 | Tuple[str, Callable[[A], A]], 130 | Tuple[str, Callable[[A, A], A]], 131 | Tuple[str, Callable[[Iterable[A]], A]], 132 | ]: 133 | """ 134 | Returns a list of all the math tests. 135 | """ 136 | one_arg = [] 137 | two_arg = [] 138 | red_arg = [] 139 | for k in dir(MathTest): 140 | if callable(getattr(MathTest, k)) and not k.startswith("_"): 141 | base_fn = getattr(cls, k) 142 | # scalar_fn = getattr(cls, k) 143 | tup = (k, base_fn) 144 | if k.endswith("2"): 145 | two_arg.append(tup) 146 | elif k.endswith("red"): 147 | red_arg.append(tup) 148 | else: 149 | one_arg.append(tup) 150 | return one_arg, two_arg, red_arg 151 | 152 | @classmethod 153 | def _comp_testing(cls): 154 | one_arg, two_arg, red_arg = cls._tests() 155 | one_argv, two_argv, red_argv = MathTest._tests() 156 | one_arg = [(n1, f2, f1) for (n1, f1), (n2, f2) in zip(one_arg, one_argv)] 157 | two_arg = [(n1, f2, f1) for (n1, f1), (n2, f2) in zip(two_arg, two_argv)] 158 | red_arg = [(n1, f2, f1) for (n1, f1), (n2, f2) in zip(red_arg, red_argv)] 159 | return one_arg, two_arg, red_arg 160 | 161 | 162 | class MathTestVariable(MathTest): 163 | @staticmethod 164 | def inv(a): 165 | return 1.0 / (a + 3.5) 166 | 167 | @staticmethod 168 | def sig(x): 169 | return x.sigmoid() 170 | 171 | @staticmethod 172 | def log(x): 173 | return (x + 100000).log() 174 | 175 | @staticmethod 176 | def relu(x): 177 | return (x + 5.5).relu() 178 | 179 | @staticmethod 180 | def exp(a): 181 | return (a - 200).exp() 182 | 183 | @staticmethod 184 | def explog(a): 185 | return (a + 100000).log() + (a - 200).exp() 186 | 187 | @staticmethod 188 | def sum_red(a): 189 | return a.sum(0) 190 | 191 | @staticmethod 192 | def mean_red(a): 193 | return a.mean(0) 194 | 195 | @staticmethod 196 | def mean_full_red(a): 197 | return a.mean() 198 | 199 | @staticmethod 200 | def eq2(a, b): 201 | return a == (b + 5.5) 202 | 203 | @staticmethod 204 | def gt2(a, b): 205 | return a + 1.2 > b 206 | 207 | @staticmethod 208 | def lt2(a, b): 209 | return a + 1.2 < b 210 | 211 | @staticmethod 212 | def complex(a): 213 | return (((a * 10 + 7).relu() * 6 + 5).relu() * 10).sigmoid().log() / 50 214 | -------------------------------------------------------------------------------- /minitorch/operators.py: -------------------------------------------------------------------------------- 1 | """ 2 | Collection of the core mathematical operators used throughout the code base. 3 | """ 4 | 5 | import math 6 | from typing import Callable, Iterable 7 | 8 | # ## Task 0.1 9 | # 10 | # Implementation of a prelude of elementary functions. 11 | 12 | 13 | def mul(x: float, y: float) -> float: 14 | "$f(x, y) = x * y$" 15 | raise NotImplementedError("Need to include this file from past assignment.") 16 | 17 | 18 | def id(x: float) -> float: 19 | "$f(x) = x$" 20 | raise NotImplementedError("Need to include this file from past assignment.") 21 | 22 | 23 | def add(x: float, y: float) -> float: 24 | "$f(x, y) = x + y$" 25 | raise NotImplementedError("Need to include this file from past assignment.") 26 | 27 | 28 | def neg(x: float) -> float: 29 | "$f(x) = -x$" 30 | raise NotImplementedError("Need to include this file from past assignment.") 31 | 32 | 33 | def lt(x: float, y: float) -> float: 34 | "$f(x) =$ 1.0 if x is less than y else 0.0" 35 | raise NotImplementedError("Need to include this file from past assignment.") 36 | 37 | 38 | def eq(x: float, y: float) -> float: 39 | "$f(x) =$ 1.0 if x is equal to y else 0.0" 40 | raise NotImplementedError("Need to include this file from past assignment.") 41 | 42 | 43 | def max(x: float, y: float) -> float: 44 | "$f(x) =$ x if x is greater than y else y" 45 | raise NotImplementedError("Need to include this file from past assignment.") 46 | 47 | 48 | def is_close(x: float, y: float) -> float: 49 | "$f(x) = |x - y| < 1e-2$" 50 | raise NotImplementedError("Need to include this file from past assignment.") 51 | 52 | 53 | def sigmoid(x: float) -> float: 54 | r""" 55 | $f(x) = \frac{1.0}{(1.0 + e^{-x})}$ 56 | 57 | (See https://en.wikipedia.org/wiki/Sigmoid_function ) 58 | 59 | Calculate as 60 | 61 | $f(x) = \frac{1.0}{(1.0 + e^{-x})}$ if x >=0 else $\frac{e^x}{(1.0 + e^{x})}$ 62 | 63 | for stability. 64 | """ 65 | raise NotImplementedError("Need to include this file from past assignment.") 66 | 67 | 68 | def relu(x: float) -> float: 69 | """ 70 | $f(x) =$ x if x is greater than 0, else 0 71 | 72 | (See https://en.wikipedia.org/wiki/Rectifier_(neural_networks) .) 73 | """ 74 | raise NotImplementedError("Need to include this file from past assignment.") 75 | 76 | 77 | EPS = 1e-6 78 | 79 | 80 | def log(x: float) -> float: 81 | "$f(x) = log(x)$" 82 | return math.log(x + EPS) 83 | 84 | 85 | def exp(x: float) -> float: 86 | "$f(x) = e^{x}$" 87 | return math.exp(x) 88 | 89 | 90 | def log_back(x: float, d: float) -> float: 91 | r"If $f = log$ as above, compute $d \times f'(x)$" 92 | raise NotImplementedError("Need to include this file from past assignment.") 93 | 94 | 95 | def inv(x: float) -> float: 96 | "$f(x) = 1/x$" 97 | raise NotImplementedError("Need to include this file from past assignment.") 98 | 99 | 100 | def inv_back(x: float, d: float) -> float: 101 | r"If $f(x) = 1/x$ compute $d \times f'(x)$" 102 | raise NotImplementedError("Need to include this file from past assignment.") 103 | 104 | 105 | def relu_back(x: float, d: float) -> float: 106 | r"If $f = relu$ compute $d \times f'(x)$" 107 | raise NotImplementedError("Need to include this file from past assignment.") 108 | 109 | 110 | # ## Task 0.3 111 | 112 | # Small practice library of elementary higher-order functions. 113 | 114 | 115 | def map(fn: Callable[[float], float]) -> Callable[[Iterable[float]], Iterable[float]]: 116 | """ 117 | Higher-order map. 118 | 119 | See https://en.wikipedia.org/wiki/Map_(higher-order_function) 120 | 121 | Args: 122 | fn: Function from one value to one value. 123 | 124 | Returns: 125 | A function that takes a list, applies `fn` to each element, and returns a 126 | new list 127 | """ 128 | raise NotImplementedError("Need to include this file from past assignment.") 129 | 130 | 131 | def negList(ls: Iterable[float]) -> Iterable[float]: 132 | "Use `map` and `neg` to negate each element in `ls`" 133 | raise NotImplementedError("Need to include this file from past assignment.") 134 | 135 | 136 | def zipWith( 137 | fn: Callable[[float, float], float] 138 | ) -> Callable[[Iterable[float], Iterable[float]], Iterable[float]]: 139 | """ 140 | Higher-order zipwith (or map2). 141 | 142 | See https://en.wikipedia.org/wiki/Map_(higher-order_function) 143 | 144 | Args: 145 | fn: combine two values 146 | 147 | Returns: 148 | Function that takes two equally sized lists `ls1` and `ls2`, produce a new list by 149 | applying fn(x, y) on each pair of elements. 150 | 151 | """ 152 | raise NotImplementedError("Need to include this file from past assignment.") 153 | 154 | 155 | def addLists(ls1: Iterable[float], ls2: Iterable[float]) -> Iterable[float]: 156 | "Add the elements of `ls1` and `ls2` using `zipWith` and `add`" 157 | raise NotImplementedError("Need to include this file from past assignment.") 158 | 159 | 160 | def reduce( 161 | fn: Callable[[float, float], float], start: float 162 | ) -> Callable[[Iterable[float]], float]: 163 | r""" 164 | Higher-order reduce. 165 | 166 | Args: 167 | fn: combine two values 168 | start: start value $x_0$ 169 | 170 | Returns: 171 | Function that takes a list `ls` of elements 172 | $x_1 \ldots x_n$ and computes the reduction :math:`fn(x_3, fn(x_2, 173 | fn(x_1, x_0)))` 174 | """ 175 | raise NotImplementedError("Need to include this file from past assignment.") 176 | 177 | 178 | def sum(ls: Iterable[float]) -> float: 179 | "Sum up a list using `reduce` and `add`." 180 | raise NotImplementedError("Need to include this file from past assignment.") 181 | 182 | 183 | def prod(ls: Iterable[float]) -> float: 184 | "Product of a list using `reduce` and `mul`." 185 | raise NotImplementedError("Need to include this file from past assignment.") 186 | -------------------------------------------------------------------------------- /project/math_interface.py: -------------------------------------------------------------------------------- 1 | import graph_builder 2 | import networkx as nx 3 | import plotly.graph_objects as go 4 | import streamlit as st 5 | from interface.streamlit_utils import render_function 6 | 7 | import minitorch 8 | from minitorch import MathTest, MathTestVariable 9 | 10 | MyModule = None 11 | minitorch 12 | 13 | 14 | def render_math_sandbox(use_scalar=False, use_tensor=False): 15 | st.write("## Sandbox for Math Functions") 16 | st.write("Visualization of the mathematical tests run on the underlying code.") 17 | 18 | if use_scalar: 19 | one, two, red = MathTestVariable._comp_testing() 20 | else: 21 | one, two, red = MathTest._comp_testing() 22 | f_type = st.selectbox("Function Type", ["One Arg", "Two Arg", "Reduce"]) 23 | select = {"One Arg": one, "Two Arg": two, "Reduce": red} 24 | 25 | fn = st.selectbox("Function", select[f_type], format_func=lambda a: a[0]) 26 | name, _, scalar = fn 27 | if f_type == "One Arg": 28 | st.write("### " + name) 29 | render_function(scalar) 30 | st.write("Function f(x)") 31 | xs = [((x / 1.0) - 50.0 + 1e-5) for x in range(1, 100)] 32 | if use_scalar: 33 | if use_tensor: 34 | ys = [scalar(minitorch.tensor([p]))[0] for p in xs] 35 | else: 36 | ys = [scalar(minitorch.Scalar(p)).data for p in xs] 37 | else: 38 | ys = [scalar(p) for p in xs] 39 | scatter = go.Scatter(mode="lines", x=xs, y=ys) 40 | fig = go.Figure(scatter) 41 | st.write(fig) 42 | 43 | if use_scalar: 44 | st.write("Derivative f'(x)") 45 | if use_tensor: 46 | x_var = [minitorch.tensor(x, requires_grad=True) for x in xs] 47 | else: 48 | x_var = [minitorch.Scalar(x) for x in xs] 49 | for x in x_var: 50 | out = scalar(x) 51 | if use_tensor: 52 | out.backward(minitorch.tensor([1.0])) 53 | else: 54 | out.backward() 55 | if use_tensor: 56 | scatter = go.Scatter(mode="lines", x=xs, y=[x.grad[0] for x in x_var]) 57 | else: 58 | scatter = go.Scatter( 59 | mode="lines", x=xs, y=[x.derivative for x in x_var] 60 | ) 61 | fig = go.Figure(scatter) 62 | st.write(fig) 63 | G = graph_builder.GraphBuilder().run(out) 64 | G.graph["graph"] = {"rankdir": "LR"} 65 | st.graphviz_chart(nx.nx_pydot.to_pydot(G).to_string()) 66 | 67 | if f_type == "Two Arg": 68 | 69 | st.write("### " + name) 70 | render_function(scalar) 71 | st.write("Function f(x, y)") 72 | xs = [((x / 1.0) - 50.0 + 1e-5) for x in range(1, 100)] 73 | ys = [((x / 1.0) - 50.0 + 1e-5) for x in range(1, 100)] 74 | if use_scalar: 75 | if use_tensor: 76 | zs = [ 77 | [ 78 | scalar(minitorch.tensor([x]), minitorch.tensor([y]))[0] 79 | for x in xs 80 | ] 81 | for y in ys 82 | ] 83 | else: 84 | zs = [ 85 | [scalar(minitorch.Scalar(x), minitorch.Scalar(y)).data for x in xs] 86 | for y in ys 87 | ] 88 | else: 89 | zs = [[scalar(x, y) for x in xs] for y in ys] 90 | 91 | scatter = go.Surface(x=xs, y=ys, z=zs) 92 | 93 | fig = go.Figure(scatter) 94 | st.write(fig) 95 | if use_scalar: 96 | a, b = [], [] 97 | for x in xs: 98 | oa, ob = [], [] 99 | 100 | if use_tensor: 101 | for y in ys: 102 | x1 = minitorch.tensor([x]) 103 | y1 = minitorch.tensor([y]) 104 | out = scalar(x1, y1) 105 | out.backward(minitorch.tensor([1])) 106 | oa.append((x, y, x1.derivative[0])) 107 | ob.append((x, y, y1.derivative[0])) 108 | else: 109 | for y in ys: 110 | x1 = minitorch.Scalar(x) 111 | y1 = minitorch.Scalar(y) 112 | out = scalar(x1, y1) 113 | out.backward() 114 | oa.append((x, y, x1.derivative)) 115 | ob.append((x, y, y1.derivative)) 116 | a.append(oa) 117 | b.append(ob) 118 | st.write("Derivative f'_x(x, y)") 119 | 120 | scatter = go.Surface( 121 | x=[[c[0] for c in a2] for a2 in a], 122 | y=[[c[1] for c in a2] for a2 in a], 123 | z=[[c[2] for c in a2] for a2 in a], 124 | ) 125 | fig = go.Figure(scatter) 126 | st.write(fig) 127 | st.write("Derivative f'_y(x, y)") 128 | scatter = go.Surface( 129 | x=[[c[0] for c in a2] for a2 in b], 130 | y=[[c[1] for c in a2] for a2 in b], 131 | z=[[c[2] for c in a2] for a2 in b], 132 | ) 133 | fig = go.Figure(scatter) 134 | st.write(fig) 135 | if f_type == "Reduce": 136 | st.write("### " + name) 137 | render_function(scalar) 138 | xs = [((x / 1.0) - 50.0 + 1e-5) for x in range(1, 100)] 139 | ys = [((x / 1.0) - 50.0 + 1e-5) for x in range(1, 100)] 140 | 141 | if use_tensor: 142 | scatter = go.Surface( 143 | x=xs, 144 | y=ys, 145 | z=[[scalar(minitorch.tensor([x, y]))[0] for x in xs] for y in ys], 146 | ) 147 | else: 148 | scatter = go.Surface( 149 | x=xs, y=ys, z=[[scalar([x, y]) for x in xs] for y in ys] 150 | ) 151 | fig = go.Figure(scatter) 152 | st.write(fig) 153 | -------------------------------------------------------------------------------- /tests/test_operators.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Tuple 2 | 3 | import pytest 4 | from hypothesis import given 5 | from hypothesis.strategies import lists 6 | 7 | from minitorch import MathTest 8 | from minitorch.operators import ( 9 | add, 10 | addLists, 11 | eq, 12 | id, 13 | inv, 14 | inv_back, 15 | log_back, 16 | lt, 17 | max, 18 | mul, 19 | neg, 20 | negList, 21 | prod, 22 | relu, 23 | relu_back, 24 | sigmoid, 25 | sum, 26 | ) 27 | 28 | from .strategies import assert_close, small_floats 29 | 30 | # ## Task 0.1 Basic hypothesis tests. 31 | 32 | 33 | @pytest.mark.task0_1 34 | @given(small_floats, small_floats) 35 | def test_same_as_python(x: float, y: float) -> None: 36 | "Check that the main operators all return the same value of the python version" 37 | assert_close(mul(x, y), x * y) 38 | assert_close(add(x, y), x + y) 39 | assert_close(neg(x), -x) 40 | assert_close(max(x, y), x if x > y else y) 41 | if abs(x) > 1e-5: 42 | assert_close(inv(x), 1.0 / x) 43 | 44 | 45 | @pytest.mark.task0_1 46 | @given(small_floats) 47 | def test_relu(a: float) -> None: 48 | if a > 0: 49 | assert relu(a) == a 50 | if a < 0: 51 | assert relu(a) == 0.0 52 | 53 | 54 | @pytest.mark.task0_1 55 | @given(small_floats, small_floats) 56 | def test_relu_back(a: float, b: float) -> None: 57 | if a > 0: 58 | assert relu_back(a, b) == b 59 | if a < 0: 60 | assert relu_back(a, b) == 0.0 61 | 62 | 63 | @pytest.mark.task0_1 64 | @given(small_floats) 65 | def test_id(a: float) -> None: 66 | assert id(a) == a 67 | 68 | 69 | @pytest.mark.task0_1 70 | @given(small_floats) 71 | def test_lt(a: float) -> None: 72 | "Check that a - 1.0 is always less than a" 73 | assert lt(a - 1.0, a) == 1.0 74 | assert lt(a, a - 1.0) == 0.0 75 | 76 | 77 | @pytest.mark.task0_1 78 | @given(small_floats) 79 | def test_max(a: float) -> None: 80 | assert max(a - 1.0, a) == a 81 | assert max(a, a - 1.0) == a 82 | assert max(a + 1.0, a) == a + 1.0 83 | assert max(a, a + 1.0) == a + 1.0 84 | 85 | 86 | @pytest.mark.task0_1 87 | @given(small_floats) 88 | def test_eq(a: float) -> None: 89 | assert eq(a, a) == 1.0 90 | assert eq(a, a - 1.0) == 0.0 91 | assert eq(a, a + 1.0) == 0.0 92 | 93 | 94 | # ## Task 0.2 - Property Testing 95 | 96 | # Implement the following property checks 97 | # that ensure that your operators obey basic 98 | # mathematical rules. 99 | 100 | 101 | @pytest.mark.task0_2 102 | @given(small_floats) 103 | def test_sigmoid(a: float) -> None: 104 | """Check properties of the sigmoid function, specifically 105 | * It is always between 0.0 and 1.0. 106 | * one minus sigmoid is the same as sigmoid of the negative 107 | * It crosses 0 at 0.5 108 | * It is strictly increasing. 109 | """ 110 | raise NotImplementedError("Need to include this file from past assignment.") 111 | 112 | 113 | @pytest.mark.task0_2 114 | @given(small_floats, small_floats, small_floats) 115 | def test_transitive(a: float, b: float, c: float) -> None: 116 | "Test the transitive property of less-than (a < b and b < c implies a < c)" 117 | raise NotImplementedError("Need to include this file from past assignment.") 118 | 119 | 120 | @pytest.mark.task0_2 121 | def test_symmetric() -> None: 122 | """ 123 | Write a test that ensures that :func:`minitorch.operators.mul` is symmetric, i.e. 124 | gives the same value regardless of the order of its input. 125 | """ 126 | raise NotImplementedError("Need to include this file from past assignment.") 127 | 128 | 129 | @pytest.mark.task0_2 130 | def test_distribute() -> None: 131 | r""" 132 | Write a test that ensures that your operators distribute, i.e. 133 | :math:`z \times (x + y) = z \times x + z \times y` 134 | """ 135 | raise NotImplementedError("Need to include this file from past assignment.") 136 | 137 | 138 | @pytest.mark.task0_2 139 | def test_other() -> None: 140 | """ 141 | Write a test that ensures some other property holds for your functions. 142 | """ 143 | raise NotImplementedError("Need to include this file from past assignment.") 144 | 145 | 146 | # ## Task 0.3 - Higher-order functions 147 | 148 | # These tests check that your higher-order functions obey basic 149 | # properties. 150 | 151 | 152 | @pytest.mark.task0_3 153 | @given(small_floats, small_floats, small_floats, small_floats) 154 | def test_zip_with(a: float, b: float, c: float, d: float) -> None: 155 | x1, x2 = addLists([a, b], [c, d]) 156 | y1, y2 = a + c, b + d 157 | assert_close(x1, y1) 158 | assert_close(x2, y2) 159 | 160 | 161 | @pytest.mark.task0_3 162 | @given( 163 | lists(small_floats, min_size=5, max_size=5), 164 | lists(small_floats, min_size=5, max_size=5), 165 | ) 166 | def test_sum_distribute(ls1: List[float], ls2: List[float]) -> None: 167 | """ 168 | Write a test that ensures that the sum of `ls1` plus the sum of `ls2` 169 | is the same as the sum of each element of `ls1` plus each element of `ls2`. 170 | """ 171 | raise NotImplementedError("Need to include this file from past assignment.") 172 | 173 | 174 | @pytest.mark.task0_3 175 | @given(lists(small_floats)) 176 | def test_sum(ls: List[float]) -> None: 177 | assert_close(sum(ls), sum(ls)) 178 | 179 | 180 | @pytest.mark.task0_3 181 | @given(small_floats, small_floats, small_floats) 182 | def test_prod(x: float, y: float, z: float) -> None: 183 | assert_close(prod([x, y, z]), x * y * z) 184 | 185 | 186 | @pytest.mark.task0_3 187 | @given(lists(small_floats)) 188 | def test_negList(ls: List[float]) -> None: 189 | check = negList(ls) 190 | for i, j in zip(ls, check): 191 | assert_close(i, -j) 192 | 193 | 194 | # ## Generic mathematical tests 195 | 196 | # For each unit this generic set of mathematical tests will run. 197 | 198 | 199 | one_arg, two_arg, _ = MathTest._tests() 200 | 201 | 202 | @given(small_floats) 203 | @pytest.mark.parametrize("fn", one_arg) 204 | def test_one_args(fn: Tuple[str, Callable[[float], float]], t1: float) -> None: 205 | name, base_fn = fn 206 | base_fn(t1) 207 | 208 | 209 | @given(small_floats, small_floats) 210 | @pytest.mark.parametrize("fn", two_arg) 211 | def test_two_args( 212 | fn: Tuple[str, Callable[[float, float], float]], t1: float, t2: float 213 | ) -> None: 214 | name, base_fn = fn 215 | base_fn(t1, t2) 216 | 217 | 218 | @given(small_floats, small_floats) 219 | def test_backs(a: float, b: float) -> None: 220 | relu_back(a, b) 221 | inv_back(a + 2.4, b) 222 | log_back(abs(a) + 4, b) 223 | -------------------------------------------------------------------------------- /minitorch/scalar_functions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | import minitorch 6 | 7 | from . import operators 8 | from .autodiff import Context 9 | 10 | if TYPE_CHECKING: 11 | from typing import Tuple 12 | 13 | from .scalar import Scalar, ScalarLike 14 | 15 | 16 | def wrap_tuple(x): # type: ignore 17 | "Turn a possible value into a tuple" 18 | if isinstance(x, tuple): 19 | return x 20 | return (x,) 21 | 22 | 23 | def unwrap_tuple(x): # type: ignore 24 | "Turn a singleton tuple into a value" 25 | if len(x) == 1: 26 | return x[0] 27 | return x 28 | 29 | 30 | class ScalarFunction: 31 | """ 32 | A wrapper for a mathematical function that processes and produces 33 | Scalar variables. 34 | 35 | This is a static class and is never instantiated. We use `class` 36 | here to group together the `forward` and `backward` code. 37 | """ 38 | 39 | @classmethod 40 | def _backward(cls, ctx: Context, d_out: float) -> Tuple[float, ...]: 41 | return wrap_tuple(cls.backward(ctx, d_out)) # type: ignore 42 | 43 | @classmethod 44 | def _forward(cls, ctx: Context, *inps: float) -> float: 45 | return cls.forward(ctx, *inps) # type: ignore 46 | 47 | @classmethod 48 | def apply(cls, *vals: "ScalarLike") -> Scalar: 49 | raw_vals = [] 50 | scalars = [] 51 | for v in vals: 52 | if isinstance(v, minitorch.scalar.Scalar): 53 | scalars.append(v) 54 | raw_vals.append(v.data) 55 | else: 56 | scalars.append(minitorch.scalar.Scalar(v)) 57 | raw_vals.append(v) 58 | 59 | # Create the context. 60 | ctx = Context(False) 61 | 62 | # Call forward with the variables. 63 | c = cls._forward(ctx, *raw_vals) 64 | assert isinstance(c, float), "Expected return type float got %s" % (type(c)) 65 | 66 | # Create a new variable from the result with a new history. 67 | back = minitorch.scalar.ScalarHistory(cls, ctx, scalars) 68 | return minitorch.scalar.Scalar(c, back) 69 | 70 | 71 | # Examples 72 | class Add(ScalarFunction): 73 | "Addition function $f(x, y) = x + y$" 74 | 75 | @staticmethod 76 | def forward(ctx: Context, a: float, b: float) -> float: 77 | return a + b 78 | 79 | @staticmethod 80 | def backward(ctx: Context, d_output: float) -> Tuple[float, ...]: 81 | return d_output, d_output 82 | 83 | 84 | class Log(ScalarFunction): 85 | "Log function $f(x) = log(x)$" 86 | 87 | @staticmethod 88 | def forward(ctx: Context, a: float) -> float: 89 | ctx.save_for_backward(a) 90 | return operators.log(a) 91 | 92 | @staticmethod 93 | def backward(ctx: Context, d_output: float) -> float: 94 | (a,) = ctx.saved_values 95 | return operators.log_back(a, d_output) 96 | 97 | 98 | # To implement. 99 | 100 | 101 | class Mul(ScalarFunction): 102 | "Multiplication function" 103 | 104 | @staticmethod 105 | def forward(ctx: Context, a: float, b: float) -> float: 106 | # TODO: Implement for Task 1.2. 107 | raise NotImplementedError("Need to implement for Task 1.2") 108 | 109 | @staticmethod 110 | def backward(ctx: Context, d_output: float) -> Tuple[float, float]: 111 | # TODO: Implement for Task 1.4. 112 | raise NotImplementedError("Need to implement for Task 1.4") 113 | 114 | 115 | class Inv(ScalarFunction): 116 | "Inverse function" 117 | 118 | @staticmethod 119 | def forward(ctx: Context, a: float) -> float: 120 | # TODO: Implement for Task 1.2. 121 | raise NotImplementedError("Need to implement for Task 1.2") 122 | 123 | @staticmethod 124 | def backward(ctx: Context, d_output: float) -> float: 125 | # TODO: Implement for Task 1.4. 126 | raise NotImplementedError("Need to implement for Task 1.4") 127 | 128 | 129 | class Neg(ScalarFunction): 130 | "Negation function" 131 | 132 | @staticmethod 133 | def forward(ctx: Context, a: float) -> float: 134 | # TODO: Implement for Task 1.2. 135 | raise NotImplementedError("Need to implement for Task 1.2") 136 | 137 | @staticmethod 138 | def backward(ctx: Context, d_output: float) -> float: 139 | # TODO: Implement for Task 1.4. 140 | raise NotImplementedError("Need to implement for Task 1.4") 141 | 142 | 143 | class Sigmoid(ScalarFunction): 144 | "Sigmoid function" 145 | 146 | @staticmethod 147 | def forward(ctx: Context, a: float) -> float: 148 | # TODO: Implement for Task 1.2. 149 | raise NotImplementedError("Need to implement for Task 1.2") 150 | 151 | @staticmethod 152 | def backward(ctx: Context, d_output: float) -> float: 153 | # TODO: Implement for Task 1.4. 154 | raise NotImplementedError("Need to implement for Task 1.4") 155 | 156 | 157 | class ReLU(ScalarFunction): 158 | "ReLU function" 159 | 160 | @staticmethod 161 | def forward(ctx: Context, a: float) -> float: 162 | # TODO: Implement for Task 1.2. 163 | raise NotImplementedError("Need to implement for Task 1.2") 164 | 165 | @staticmethod 166 | def backward(ctx: Context, d_output: float) -> float: 167 | # TODO: Implement for Task 1.4. 168 | raise NotImplementedError("Need to implement for Task 1.4") 169 | 170 | 171 | class Exp(ScalarFunction): 172 | "Exp function" 173 | 174 | @staticmethod 175 | def forward(ctx: Context, a: float) -> float: 176 | # TODO: Implement for Task 1.2. 177 | raise NotImplementedError("Need to implement for Task 1.2") 178 | 179 | @staticmethod 180 | def backward(ctx: Context, d_output: float) -> float: 181 | # TODO: Implement for Task 1.4. 182 | raise NotImplementedError("Need to implement for Task 1.4") 183 | 184 | 185 | class LT(ScalarFunction): 186 | "Less-than function $f(x) =$ 1.0 if x is less than y else 0.0" 187 | 188 | @staticmethod 189 | def forward(ctx: Context, a: float, b: float) -> float: 190 | # TODO: Implement for Task 1.2. 191 | raise NotImplementedError("Need to implement for Task 1.2") 192 | 193 | @staticmethod 194 | def backward(ctx: Context, d_output: float) -> Tuple[float, float]: 195 | # TODO: Implement for Task 1.4. 196 | raise NotImplementedError("Need to implement for Task 1.4") 197 | 198 | 199 | class EQ(ScalarFunction): 200 | "Equal function $f(x) =$ 1.0 if x is equal to y else 0.0" 201 | 202 | @staticmethod 203 | def forward(ctx: Context, a: float, b: float) -> float: 204 | # TODO: Implement for Task 1.2. 205 | raise NotImplementedError("Need to implement for Task 1.2") 206 | 207 | @staticmethod 208 | def backward(ctx: Context, d_output: float) -> Tuple[float, float]: 209 | # TODO: Implement for Task 1.4. 210 | raise NotImplementedError("Need to implement for Task 1.4") 211 | -------------------------------------------------------------------------------- /minitorch/scalar.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import Any, Iterable, Optional, Sequence, Tuple, Type, Union 5 | 6 | import numpy as np 7 | 8 | from .autodiff import Context, Variable, backpropagate, central_difference 9 | from .scalar_functions import ( 10 | EQ, 11 | LT, 12 | Add, 13 | Exp, 14 | Inv, 15 | Log, 16 | Mul, 17 | Neg, 18 | ReLU, 19 | ScalarFunction, 20 | Sigmoid, 21 | ) 22 | 23 | ScalarLike = Union[float, int, "Scalar"] 24 | 25 | 26 | @dataclass 27 | class ScalarHistory: 28 | """ 29 | `ScalarHistory` stores the history of `Function` operations that was 30 | used to construct the current Variable. 31 | 32 | Attributes: 33 | last_fn : The last Function that was called. 34 | ctx : The context for that Function. 35 | inputs : The inputs that were given when `last_fn.forward` was called. 36 | 37 | """ 38 | 39 | last_fn: Optional[Type[ScalarFunction]] = None 40 | ctx: Optional[Context] = None 41 | inputs: Sequence[Scalar] = () 42 | 43 | 44 | # ## Task 1.2 and 1.4 45 | # Scalar Forward and Backward 46 | 47 | _var_count = 0 48 | 49 | 50 | class Scalar: 51 | """ 52 | A reimplementation of scalar values for autodifferentiation 53 | tracking. Scalar Variables behave as close as possible to standard 54 | Python numbers while also tracking the operations that led to the 55 | number's creation. They can only be manipulated by 56 | `ScalarFunction`. 57 | """ 58 | 59 | history: Optional[ScalarHistory] 60 | derivative: Optional[float] 61 | data: float 62 | unique_id: int 63 | name: str 64 | 65 | def __init__( 66 | self, 67 | v: float, 68 | back: ScalarHistory = ScalarHistory(), 69 | name: Optional[str] = None, 70 | ): 71 | global _var_count 72 | _var_count += 1 73 | self.unique_id = _var_count 74 | self.data = float(v) 75 | self.history = back 76 | self.derivative = None 77 | if name is not None: 78 | self.name = name 79 | else: 80 | self.name = str(self.unique_id) 81 | 82 | def __repr__(self) -> str: 83 | return "Scalar(%f)" % self.data 84 | 85 | def __mul__(self, b: ScalarLike) -> Scalar: 86 | return Mul.apply(self, b) 87 | 88 | def __truediv__(self, b: ScalarLike) -> Scalar: 89 | return Mul.apply(self, Inv.apply(b)) 90 | 91 | def __rtruediv__(self, b: ScalarLike) -> Scalar: 92 | return Mul.apply(b, Inv.apply(self)) 93 | 94 | def __add__(self, b: ScalarLike) -> Scalar: 95 | # TODO: Implement for Task 1.2. 96 | raise NotImplementedError("Need to implement for Task 1.2") 97 | 98 | def __bool__(self) -> bool: 99 | return bool(self.data) 100 | 101 | def __lt__(self, b: ScalarLike) -> Scalar: 102 | # TODO: Implement for Task 1.2. 103 | raise NotImplementedError("Need to implement for Task 1.2") 104 | 105 | def __gt__(self, b: ScalarLike) -> Scalar: 106 | # TODO: Implement for Task 1.2. 107 | raise NotImplementedError("Need to implement for Task 1.2") 108 | 109 | def __eq__(self, b: ScalarLike) -> Scalar: # type: ignore[override] 110 | # TODO: Implement for Task 1.2. 111 | raise NotImplementedError("Need to implement for Task 1.2") 112 | 113 | def __sub__(self, b: ScalarLike) -> Scalar: 114 | # TODO: Implement for Task 1.2. 115 | raise NotImplementedError("Need to implement for Task 1.2") 116 | 117 | def __neg__(self) -> Scalar: 118 | # TODO: Implement for Task 1.2. 119 | raise NotImplementedError("Need to implement for Task 1.2") 120 | 121 | def __radd__(self, b: ScalarLike) -> Scalar: 122 | return self + b 123 | 124 | def __rmul__(self, b: ScalarLike) -> Scalar: 125 | return self * b 126 | 127 | def log(self) -> Scalar: 128 | # TODO: Implement for Task 1.2. 129 | raise NotImplementedError("Need to implement for Task 1.2") 130 | 131 | def exp(self) -> Scalar: 132 | # TODO: Implement for Task 1.2. 133 | raise NotImplementedError("Need to implement for Task 1.2") 134 | 135 | def sigmoid(self) -> Scalar: 136 | # TODO: Implement for Task 1.2. 137 | raise NotImplementedError("Need to implement for Task 1.2") 138 | 139 | def relu(self) -> Scalar: 140 | # TODO: Implement for Task 1.2. 141 | raise NotImplementedError("Need to implement for Task 1.2") 142 | 143 | # Variable elements for backprop 144 | 145 | def accumulate_derivative(self, x: Any) -> None: 146 | """ 147 | Add `val` to the the derivative accumulated on this variable. 148 | Should only be called during autodifferentiation on leaf variables. 149 | 150 | Args: 151 | x: value to be accumulated 152 | """ 153 | assert self.is_leaf(), "Only leaf variables can have derivatives." 154 | if self.derivative is None: 155 | self.derivative = 0.0 156 | self.derivative += x 157 | 158 | def is_leaf(self) -> bool: 159 | "True if this variable created by the user (no `last_fn`)" 160 | return self.history is not None and self.history.last_fn is None 161 | 162 | def is_constant(self) -> bool: 163 | return self.history is None 164 | 165 | @property 166 | def parents(self) -> Iterable[Variable]: 167 | assert self.history is not None 168 | return self.history.inputs 169 | 170 | def chain_rule(self, d_output: Any) -> Iterable[Tuple[Variable, Any]]: 171 | h = self.history 172 | assert h is not None 173 | assert h.last_fn is not None 174 | assert h.ctx is not None 175 | 176 | # TODO: Implement for Task 1.3. 177 | raise NotImplementedError("Need to implement for Task 1.3") 178 | 179 | def backward(self, d_output: Optional[float] = None) -> None: 180 | """ 181 | Calls autodiff to fill in the derivatives for the history of this object. 182 | 183 | Args: 184 | d_output (number, opt): starting derivative to backpropagate through the model 185 | (typically left out, and assumed to be 1.0). 186 | """ 187 | if d_output is None: 188 | d_output = 1.0 189 | backpropagate(self, d_output) 190 | 191 | 192 | def derivative_check(f: Any, *scalars: Scalar) -> None: 193 | """ 194 | Checks that autodiff works on a python function. 195 | Asserts False if derivative is incorrect. 196 | 197 | Parameters: 198 | f : function from n-scalars to 1-scalar. 199 | *scalars : n input scalar values. 200 | """ 201 | out = f(*scalars) 202 | out.backward() 203 | 204 | err_msg = """ 205 | Derivative check at arguments f(%s) and received derivative f'=%f for argument %d, 206 | but was expecting derivative f'=%f from central difference.""" 207 | for i, x in enumerate(scalars): 208 | check = central_difference(f, *scalars, arg=i) 209 | print(str([x.data for x in scalars]), x.derivative, i, check) 210 | assert x.derivative is not None 211 | np.testing.assert_allclose( 212 | x.derivative, 213 | check.data, 214 | 1e-2, 215 | 1e-2, 216 | err_msg=err_msg 217 | % (str([x.data for x in scalars]), x.derivative, i, check.data), 218 | ) 219 | --------------------------------------------------------------------------------