├── .github └── workflows │ ├── ci.yml │ ├── docs.yml │ ├── lint.yml │ ├── publish.yml │ └── stale.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Makefile ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── docker-compose.yml ├── docs ├── api │ ├── application.md │ ├── component-instance.md │ ├── component.md │ ├── mtable.md │ ├── props-and-state.md │ └── state-migrator.md ├── examples │ ├── application.md │ ├── design-patterns.md │ ├── hello-world.md │ └── nl-query.md ├── getting-started │ ├── concepts.md │ └── installation.md ├── images │ └── logo.png ├── index.md └── tools │ ├── cli.md │ ├── dashboard.md │ ├── migrator.md │ └── vis.md ├── ideas └── roadmap.md ├── mkdocs.yml ├── motion ├── __init__.py ├── cli.py ├── component.py ├── copy_utils.py ├── dashboard.py ├── dashboard_utils.py ├── df.py ├── dicts.py ├── discard_policy.py ├── execute.py ├── instance.py ├── migrate.py ├── mtable.py ├── res │ └── eff_short_wordlist_1.txt ├── route.py ├── server │ ├── __init__.py │ ├── application.py │ └── update_task.py ├── static │ ├── asset-manifest.json │ ├── index.html │ ├── logo.png │ ├── manifest.json │ ├── robots.txt │ └── static │ │ ├── css │ │ ├── main.4e812611.css │ │ └── main.4e812611.css.map │ │ └── js │ │ ├── 808.653ad45b.chunk.js │ │ ├── 808.653ad45b.chunk.js.map │ │ ├── main.532352eb.js │ │ ├── main.532352eb.js.LICENSE.txt │ │ └── main.532352eb.js.map └── utils.py ├── poetry.lock ├── pyproject.toml ├── tests ├── .motionrc.yml ├── __init__.py ├── component │ ├── __init__.py │ ├── test_bad_components.py │ ├── test_context_manager.py │ ├── test_counter.py │ ├── test_disabled_instance.py │ ├── test_multiple_routes.py │ ├── test_params.py │ └── test_process.py ├── conftest.py ├── parallel │ ├── __init__.py │ ├── test_async.py │ ├── test_broken_fit.py │ ├── test_generator.py │ ├── test_many_keys.py │ └── test_two_instance.py ├── server │ ├── __init__.py │ ├── test_dashboard.py │ └── test_simple_server.py ├── state │ ├── __init__.py │ ├── test_dataframe.py │ ├── test_db_conn.py │ ├── test_flush_fit.py │ ├── test_instance_cli.py │ ├── test_model.py │ ├── test_mtable_integration.py │ ├── test_mtable_unit.py │ └── test_write.py ├── test_dev_box.py ├── test_expire_policies.py ├── test_fastapi.py ├── test_migrate.py └── test_simple_pipeline.py └── ui ├── .gitignore ├── README.md ├── package.json ├── public ├── index.html ├── logo.png ├── manifest.json └── robots.txt ├── src ├── App.css ├── App.js ├── App.test.js ├── components │ ├── ComponentInfoCard.js │ ├── DetailComponent.js │ ├── DynamicTable.js │ ├── MainContent.js │ ├── MotionHeader.js │ └── Sidebar.js ├── customTheme.js ├── index.css ├── index.js ├── logo.svg ├── reportWebVitals.js └── setupTests.js └── tailwind.config.js /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: motion 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ["3.9", "3.10", "3.11"] 13 | services: 14 | redis: 15 | image: redis 16 | ports: 17 | - 6381:6379 18 | victoriametrics: 19 | image: victoriametrics/victoria-metrics 20 | ports: 21 | - 8428:8428 # Default port for VictoriaMetrics 22 | 23 | steps: 24 | - name: Checkout code 25 | uses: actions/checkout@v2 26 | 27 | - name: Start Redis 28 | run: | 29 | retries=3 30 | until redis-cli -p 6381 ping || [[ $retries == 0 ]]; do 31 | ((retries--)) 32 | sleep 1 33 | done 34 | 35 | - name: Check VictoriaMetrics readiness 36 | run: | 37 | retries=5 38 | until curl --silent --fail http://localhost:8428/health || [[ $retries == 0 ]]; do 39 | echo "Waiting for VictoriaMetrics to be ready..." 40 | ((retries--)) 41 | sleep 2 42 | done 43 | 44 | - name: Set up Python 45 | uses: actions/setup-python@v2 46 | with: 47 | python-version: ${{ matrix.python-version }} 48 | 49 | - name: Install Poetry 50 | uses: snok/install-poetry@v1 51 | 52 | - name: Install dependencies 53 | run: make install 54 | 55 | - name: Run pytest 56 | run: make tests 57 | 58 | - name: Run mypy 59 | run: make mypy 60 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | paths: 7 | - 'docs/**' 8 | - 'mkdocs.yml' 9 | permissions: 10 | contents: write 11 | jobs: 12 | deploy: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | - uses: actions/setup-python@v4 17 | with: 18 | python-version: 3.x 19 | - uses: actions/cache@v2 20 | with: 21 | key: ${{ github.ref }} 22 | path: .cache 23 | - name: Install Poetry 24 | uses: snok/install-poetry@v1 25 | - name: Install dependencies 26 | run: poetry install 27 | - name: Build docs 28 | run: poetry run mkdocs build 29 | - name: linkcheck 30 | run: poetry run linkchecker site/index.html 31 | - name: Deploy docs 32 | run: poetry run mkdocs gh-deploy --force 33 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: chartboost/ruff-action@v1 11 | with: 12 | src: "./motion" 13 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - "motion/**" 9 | - "poetry.lock" 10 | - "pyproject.toml" 11 | - ".github/workflows/publish.yml" 12 | 13 | jobs: 14 | publish-to-pypi: 15 | runs-on: ubuntu-latest 16 | if: ${{github.event.head_commit.author.name != 'github-actions[bot]' }} 17 | steps: 18 | - name: Print author 19 | run: | 20 | echo "Commit author name: ${{ github.event.head_commit.author.name }}" 21 | echo "Commit author email: ${{ github.event.head_commit.author.email }}" 22 | - name: Checkout code 23 | uses: actions/checkout@v2 24 | with: 25 | persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token 26 | fetch-depth: 0 # otherwise, you will failed to push refs to dest repo 27 | - name: Configure Git 28 | run: | 29 | git config --global user.name "github-actions[bot]" 30 | git config --global user.email "github-actions[bot]@users.noreply.github.com" 31 | - name: Set up Python 32 | uses: actions/setup-python@v2 33 | with: 34 | python-version: 3.9 35 | - name: Set up Node.js 36 | uses: actions/setup-node@v2 37 | with: 38 | node-version: "21" 39 | - name: Install Poetry 40 | uses: snok/install-poetry@v1 41 | - name: Bump version 42 | id: bump_version 43 | run: | 44 | poetry version patch 45 | - name: Build and publish 46 | run: | 47 | poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} 48 | make build 49 | poetry build 50 | poetry publish 51 | git add pyproject.toml 52 | git commit -m "Bump up version" 53 | - name: Push changes 54 | uses: ad-m/github-push-action@master 55 | with: 56 | github_token: ${{ secrets.BRANCH_PROTECTION_WORKAROUND }} 57 | branch: main 58 | 59 | - name: Set release number 60 | run: echo "RELEASE_NUMBER=$(poetry version --no-ansi | awk -F' ' '{print $2}')" >> $GITHUB_ENV 61 | - name: Create release 62 | id: create_release 63 | uses: actions/create-release@v1 64 | env: 65 | GITHUB_TOKEN: ${{ secrets.AUTORELEASE_TOKEN }} 66 | RELEASE_NUMBER: ${{ env.RELEASE_NUMBER }} 67 | with: 68 | tag_name: v${{ env.RELEASE_NUMBER }} 69 | release_name: Release ${{ env.RELEASE_NUMBER }} 70 | body: | 71 | An autorelease from the latest version of main. 72 | draft: false 73 | prerelease: true 74 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Stale Issues and Pull Requests 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' # Runs every day at midnight UTC 6 | issues: 7 | types: [opened, reopened, edited] 8 | pull_request: 9 | types: [opened, reopened, edited] 10 | 11 | jobs: 12 | stale: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Stale action 16 | uses: actions/stale@v4.1.1 17 | with: 18 | stale-issue-message: 'This issue has been closed because it has been inactive for 7 days. Please feel free to reopen if you are still experiencing this problem.' 19 | stale-pr-message: 'This pull request has been closed because it has been inactive for 7 days. Please feel free to reopen if you are still interested in contributing.' 20 | days-before-stale: 60 21 | days-before-close: 7 22 | stale-issue-label: 'no-issue-activity' 23 | stale-pr-label: 'no-pr-activity' 24 | exempt-issue-labels: 'keep open' 25 | exempt-pr-labels: 'keep open' 26 | exempt-milestone: 'upcoming release' 27 | exempt-assignees: 'octocat' 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *__pycache__* 2 | *.ipynb_checkpoints* 3 | *.pyc 4 | *.ipynb 5 | *node_modules* 6 | *.next* 7 | *__pycache__* 8 | *faiss_index/* 9 | *scratch.py 10 | *datastores* 11 | *.egg-info 12 | *.DS_Store 13 | applications* 14 | *wandb* 15 | dist* 16 | site* 17 | *package-lock.json 18 | projects 19 | unnecessary.py 20 | motionstate* 21 | *.whl 22 | *.so 23 | target* 24 | .motionenv* 25 | *motionrc* 26 | example_dashboard.py -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: false 3 | 4 | files: "^(motion|ui)/" 5 | exclude: '\__init__.py$' 6 | 7 | repos: 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v4.5.0 10 | hooks: 11 | - id: trailing-whitespace 12 | - id: end-of-file-fixer 13 | exclude: ^.*\.egg-info/ 14 | - id: check-merge-conflict 15 | - id: check-case-conflict 16 | - id: pretty-format-json 17 | args: [--autofix, --no-ensure-ascii, --no-sort-keys] 18 | - id: check-ast 19 | - id: debug-statements 20 | - id: check-docstring-first 21 | 22 | - repo: https://github.com/hadialqattan/pycln 23 | rev: v2.4.0 24 | hooks: 25 | - id: pycln 26 | args: [--all] 27 | 28 | - repo: https://github.com/psf/black 29 | rev: 24.1.1 30 | hooks: 31 | - id: black 32 | 33 | - repo: https://github.com/pycqa/isort 34 | rev: 5.13.2 35 | hooks: 36 | - id: isort 37 | name: "isort (python)" 38 | types: [python] 39 | args: [--profile, black] 40 | 41 | - repo: https://github.com/charliermarsh/ruff-pre-commit 42 | # Ruff version. 43 | rev: "v0.2.1" 44 | hooks: 45 | - id: ruff 46 | 47 | - repo: https://github.com/pre-commit/pre-commit 48 | rev: v3.6.0 49 | hooks: 50 | - id: validate_manifest 51 | 52 | - repo: https://github.com/pre-commit/mirrors-prettier 53 | rev: "v4.0.0-alpha.8" # Prettier version 54 | hooks: 55 | - id: prettier 56 | files: "^ui/" 57 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: tests lint install mypy update docs build 2 | 3 | tests: 4 | poetry run pytest 5 | 6 | lint: 7 | poetry run ruff motion/* --fix 8 | 9 | install: 10 | pip install poetry 11 | poetry install --all-extras 12 | 13 | mypy: 14 | poetry run mypy 15 | 16 | update: 17 | poetry update 18 | 19 | docs: 20 | poetry run mkdocs serve 21 | 22 | build: 23 | rm -rf motion/static 24 | cd ui && npm install && npm run build 25 | cd .. 26 | cp -r ui/build motion/static 27 | poetry install --all-extras -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | [Provide a brief description of the changes in this pull request.] 4 | 5 | ## Related Issues 6 | 7 | [Link any relevant issues that are addressed in this pull request.] 8 | 9 | ## Checklist 10 | 11 | Please make sure that you have completed the following items before submitting this pull request: 12 | 13 | - [ ] Code is up-to-date with the main branch 14 | - [ ] Code has been tested locally and all tests pass 15 | - [ ] Code changes have been documented (if necessary) 16 | - [ ] Code changes adhere to the project's coding standards 17 | - [ ] Any necessary new dependencies have been added to the project's `pyproject.toml` file 18 | 19 | ## Screenshots 20 | 21 | [Add any relevant screenshots or images to help illustrate the changes in this pull request.] 22 | 23 | ## Additional Notes 24 | 25 | [Add any additional information or context that you feel is important for reviewers to know about this pull request.] 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Motion 2 | 3 | [![motion](https://github.com/dm4ml/motion/workflows/motion/badge.svg)](https://github.com/dm4ml/motion/actions?query=workflow:"motion") 4 | [![lint (via ruff)](https://github.com/dm4ml/motion/workflows/lint/badge.svg)](https://github.com/dm4ml/motion/actions?query=workflow:"lint") 5 | [![docs](https://github.com/dm4ml/motion/workflows/docs/badge.svg)](https://github.com/dm4ml/motion/actions?query=workflow:"docs") 6 | [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) 7 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 8 | [![GitHub tag](https://img.shields.io/github/tag/dm4ml/motion?include_prereleases=&sort=semver&color=blue)](https://github.com/dm4ml/motion/releases/) 9 | [![PyPI version](https://badge.fury.io/py/motion-python.svg?branch=main&kill_cache=1)](https://badge.fury.io/py/motion-python) 10 | 11 | Motion is a system for defining and incrementally maintaining **self-updating prompts** in Python. 12 | 13 | ## Why Self-Updating Prompts? 14 | 15 | LLM accuracy often significantly improves with more context. Consider an e-commerce focused LLM pipeline that recommends products to users. The recommendations might improve if the prompt considers the user's past purchases and browsing history. A **self-updating prompt** could change over time by including LLM-generated insights from user interactions. For a concrete example of this, consider the following sequence of user interactions, where 1, 3, and 4 show prompts for event styling queries and 2 corresponds to user feedback: 16 | 17 | 1. "What apparel items should I buy for `SIGMOD in Chile`?" 18 | 2. _User `disliked "purple blazer"` and `liked "wide-leg jeans."`_ 19 | 3. "`I work in tech. I dress casually.` What apparel items should I buy for `hiking in the Bay Area`?" 20 | 4. "`I work in tech and have an active lifestyle. I dress casually.` What apparel items should I buy for `coffee with a friend`?" 21 | 22 | In the above sequence, `phrases in backticks` are dynamically generated based on previous user-generated activity. The new context can improve the quality of responses. 23 | 24 | ### Why is it Hard to Use Self-Updating Prompts? 25 | 26 | Consider the e-commerce example above. The prompt might grow to be very long---so long that there's a bunch of redundant or event useless information in the prompt. So, we might want to summarize the user's past purchases and browsing history into a single prompt. However, summarizing the user's past purchases and browsing history every time we log a new purchase or browsing event, or whenever the user requests a new recommendation, **can take too long** (e.g., 30+ seconds) and thus prohibitively increase end-to-end latency for getting a recommendation. 27 | 28 | ## What is Motion? 29 | 30 | Motion allows LLM pipeline developers to define and incrementally maintain self-updating prompts in Python. With Motion, developers define **components** that represent prompt sub-parts, and **flows** that represent how to assemble sub-parts into a prompt for an LLM in real-time and how to self-updatingly update sub-parts in the background based on new information. 31 | 32 | Motion's execution engine serves cached prompt sub-parts for minimal real-time latency and handles concurrency and sub-part consistency when running flows that update sub-parts. All prompt sub-parts are backed by a key-value store. You can run Motion components anywhere and in any number of Python processes (e.g., in a notebook, in a serverless function, in a web server) at the same time for maximal availability. 33 | 34 | As LLM pipeline developers, we want a few things when building and using self-updating prompts: 35 | 36 | - **Flexibility**: We want to be able to define our sub-parts of prompts (e.g., summaries). We also want to be able to define our own logic for how to turn sub-parts into string prompts and self-updatingly update sub-parts. 37 | - **Availability**: We want there to always be some version of prompt sub-parts available, even if they are a little stale. This way we can minimize end-to-end latency. 38 | - **Freshness**: Prompts should incorporate as much of the latest information as possible. In the case where information arrives faster than we can process it, it may be desirable to ignore older information. 39 | 40 | ## An Example Motion Component 41 | 42 | It's hard to understand Motion without an example. In Motion, you define components, which are stateful objects that can be updated incrementally with new data. A component has an `init_state` method that initializes the state of the component, and any number of **flows**, where each flow consists of a `serve` operation (state read-only) and an `update` operation (can read and write state). These operations are arbitrary user-defined Python functions. 43 | 44 | Here's an example of a component that recommends apparel to buy for an event, personalized to each user: 45 | 46 | ```python 47 | from motion import Component 48 | 49 | ECommercePrompt = Component("E-Commerce") 50 | 51 | @ECommercePrompt.init_state 52 | def setup(): 53 | return {"query_summary": "No queries yet.", "preference_summary": "No preference information yet."} 54 | 55 | @ECommercePrompt.serve("styling_query") 56 | def generate_recs(state, props): 57 | # Props = properties to this specific flow's execution 58 | # First retrieve products from the catalog 59 | catalog_products = retrieve(props['event']) 60 | prompt = f"Consider the following lifestyle and preference information about me: {state['query_summary']}, {state['preference_summary']}. Suggest 3-5 apparel items for me to buy for {props['event']}, using the catalog: {catalog_products}." 61 | return llm(prompt) 62 | 63 | @ECommercePrompt.update("styling_query") 64 | def query_summary(state, props): 65 | # props.serve_result contains the result from the serve op 66 | prompt = f"You recommended a user buy {props.serve_result} for {props['event']}. The information we currently have about them is: {state['query_summary']}. Based on their query history, give a new 3-sentence summary about their lifestyle." 67 | query_summary = llm(prompt) 68 | # Update state 69 | return {"query_summary": query_summary} 70 | ``` 71 | 72 | In the above example, the `serve` operation recommends items to buy based on an event styling query, and the `update` operation updates the context to be used in future recommendations (i.e., "styling_query" serve operations). `serve` operations execute first and cannot modify state, while `update` operations can modify state and execute after `serve` operations in the background. 73 | 74 | You can run a flow by calling `run` or `arun` (async version of `run`) on the component: 75 | 76 | ```python 77 | # Initialize component instance 78 | instance = ECommercePrompt(user_id) # Some user_id 79 | 80 | # Run the "styling_query" flow. Will return the result of the "styling_query" serve 81 | # operation, and queue the "styling_query" update operation to run in the background. 82 | rec = await instance.arun("styling_query", props={"event": "sightseeing in Santiago, Chile"}) 83 | ``` 84 | 85 | After `rec` is returned, the `update` operation will run in the background and update the state of the component (for as long as the Python process is running). The state of the component instance is always committed to the key-value store after a flow is fully run, and is loaded from the key-value store when the component instance is initialized again. 86 | 87 | Multiple clients can run flows on the same component instance, and the state of the component will be updated accordingly. Serve operations are run in parallel, while update operations are run sequentially in the order they are called. Motion maintains consistency by locking the state of the component while an update operation is running. Serve operations can run with old state while an update operation is running, so they are not blocked. 88 | 89 | ## Should I use Motion? 90 | 91 | Motion is especially useful for LLM pipelines 92 | 93 | - Need to update prompts based on new data (e.g., maintain a dynamic summary in the prompt) 94 | - Want a Pythonic interface to build a distributed system of LLM application components 95 | 96 | Motion is built for developers who know how to code in Python and want to be able to control operations in their ML applications. For low-code and domain-specific development patterns (e.g., enhancing videos), you may want to check out other tools. 97 | 98 | ## Where did Motion come from? 99 | 100 | Motion is developed and maintained by researchers at the [UC Berkeley EPIC Lab](https://epic.berkeley.edu) who specialize in data management for ML pipelines. 101 | 102 | ## Getting Started 103 | 104 | Check out the [docs](https://dm4ml.github.io/motion/) for more information. 105 | 106 | Motion is currently in alpha. We are actively working on improving the documentation and adding more features. If you are interested in using Motion and would like dedicated support from one of our team members, please reach out to us at [shreyashankar@berkeley.edu](mailto:shreyashankar@berkeley.edu). 107 | 108 | ## Testing and Development 109 | 110 | You can run `make install` to install an editable source of Motion. We use `poetry` to manage dependencies. 111 | 112 | To run tests, we use `pytest` and a local Redis cache. You should run Redis on port 6381 before you run `make tests`. To run Redis with Docker, either run the `docker-compose.yml` file in this repo (i.e., `docker-compose up`) or run the following command in your terminal: 113 | 114 | ```bash 115 | docker run -p 6381:6379 --name motion-backend-testing redis/redis-stack-server:latest 116 | ``` 117 | 118 | Then when you run `make tests`, your tests should pass. 119 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | redis_test: 4 | image: redis/redis-stack-server:latest 5 | container_name: motion-backend-testing 6 | ports: 7 | - 6381:6379 8 | redis: 9 | image: redis/redis-stack-server:latest 10 | container_name: motion-backend 11 | ports: 12 | - 6379:6379 13 | -------------------------------------------------------------------------------- /docs/api/application.md: -------------------------------------------------------------------------------- 1 | ::: motion.Application 2 | handler: python 3 | options: 4 | members: 5 | - __init__ 6 | - create_read_state_endpoint 7 | - create_write_state_endpoint 8 | - create_component_endpoint 9 | - get_app 10 | - get_credentials 11 | show_root_full_path: false 12 | show_root_toc_entry: false 13 | show_root_heading: true 14 | show_source: false 15 | show_signature_annotations: true 16 | -------------------------------------------------------------------------------- /docs/api/component-instance.md: -------------------------------------------------------------------------------- 1 | ::: motion.ComponentInstance 2 | handler: python 3 | options: 4 | members: 5 | - run 6 | - arun 7 | - gen 8 | - agen 9 | - read_state 10 | - write_state 11 | - flush_update 12 | - version 13 | - shutdown 14 | - close 15 | - instance_name 16 | show_root_full_path: false 17 | show_root_toc_entry: false 18 | show_root_heading: true 19 | show_source: false 20 | show_signature_annotations: true 21 | -------------------------------------------------------------------------------- /docs/api/component.md: -------------------------------------------------------------------------------- 1 | ::: motion.Component 2 | handler: python 3 | options: 4 | members: 5 | - __init__ 6 | - serve 7 | - update 8 | - init_state 9 | - __call__ 10 | - save_state 11 | - load_state 12 | - name 13 | - params 14 | show_root_full_path: false 15 | show_root_toc_entry: false 16 | show_root_heading: true 17 | show_source: false 18 | show_signature_annotations: true 19 | 20 | ::: motion.DiscardPolicy 21 | handler: python 22 | options: 23 | show_root_full_path: false 24 | show_root_toc_entry: true 25 | show_root_heading: true 26 | show_source: false 27 | show_signature_annotations: true 28 | -------------------------------------------------------------------------------- /docs/api/mtable.md: -------------------------------------------------------------------------------- 1 | ::: motion.MTable 2 | handler: python 3 | options: 4 | members: 5 | - data 6 | - filesystem 7 | - from_pandas 8 | - from_arrow 9 | - from_schema 10 | - add_row 11 | - remove_row 12 | - add_column 13 | - append_column 14 | - remove_column 15 | - remove_column_by_name 16 | - knn 17 | - apply_distance 18 | show_root_full_path: false 19 | show_root_toc_entry: false 20 | show_root_heading: true 21 | show_source: true 22 | show_signature_annotations: true -------------------------------------------------------------------------------- /docs/api/props-and-state.md: -------------------------------------------------------------------------------- 1 | ::: motion.dicts.Properties 2 | handler: python 3 | options: 4 | members: 5 | - serve_result 6 | show_root_full_path: false 7 | show_root_toc_entry: false 8 | show_root_heading: true 9 | show_source: false 10 | show_signature_annotations: true 11 | 12 | ::: motion.dicts.State 13 | handler: python 14 | options: 15 | members: 16 | - instance_id 17 | show_root_full_path: false 18 | show_root_toc_entry: false 19 | show_root_heading: true 20 | show_source: false 21 | show_signature_annotations: true 22 | 23 | 24 | ::: motion.df.MDataFrame 25 | handler: python 26 | options: 27 | members: 28 | - __getstate__ 29 | - __setstate__ 30 | show_root_full_path: false 31 | show_root_toc_entry: false 32 | show_root_heading: true 33 | show_source: true 34 | show_signature_annotations: true 35 | -------------------------------------------------------------------------------- /docs/api/state-migrator.md: -------------------------------------------------------------------------------- 1 | ::: motion.migrate.StateMigrator 2 | handler: python 3 | options: 4 | members: 5 | - __init__ 6 | - migrate 7 | show_root_full_path: false 8 | show_root_toc_entry: false 9 | show_root_heading: true 10 | show_source: false 11 | show_signature_annotations: true 12 | 13 | ::: motion.copy_db 14 | handler: python 15 | options: 16 | show_root_full_path: false 17 | show_root_toc_entry: false 18 | show_root_heading: true 19 | show_source: false 20 | show_signature_annotations: true 21 | -------------------------------------------------------------------------------- /docs/examples/application.md: -------------------------------------------------------------------------------- 1 | # Building a Motion Application 2 | 3 | In this tutorial, we'll go through setting up an application of Motion components. We'll go through setting up a simple server with custom components and then demonstrate how to interact with it using Axios in a TypeScript environment. 4 | 5 | ## Prerequisites 6 | 7 | - [Motion](/motion/getting-started/installation/) 8 | - [FastAPI](https://fastapi.tiangolo.com/) -- installed automatically with Motion 9 | - [uvicorn](https://www.uvicorn.org/) -- To serve the application 10 | 11 | ## Step 1: Define Some Components 12 | 13 | We'll write two components: one representing a counter, and another representing a calculator. The counter will increment a count every time an operation is called. The calculator will add and subtract two numbers. 14 | 15 | ```python title="sample_components.py" linenums="1" 16 | from motion import Component 17 | 18 | Counter = Component("Counter") 19 | 20 | @Counter.init_state 21 | def setup(): 22 | return {"count": 0} 23 | 24 | @Counter.serve("increment") 25 | def increment(state, props): 26 | return state["count"] + 1 27 | 28 | @Counter.update("increment") 29 | def update_count(state, props): 30 | return {"count": state["count"] + 1} 31 | 32 | Calculator = Component("Calculator") 33 | 34 | @Calculator.serve("add") 35 | def add(state, props): 36 | return props["a"] + props["b"] 37 | 38 | @Calculator.serve("subtract") 39 | def subtract(state, props): 40 | return props["a"] - props["b"] 41 | ``` 42 | 43 | ## Step 2: Create an Application 44 | 45 | We'll create an application that serves the two components we just defined. 46 | 47 | ```python title="app.py" linenums="1" 48 | # app.py 49 | from motion import Application 50 | from sample_components import Counter, Calculator 51 | import uvicorn 52 | 53 | # Create the Motion application with both components 54 | motion_app = Application(components=[Counter, Calculator]) 55 | fastapi_app = motion_app.get_app() 56 | 57 | # Run the application using Uvicorn 58 | if __name__ == "__main__": 59 | # Print secret key 60 | print(motion_app.get_credentials()) 61 | 62 | uvicorn.run(fastapi_app, host="0.0.0.0", port=8000) 63 | ``` 64 | 65 | In the script, we include a statement to print the secret key (which is useful if you don't specify your secret key and Motion automatically creates one for your application). Keep this key safe, as it is used to authenticate requests to the application. 66 | 67 | To run the application, we can run `python app.py` in the terminal. We can also run `uvicorn app:fastapi_app --reload` to run the application with hot reloading. 68 | 69 | One of the powerful features of FastAPI is its automatic generation of interactive API documentation. Once your server is running, you can access the API documentation by visiting `http://localhost:8000/docs` in your web browser. This documentation provides a detailed overview of all the available routes (i.e., the Counter and Calculator component flows) and allows you to directly test the API endpoints from the browser. 70 | 71 | ## Step 3: Interact with the Application 72 | 73 | Suppose we are in TypeScript and want to interact with the application. We can use Axios to make requests to the application. First, install Axios in your web application: 74 | 75 | ```bash 76 | npm install axios 77 | ``` 78 | 79 | Then, we can write a simple Typescript function to make requests to the application: 80 | 81 | ```typescript title="queryMotionApp.ts" linenums="1" 82 | import axios from "axios"; 83 | 84 | const queryServer = async () => { 85 | const secretToken = "your_secret_key"; // Replace with the secret key from your Motion application 86 | 87 | try { 88 | // Increment the Counter 89 | const incrementResponse = await axios.post( 90 | "http://localhost:8000/Counter", 91 | { 92 | instance_id: "ts_testid", 93 | flow_key: "increment", 94 | props: {}, 95 | }, 96 | { 97 | headers: { Authorization: `Bearer ${secretToken}` }, 98 | } 99 | ); 100 | console.log("Counter Increment Result:", incrementResponse.data); 101 | 102 | // Perform an addition using the Calculator 103 | const addResponse = await axios.post( 104 | "http://localhost:8000/Calculator", 105 | { 106 | instance_id: "ts_testid", 107 | flow_key: "add", 108 | props: { a: 20, b: 10 }, 109 | }, 110 | { 111 | headers: { Authorization: `Bearer ${secretToken}` }, 112 | } 113 | ); 114 | console.log("Addition Result:", addResponse.data); 115 | } catch (error) { 116 | console.error("Error querying server:", error); 117 | } 118 | }; 119 | 120 | queryServer(); 121 | ``` 122 | 123 | Since the application is served as a REST API, we can also use any other HTTP client to interact with the application. 124 | -------------------------------------------------------------------------------- /docs/examples/design-patterns.md: -------------------------------------------------------------------------------- 1 | - Component instance-per-user (prompt personalization) 2 | - Reusing component instances (dynamic retrieval) 3 | - Caching results to save cost (wrapper around OpenAI LLM) 4 | - Disabled component instances 5 | - Context manager component instances 6 | - Generator UDFs (e.g., for streaming LLM output) 7 | -------------------------------------------------------------------------------- /docs/examples/hello-world.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/docs/examples/hello-world.md -------------------------------------------------------------------------------- /docs/examples/nl-query.md: -------------------------------------------------------------------------------- 1 | In this guide, we'll create a Motion application that runs natural language queries on a group of datasets, learning from feedback and new data over time. 2 | 3 | ## Install Motion 4 | 5 | Install Motion via pip: 6 | 7 | ```bash 8 | pip install motion-python 9 | ``` 10 | 11 | ## Write Components 12 | 13 | Our application will consist of two components: 14 | 15 | 1. A component that keeps track of the datasets we have available and their metadata. 16 | 2. A component that uses an LLM to execute our natural language queries, learning prompts based on our feedback. 17 | 18 | TODO 19 | 20 | 27 | -------------------------------------------------------------------------------- /docs/getting-started/concepts.md: -------------------------------------------------------------------------------- 1 | # Motion Concepts 2 | 3 | Motion applications consist of _components_ that hold state and _operations_ that read and update state. Components and operations are connected by _flows_. Think of a component as representing the prompt and LLM pipeline(s) for a particular task, the state as the prompt sub-parts, and flows as the different ways to interact with the state (e.g., assemble the sub-parts into a prompt, update the sub-parts, etc). 4 | 5 | ## The Component Lifecycle 6 | 7 | When a component instance is first created, an `init` function initializes the component's state. The state is a dictionary of key-value pairs, representing the initial sub-parts you might want to include in your prompt. The state is persisted in a key-value store (Redis) and is loaded when the component instance is initialized again. 8 | 9 | Components can have multiple flows that read and update the state (i.e., prompt sub-parts). A flow is represented by a string _key_ and consists of two user-defined operations, which run back-to-back: 10 | 11 | - **serve**: a function that takes in (1) a state dictionary that may not reflect all new information yet, and (2) a user-defined ` props` dictionary (passed in at runtime), then returns a result back to the user. 12 | 13 | - **update**: a function that runs in the background and takes in (1) the current state dictionary, and (2) the user-defined `props` dictionary, including the result of the serve op (accessed via `props.serve_result`). The `update` operation returns any updates to the state, which can be used in future operations. The `props` dictionary dies after the update operation for a flow. We run update operations in the background because they may be expensive and we don't want to block the serves. 14 | 15 | Serve operations do not modify the state, while update operations do. 16 | 17 | ## Concurrency and Consistency in Motion's Execution Engine 18 | 19 | Since serve operations do not modify the state, you can run multiple serve operations for the same component instance in parallel (e.g., in different Python processes). However, since update operations modify the state, Motion ensures that only one update operation is running at a time for a given component instance. This is done by maintaining queues of pending update operations and issuing exclusive write locks to update operations. Each component instance has its own lock and has a queue for each of its update operations. While update operations are running, serve operations can still run with low latency using stale state. The update queue is processed in a FIFO manner. 20 | 21 | ### Backpressure in Processing Update Operations 22 | 23 | Motion's execution engine experiences backpressure if a queue of pending update operations grows faster than the rate at which its update operations are completed. For example, if an update operation calls an LLM for a long prompt and takes 10 seconds to complete, and new update operations are being added to the queue every second, the queue will grow by 10 operations every second. While this does not pose problems for serve operations because serve operations can read stale state, it can cause the component instance to fall behind in processing update operations. 24 | 25 | Our solution to limit queue growth is to offer a configurable `DiscardPolicy` parameter for each update operation. There are two options for `DiscardPolicy`: 26 | 27 | - `DiscardPolicy.SECONDS`: If more than `discard_after` seconds have passed since the update operation _u_ was added to the queue, _u_ is removed from the queue and the state is not updated with _u_'s results. 28 | - `DiscardPolicy.NUM_NEW_UPDATES`: If more than `discard_after` new update operations have been added to the queue since an update operation _u_ was added, _u_ is removed from the queue and the state is not updated with _u_'s results. 29 | 30 | See the [API docs](/motion/api/component/#motion.DiscardPolicy) for how to use `DiscardPolicy`. 31 | 32 | ## State vs Props 33 | 34 | The difference between state and props can be a little confusing, since both are dictionaries. The main difference is that state is persistent, while props are ephemeral/limited to a flow. 35 | 36 | State is initialized when the component is created and persists between successive flows. Since Motion is backed by Redis, state also persists when the component is restarted. State is available to all operations for all flows, but can only be changed by update operations. 37 | 38 | On the other hand, props are passed in at runtime and are only available to the serve and update operations for a _single_ flow. Props can be modified in serve operation, so they can be used to pass data between serve and update operations. Of note is `props.serve_result`, which is the result of the serve operation for a flow (and thus only accessible in update operations). This is useful for update operations that need to use the result of the serve operation. Think of props like a kwargs dictionary that becomes irrelevant after the particular flow is finished. 39 | 40 | ### Things to Keep in Mind 41 | 42 | - Components can have many flows, each with their own key, serve operation, and update operation(s). 43 | - Components can only have one serve operation per key. 44 | - The `serve` operation is run on the main thread, while the `update` operation is run in the background. You directly get access to `serve` results, but `update` results are not accessible unless you read values from the state dictionary. 45 | - `serve` results are cached, with a default discard time of 24 hours. If you run a component twice on the same flow key-value pair, the second run will return the result of the first run. To override the caching behavior, see the [API docs](/motion/api/component-instance/#motion.instance.ComponentInstance.run). 46 | - `update` operations are processed sequentially in first-in-first-out (FIFO) order. This allows state to be updated incrementally. To handle backpressure, update operations can be configured to expire after a certain amount of time or after a certain number of new update operations have been added to the queue. See the [API docs](/motion/api/component/#motion.DiscardPolicy) for how to use `DiscardPolicy`. 47 | 48 | ## Example Component 49 | 50 | Here is an example component that computes the z-score of a value with respect to its history. 51 | 52 | ```python title="main.py" linenums="1" 53 | from motion import Component 54 | 55 | ZScoreComponent = Component("ZScore") 56 | 57 | 58 | @ZScoreComponent.init_state 59 | def setUp(): 60 | return {"mean": None, "std": None, "values": []} 61 | 62 | 63 | @ZScoreComponent.serve("number") 64 | def serve(state, props): # (1)! 65 | if state["mean"] is None: 66 | return None 67 | return abs(props["value"] - state["mean"]) / state["std"] 68 | 69 | 70 | @ZScoreComponent.update("number") 71 | def update(state, props): # (2)! 72 | # Result of the serve op can be accessed via 73 | # props.serve_result 74 | # We don't do anything with the results, but we could! 75 | value_list = state["values"] 76 | value_list.append(props["value"]) 77 | 78 | mean = sum(value_list) / len(value_list) 79 | std = sum((n - mean) ** 2 for n in value_list) / len(value_list) 80 | return {"mean": mean, "std": std, "values": value_list} 81 | ``` 82 | 83 | 1. This function is executed on the main thread, and the result is immediately returned to the user. 84 | 2. This function is executed in the background and merges the updates back to the state when ready. 85 | 86 | To run the component, we can create an instance of our component, `c`, and call `c.run` on the flow's key and value: 87 | 88 | ```python title="main.py" linenums="29" 89 | if __name__ == "__main__": 90 | import time 91 | c = ZScoreComponent() # Create instance of component 92 | 93 | # Observe 10 values of the flow's key 94 | for i in range(9): 95 | print(c.run("number", props={"value": i})) # (1)! 96 | 97 | c.run("number", props={"value": 9}, flush_update=True) # (2)! 98 | for i in range(10, 19): 99 | print(c.run("number", props={"value": i})) # (3)! 100 | 101 | print(c.run("number", props={"value": 10})) # (4)! 102 | time.sleep(5) # Give time for the second update to finish 103 | print(c.run("number", props={"value": 10}, force_refresh=True)) 104 | ``` 105 | 106 | 1. The first few runs might return None, as the mean and std are not yet initialized. 107 | 2. This will block until the resulting update operation has finished running. update ops run in the order that flows were executed (i.e., the update op for number 8 will run before the update op for number 9). 108 | 3. This uses the updated state dictionary from the previous run operation, since `flush_update` also updates the state. 109 | 4. This uses the cached result for 10. To ignore the cached result and rerun the serve op with a (potentially old) state, we should call `c.run("number", props={"value": 10}, ignore_cache=True)`. To make sure we have the latest state, we can call `c.run("number", props={"value": 10}, force_refresh=True)`. 110 | 111 | The output of the above code is: 112 | 113 | ```bash 114 | > python main.py 115 | None 116 | None 117 | None 118 | None 119 | None 120 | None 121 | None 122 | None 123 | None 124 | 0.6666666666666666 125 | 0.7878787878787878 126 | 0.9090909090909091 127 | 1.0303030303030303 128 | 1.1515151515151516 129 | 1.2727272727272727 130 | 1.393939393939394 131 | 1.5151515151515151 132 | 1.6363636363636365 133 | 0.6666666666666666 134 | 0.03327787021630613 135 | ``` 136 | 137 | Note that the `update` operation is running in a separate process, whenever new results come in. This is why the first several calls to `c.run` return `None`. 138 | 139 | ## Component Parameters 140 | 141 | You can inject static component parameters into your flow operations by passing them to the component constructor: 142 | 143 | ```python 144 | from motion import Component 145 | 146 | ZScoreComponent = Component("ZScore", params={"alert_threshold": 2.0}) 147 | ``` 148 | 149 | Then, you can access the parameters in your operations: 150 | 151 | ```python 152 | @ZScoreComponent.serve("number") 153 | def serve(state, props): 154 | if state["mean"] is None: 155 | return None 156 | z_score = abs(props["value"] - state["mean"]) / state["std"] 157 | if z_score > ZScoreComponent.params["alert_threshold"]: 158 | print("Alert!") 159 | return z_score 160 | ``` 161 | 162 | The `params` dictionary is immutable, so you can't modify it in your operations. This functionality is useful for experimenting with different values of a parameter without having to modify your code. 163 | -------------------------------------------------------------------------------- /docs/getting-started/installation.md: -------------------------------------------------------------------------------- 1 | # Installing Motion 2 | 3 | Motion is available on PyPI. Motion requires Python 3.8 or later. To install Motion, run the following command: 4 | 5 | ```bash 6 | pip install motion-python 7 | ``` 8 | 9 | To verify motion is working as intended, run `motion` in your terminal. An usage explanation should be returned, as well as a list of CLI commands that can be executed. 10 | 11 | To install Motion with support for Applications (FastAPI apps that serve Motion components) and Tables (wrapper around PyArrow table with zero-copy vector search), run the following command: 12 | 13 | ```bash 14 | pip install motion-python[application] 15 | pip install motion-python[table] 16 | ``` 17 | 18 | Optionally, the shorthand `pip install motion-python[application,table]` or `pip install motion-python[all]` can be used. 19 | 20 | ## Setting up the database 21 | 22 | Motion relies on Redis to store component state and metadata. You can install Redis [here](https://redis.io/download) and run it however you like, e.g., via [Docker](https://redis.io/docs/stack/get-started/install/docker/). You will need to configure the following environment variables: 23 | 24 | - `MOTION_REDIS_HOST`: The host of the Redis server. Defaults to `localhost`. 25 | - `MOTION_REDIS_PORT`: The port of the Redis server. Defaults to `6379`. 26 | - `MOTION_REDIS_PASSWORD`: The password of the Redis server. Defaults to `None`. 27 | - `MOTION_REDIS_DB`: The database of the Redis server. Defaults to `0`. 28 | 29 | ## (Optional) Installing from source 30 | 31 | Motion is developed and maintained on Github. We use `poetry` to manage dependencies and build the package. To install Motion from source, run the following commands: 32 | 33 | ```bash 34 | git clone https://github.com/dm4ml/motion 35 | cd motion 36 | make install 37 | ``` 38 | 39 | ## (Optional) Component Visualization Tool 40 | 41 | Check out the component visualization tool [here](https://dm4ml.github.io/motion-vis/). 42 | -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/docs/images/logo.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to Motion 2 | 3 | Motion is a system for defining and incrementally maintaining **self-updating prompts** in Python. 4 | 5 | !!! tip "Alpha Release" 6 | 7 | Motion is currently in alpha. We are actively working on improving the documentation and adding more features. If you are interested in using Motion and would like dedicated support from one of our team members, please reach out to us at [shreyashankar@berkeley.edu](mailto:shreyashankar@berkeley.edu). 8 | 9 | ## Why Self-Updating Prompts? 10 | 11 | LLM accuracy often significantly improves with more context. Consider an e-commerce focused LLM pipeline that recommends products to users. The recommendations might improve if the prompt considers the user's past purchases and browsing history. Ideally, any new information about the user (e.g., a new purchase or browsing event) should be incorporated into the LLM pipeline's prompts as soon as possible; thus, we call them **self-updating prompts**. 12 | 13 | ### Why is it Hard to Use Self-Updating Prompts? 14 | 15 | Consider the e-commerce example above. The prompt might grow to be very long---so long that there's a bunch of redundant or event useless information in the prompt. So, we might want to summarize the user's past purchases and browsing history into a single prompt. However, summarizing the user's past purchases and browsing history every time we log a new purchase or browsing event, or whenever the user requests a new recommendation, **can take too long** and thus prohibitively increase end-to-end latency for getting a recommendation. 16 | 17 | In general, we may want to use LLMs or run some other expensive operation when incrementally processing new information, e.g., through summarization, extracting structured information, or generating new data. When there is a lot of information to process, the best LLMs can take upwards of 30 seconds. This can be unacceptable for production latency. 18 | 19 | ## What is Motion? 20 | 21 | As LLM pipeline developers, we want a few things when building and using self-updating prompts: 22 | 23 | - **Flexibility**: We want to be able to define our sub-parts of prompts (e.g., summaries). We also want to be able to define our own logic for how to turn sub-parts into string prompts and self-updatingly update sub-parts. 24 | - **Availability**: We want there to always be some version of prompt sub-parts available, even if they are a little stale. This way we can minimize end-to-end latency. 25 | - **Freshness**: Prompts should incorporate as much of the latest information as possible. In the case where information arrives faster than we can process it, it may be desirable to ignore older information. 26 | 27 | Motion allows LLM pipeline developers to define and incrementally maintain self-updating prompts in Python. With Motion, we define **components** that represent prompt sub-parts, and **flows** that represent how to assemble sub-parts into a prompt for an LLM in real-time and how to self-updatingly update sub-parts in the background based on new information. 28 | 29 | Motion's execution engine serves cached prompt sub-parts for minimal real-time latency and handles concurrency and sub-part consistency when running flows that update sub-parts. All prompt sub-parts are backed by a key-value store. You can run Motion components anywhere and in any number of Python processes (e.g., in a notebook, in a serverless function, in a web server) at the same time for maximal availability. 30 | 31 | ## Should I use Motion? 32 | 33 | Motion is especially useful for LLM pipelines 34 | 35 | - Need to update prompts based on new data (e.g., maintain a dynamic summary in the prompt) 36 | - Want a Pythonic interface to build a distributed system of LLM application components 37 | 38 | Motion is built for developers who know how to code in Python and want to be able to control operations in their ML applications. For low-code and domain-specific development patterns (e.g., enhancing videos), you may want to check out other tools. 39 | 40 | ## Where did Motion come from? 41 | 42 | Motion is developed and maintained by researchers at the [UC Berkeley EPIC Lab](https://epic.berkeley.edu) who specialize in data management for ML pipelines. 43 | 44 | 57 | -------------------------------------------------------------------------------- /docs/tools/cli.md: -------------------------------------------------------------------------------- 1 | # Component Instance CLI Utils 2 | 3 | ## `motion clear` 4 | 5 | To easily clear a component instance, you can use the CLI command `motion clear`. If you type `motion clear --help`, you will see the following: 6 | 7 | ```bash 8 | $ motion clear --help 9 | Usage: motion clear [OPTIONS] INSTANCE 10 | 11 | Clears the state and cached results for a component instance. 12 | 13 | Args: instance (str): Instance name of the component to clear. 14 | In the form `componentname__instancename`. 15 | 16 | Options: 17 | --help Show this message and exit. 18 | 19 | Example usage: motion clear MyComponent__myinstance 20 | ``` 21 | 22 | ## `motion inspect` 23 | 24 | To easily view the state stored for a component instance, you can use the CLI command `motion inspect`. If you type `motion inspect --help`, you will see the following: 25 | 26 | ```bash 27 | $ motion inspect --help 28 | Usage: motion inspect [OPTIONS] INSTANCE 29 | 30 | Prints the saved state for a component instance. Does not apply any 31 | loadState() transformations. 32 | 33 | Args: instance (str): Instance name of the component to clear. 34 | In the form `componentname__instancename`. 35 | 36 | Options: 37 | --help Show this message and exit. 38 | 39 | Example usage: motion inspect MyComponent__myinstance 40 | ``` 41 | 42 | ## Python Documentation 43 | 44 | ::: motion.utils.clear_instance 45 | handler: python 46 | options: 47 | show_root_full_path: false 48 | show_root_toc_entry: false 49 | show_root_heading: true 50 | show_source: false 51 | show_signature_annotations: true 52 | heading_level: 3 53 | 54 | ::: motion.utils.inspect_state 55 | handler: python 56 | options: 57 | show_root_full_path: false 58 | show_root_toc_entry: false 59 | show_root_heading: true 60 | show_source: false 61 | show_signature_annotations: true 62 | heading_level: 3 63 | -------------------------------------------------------------------------------- /docs/tools/dashboard.md: -------------------------------------------------------------------------------- 1 | # Component Dashboard 2 | 3 | The dashboard is a web app that allows you to inspect and edit the states of your component instances. It is built with React and served with FastAPI. 4 | 5 | ## Running the Dashboard 6 | 7 | The dashboard is exposed as a FastAPI app, available via importing `motion.dashboard`. For example, to run the dashboard on `localhost:8000`, you can run the following code: 8 | 9 | ```python 10 | from motion.dashboard import dashboard_app 11 | 12 | if __name__ == "__main__": 13 | import uvicorn 14 | uvicorn.run(dashboard_app) 15 | ``` 16 | 17 | You can serve or deploy the dashboard app in any way you would serve a FastAPI app. 18 | 19 | ## Using the Dashboard 20 | 21 | The dashboard allows you to inspect component states and edit state key-value pairs. You can only edit key-value pairs with string, int, float, bool, list, or dict values. The dashboard does not support editing more complex types like numpy arrays or pandas dataframes, but you can still inspect these types. 22 | 23 | ## VictoriaMetrics Integration 24 | 25 | The dashboard also integrates with VictoriaMetrics, a time-series database. If you have a VictoriaMetrics instance running, you can configure Motion to log flow runs and visualize flow status in the dashboard. To do this, you can set the `MOTION_VICTORIAMETRICS_URL` environment variable to the URL of your VictoriaMetrics instance. For example, to use a VictoriaMetrics instance running on `localhost:8428`, you can add the following line to your .motionrc.yml file: 26 | 27 | ```yaml 28 | MOTION_VICTORIAMETRICS_URL: "http://localhost:8428" 29 | ``` 30 | -------------------------------------------------------------------------------- /docs/tools/migrator.md: -------------------------------------------------------------------------------- 1 | # State Migrator Tool 2 | 3 | Motion provides the [`StateMigrator`](/motion/api/state-migrator/#motion.migrate.StateMigrator). to migrate the state for all instances of a component, given a migration function that takes the old state and returns the new state. 4 | 5 | ## Usage 6 | 7 | The `StateMigrator` is a class that is initialized with a component and migration function. The migration function must have only one argument representing the state of a component instance, and must return a dictionary that replaces the state for that component instance. The migration function is applied to each component instance's state. 8 | 9 | This code snippet shows how to use the `StateMigrator` to add a new key to the state of all instances of a component. The migrator is run with a pool of 4 workers. 10 | 11 | ```python 12 | from motion import Component, StateMigrator 13 | 14 | # Create a component with a state with one key 15 | Something = Component("Something") 16 | 17 | 18 | @Something.init_state 19 | def setup(): 20 | return {"state_val": 0} 21 | 22 | # Create a migration function that adds a new key to the state 23 | def my_migrate_func(state): 24 | state.update({"another_val": 0}) 25 | return state 26 | 27 | if __name__ == "__main__": 28 | # Create a StateMigrator with the component and migration function 29 | sm = StateMigrator(Something, my_migrate_func) 30 | 31 | # Migrate the state for all instances of the component using a pool of 4 workers 32 | results = sm.migrate(num_workers=4) 33 | 34 | # See if there were any exceptions 35 | for result in results: 36 | if result.exception is not None: 37 | print(f"Exception for instance {result.instance_id}: {result.exception}") 38 | ``` 39 | 40 | If you want to migrate the state for a single component instance, you can pass in the instance IDs of the component instances to migrate: 41 | 42 | ```python 43 | results = sm.migrate(instance_ids=[...], num_workers=4) 44 | ``` 45 | -------------------------------------------------------------------------------- /docs/tools/vis.md: -------------------------------------------------------------------------------- 1 | # Component Visualization Tool 2 | 3 | We have developed a tool to visualize the structure of a Motion component. The tool is available [here](https://dm4ml.github.io/motion-vis/). 4 | 5 | ## Usage 6 | 7 | To get a Motion component file, you should run the CLI tool in the repository with your Motion component: 8 | 9 | ```bash 10 | $ motion vis : 11 | ``` 12 | 13 | For example, if I had a file called `main.py` like this: 14 | 15 | ```python 16 | from motion import Component 17 | 18 | ZScoreComponent = Component("ZScore") 19 | 20 | 21 | @ZScoreComponent.init_state 22 | def setUp(): 23 | return {"mean": None, "std": None, "values": []} 24 | 25 | 26 | @ZScoreComponent.serve("number") 27 | def serve(state, props): # (1)! 28 | if state["mean"] is None: 29 | return None 30 | return abs(props["value"] - state["mean"]) / state["std"] 31 | 32 | 33 | @ZScoreComponent.update("number") 34 | def update(state, props): # (2)! 35 | # Result of the serve op can be accessed via 36 | # props.serve_result 37 | # We don't do anything with the results, but we could! 38 | value_list = state["values"] 39 | value_list.append(props["value"]) 40 | 41 | mean = sum(value_list) / len(value_list) 42 | std = sum((n - mean) ** 2 for n in value_list) / len(value_list) 43 | return {"mean": mean, "std": std, "values": value_list} 44 | ``` 45 | 46 | I would run the CLI tool like this: 47 | 48 | ```bash 49 | $ motion vis main.py:ZScoreComponent 50 | ``` 51 | 52 | This will generate and save a JSON file to the current directory. You can then upload this file to the [vis tool](https://dm4ml.github.io/motion-vis) visualize the component. 53 | 54 | ## CLI Documentation 55 | 56 | Running `motion vis --help` will show the following: 57 | 58 | ```bash 59 | $ motion vis --help 60 | Usage: motion vis [OPTIONS] FILENAME 61 | 62 | Visualize a component. 63 | 64 | Options: 65 | --output TEXT JSON filename to output the component graph to. 66 | --help Show this message and exit. 67 | 68 | Example usage: motion vis main.py:MyComponent 69 | ``` 70 | -------------------------------------------------------------------------------- /ideas/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | - Remote update operations 4 | - Remote serve operations 5 | - State serialization 6 | - Observability/logging all inputs and outputs 7 | - Data validation on logged inputs/outputs 8 | - Trigger update from data validation result, not just from batch size 9 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Motion Docs 2 | site_url: https://dm4ml.github.io/motion/ 3 | repo_url: https://github.com/dm4ml/motion 4 | repo_name: dm4ml/motion 5 | remote_branch: gh-pages 6 | nav: 7 | - Getting Started: 8 | - Welcome: index.md 9 | - Installation: getting-started/installation.md 10 | - Concepts: getting-started/concepts.md 11 | - Examples: 12 | - Hello World: examples/hello-world.md 13 | - Common Design Patterns: examples/design-patterns.md 14 | - Orchestrating an Application of Components: examples/application.md 15 | - Querying Data with Natural Language: examples/nl-query.md 16 | - API Reference: 17 | - Component: api/component.md 18 | - ComponentInstance: api/component-instance.md 19 | - Props and State: api/props-and-state.md 20 | - MTable: api/mtable.md 21 | - Application: api/application.md 22 | - StateMigrator: api/state-migrator.md 23 | - Helpful Tools: 24 | - Component Dashboard: tools/dashboard.md 25 | - CLI: tools/cli.md 26 | - State Migrator: tools/migrator.md 27 | - Component Flow Visualization: tools/vis.md 28 | 29 | theme: 30 | name: material 31 | icon: 32 | logo: material/fast-forward-outline 33 | repo: fontawesome/brands/git-alt 34 | favicon: images/logo.png 35 | extra_files: 36 | - images/ 37 | palette: 38 | # Palette toggle for automatic mode 39 | - media: "(prefers-color-scheme)" 40 | primary: blue 41 | accent: orange 42 | toggle: 43 | icon: material/brightness-auto 44 | name: Switch to light mode 45 | 46 | # Palette toggle for light mode 47 | - media: "(prefers-color-scheme: light)" 48 | primary: blue 49 | accent: orange 50 | scheme: default 51 | toggle: 52 | icon: material/brightness-7 53 | name: Switch to dark mode 54 | 55 | # Palette toggle for dark mode 56 | - media: "(prefers-color-scheme: dark)" 57 | primary: blue 58 | accent: orange 59 | scheme: slate 60 | toggle: 61 | icon: material/brightness-4 62 | name: Switch to system preference 63 | font: 64 | text: Ubuntu 65 | code: Ubuntu Mono 66 | 67 | features: 68 | - navigation.instant 69 | - navigation.tracking 70 | - navigation.tabs 71 | - navigation.tabs.sticky 72 | - navigation.expand 73 | - navigation.path 74 | - toc.follow 75 | - header.autohide 76 | - content.code.copy 77 | - content.code.annotate 78 | 79 | plugins: 80 | - search 81 | - mkdocstrings 82 | - autorefs 83 | 84 | markdown_extensions: 85 | - abbr 86 | - admonition 87 | - def_list 88 | - footnotes 89 | - md_in_html 90 | - tables 91 | - pymdownx.snippets 92 | - pymdownx.inlinehilite 93 | - pymdownx.tabbed: 94 | alternate_style: true 95 | - pymdownx.superfences: 96 | custom_fences: 97 | - name: mermaid 98 | class: mermaid 99 | format: !!python/name:pymdownx.superfences.fence_code_format 100 | - pymdownx.details 101 | - attr_list 102 | - pymdownx.emoji: 103 | emoji_index: !!python/name:material.extensions.emoji.twemoji 104 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 105 | 106 | extra: 107 | version: 108 | provider: mike 109 | -------------------------------------------------------------------------------- /motion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from motion.component import Component 3 | from motion.utils import ( 4 | UpdateEventGroup, 5 | clear_instance, 6 | inspect_state, 7 | get_instances, 8 | RedisParams, 9 | ) 10 | from motion.instance import ComponentInstance 11 | from motion.migrate import StateMigrator 12 | from motion.copy_utils import copy_db 13 | from motion.discard_policy import DiscardPolicy 14 | 15 | __all__ = [ 16 | "Component", 17 | "UpdateEventGroup", 18 | "ComponentInstance", 19 | "clear_instance", 20 | "inspect_state", 21 | "StateMigrator", 22 | "get_instances", 23 | "copy_db", 24 | "RedisParams", 25 | "DiscardPolicy", 26 | ] 27 | 28 | # Conditionally import Application 29 | try: 30 | from motion.server.application import Application 31 | 32 | __all__.append("Application") 33 | except ImportError: 34 | 35 | class ApplicationImportError(ImportError): 36 | def __init__(self, *args: Any, **kwargs: Any) -> None: 37 | message = ( 38 | "The 'Application' class requires additional dependencies. " 39 | "Please install the 'application' extras by running: " 40 | "`pip install motion[application]`" 41 | ) 42 | super().__init__(message, *args, **kwargs) 43 | 44 | class Application: # type: ignore 45 | def __init__(self, *args: Any, **kwargs: Any) -> None: 46 | raise ApplicationImportError() 47 | 48 | __all__.append("Application") 49 | 50 | # Conditionally import MDataFrame and MTable 51 | try: 52 | from motion.df import MDataFrame 53 | from motion.mtable import MTable 54 | 55 | __all__.extend(["MDataFrame", "MTable"]) 56 | except ImportError: 57 | 58 | class TableImportError(ImportError): 59 | def __init__(self, *args: Any, **kwargs: Any) -> None: 60 | message = ( 61 | "The 'MDataFrame' and 'MTable' classes require additional dependencies. " 62 | "Please install the 'table' extras by running: " 63 | "`pip install motion[table]`" 64 | ) 65 | super().__init__(message, *args, **kwargs) 66 | 67 | class MDataFrame: # type: ignore 68 | def __init__(self, *args: Any, **kwargs: Any) -> None: 69 | raise TableImportError() 70 | 71 | class MTable: # type: ignore 72 | def __init__(self, *args: Any, **kwargs: Any) -> None: 73 | raise TableImportError() 74 | 75 | __all__.extend(["MDataFrame", "MTable"]) 76 | -------------------------------------------------------------------------------- /motion/cli.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import json 3 | import os 4 | import sys 5 | from datetime import datetime 6 | 7 | import click 8 | import redis 9 | import yaml 10 | from rich.console import Console 11 | 12 | from motion import clear_instance, get_instances, inspect_state 13 | 14 | 15 | @click.group() 16 | def motioncli() -> None: 17 | """Motion commands.""" 18 | pass 19 | 20 | 21 | @motioncli.command("init", epilog="Example usage:\n motion init") 22 | def init() -> None: 23 | """Initializes a motion project.""" 24 | # If .motionrc.yml already exists, do nothing 25 | if os.path.exists(".motionrc.yml"): 26 | click.echo("A .motionrc.yml file already exists in this directory.") 27 | return 28 | 29 | console = Console() 30 | checkmark = "\u2705" 31 | 32 | config = { 33 | "MOTION_REDIS_HOST": os.getenv("MOTION_REDIS_HOST", "localhost"), 34 | "MOTION_REDIS_PORT": int(os.getenv("MOTION_REDIS_PORT", "6379")), 35 | "MOTION_REDIS_DB": int(os.getenv("MOTION_REDIS_DB", "0")), 36 | "MOTION_REDIS_PASSWORD": os.getenv("MOTION_REDIS_PASSWORD"), 37 | "MOTION_REDIS_SSL": os.getenv("MOTION_REDIS_SSL", False), 38 | "MOTION_ENV": os.getenv("MOTION_ENV", "prod"), 39 | } 40 | 41 | # Creates an .motionrc.yml file 42 | with console.status("Creating .motionrc.yml", spinner="dots"): 43 | with open(".motionrc.yml", "w") as f: 44 | # Write the following 45 | yaml.dump(config, f) 46 | 47 | # Done 48 | click.echo(f"{checkmark} Created .motionrc.yml in current directory {os.getcwd()}.") 49 | 50 | 51 | @motioncli.command( 52 | "vis", 53 | epilog="Example usage:\n motion vis main.py:MyComponent", 54 | ) 55 | @click.argument( 56 | "filename", 57 | type=str, 58 | required=True, 59 | ) 60 | @click.option( 61 | "--output", 62 | type=str, 63 | default="graph.json", 64 | help="JSON filename to output the component graph to.", 65 | ) 66 | def visualize(filename: str, output: str) -> None: 67 | """Visualize a component.""" 68 | red_x = "\u274C" # Unicode code point for red "X" emoji 69 | if ":" not in filename: 70 | click.echo( 71 | f"{red_x} Component must be in the format " + "`filename:component`." 72 | ) 73 | return 74 | 75 | # Remove the file extension if present 76 | module = filename.replace(".py", "") 77 | 78 | first, instance = module.split(":") 79 | if not first or not instance: 80 | click.echo( 81 | f"{red_x} Component must be in the format " + "`filename:component`." 82 | ) 83 | return 84 | 85 | module_dir = os.getcwd() 86 | sys.path.insert(0, module_dir) 87 | module = importlib.import_module(first) # type: ignore 88 | 89 | # Get the class instance 90 | try: 91 | class_instance = getattr(module, instance) 92 | except AttributeError as e: 93 | click.echo(f"{red_x} {e}") 94 | return 95 | 96 | # Get the graph 97 | graph = class_instance.get_graph() 98 | 99 | # Dump the graph to a file with the date 100 | ts = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") 101 | out_filename = f"{ts}_{instance}_{output}" 102 | checkmark = "\u2705" # Unicode code point for checkmark emoji 103 | 104 | with open(out_filename, "w") as f: 105 | json.dump(graph, f, indent=4) 106 | click.echo(f"{checkmark} Graph dumped to {out_filename}") 107 | 108 | 109 | @motioncli.command( 110 | "clear", epilog="Example usage:\n motion clear MyComponent__myinstance" 111 | ) 112 | @click.argument("instance", type=str, required=True) 113 | def clear(instance: str) -> None: 114 | """Clears the state and cached results for a component instance. 115 | 116 | Args: 117 | instance (str): Instance name of the component to clear. 118 | In the form `componentname__instancename`. 119 | """ 120 | console = Console() 121 | red_x = "\u274C" 122 | checkmark = "\u2705" # Unicode code point for checkmark emoji 123 | with console.status("Clearing instance", spinner="dots"): 124 | try: 125 | found = clear_instance(instance) 126 | except ValueError as e: 127 | click.echo(f"{red_x} {e}") 128 | return 129 | except redis.exceptions.ConnectionError as e: 130 | click.echo(f"{red_x} {e}") 131 | return 132 | 133 | if not found: 134 | click.echo(f"{red_x} Instance {instance} not found.") 135 | 136 | else: 137 | click.echo(f"{checkmark} Instance {instance} cleared.") 138 | 139 | 140 | @motioncli.command( 141 | "inspect", epilog="Example usage:\n motion inspect MyComponent__myinstance" 142 | ) 143 | @click.argument("instance", type=str, required=True) 144 | def inspect(instance: str) -> None: 145 | """Prints the saved state for a component instance. Does not apply 146 | any loadState() transformations. 147 | 148 | Args: 149 | instance (str): Instance name of the component to inspect. 150 | In the form `componentname__instancename`. 151 | """ 152 | console = Console() 153 | red_x = "\u274C" 154 | checkmark = "\u2705" # Unicode code point for checkmark emoji 155 | with console.status("Inspecting instance", spinner="dots"): 156 | try: 157 | state = inspect_state(instance) 158 | except ValueError as e: 159 | click.echo(f"{red_x} {e}") 160 | return 161 | except redis.exceptions.ConnectionError as e: 162 | click.echo(f"{red_x} {e}") 163 | return 164 | 165 | console.print(state) 166 | click.echo(f"{checkmark} Printed state for instance {instance}.") 167 | 168 | 169 | @motioncli.command("list", epilog="Example usage:\n motion list MyComponent") 170 | @click.argument("component", type=str, required=True) 171 | def list(component: str) -> None: 172 | """Lists all instances of a component. 173 | 174 | Args: 175 | component (str): Name of the component to list instances of. 176 | """ 177 | console = Console() 178 | red_x = "\u274C" 179 | checkmark = "\u2705" # Unicode code point for checkmark emoji 180 | with console.status("Getting instances for component", spinner="dots"): 181 | try: 182 | instances = get_instances(component) 183 | except ValueError as e: 184 | click.echo(f"{red_x} {e}") 185 | return 186 | except redis.exceptions.ConnectionError as e: 187 | click.echo(f"{red_x} {e}") 188 | return 189 | 190 | for instance_id in instances: 191 | console.print(instance_id) 192 | 193 | click.echo( 194 | f"{checkmark} Listed all {len(instances)} instances for component {component}." 195 | ) 196 | 197 | 198 | if __name__ == "__main__": 199 | motioncli() 200 | -------------------------------------------------------------------------------- /motion/copy_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file has utilities to copy components and their state 3 | from one Redis instance to another. 4 | """ 5 | 6 | import logging 7 | 8 | import redis.asyncio as redis 9 | 10 | from motion.utils import RedisParams 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | async def copy_db(src: RedisParams, dest: RedisParams) -> None: 16 | """ 17 | Copy a component and its state from one Redis instance to another. 18 | 19 | Args: 20 | src: RedisParams for the source Redis instance. 21 | dest: RedisParams for the destination Redis instance. 22 | """ 23 | 24 | # Verify that src and dest are different 25 | if src.dict() is dest.dict(): 26 | raise ValueError("Source and destination must be different.") 27 | 28 | # establish connections 29 | 30 | src_con: redis.Redis = redis.Redis(**src.dict()) 31 | 32 | if await src_con.ping() is False: 33 | raise ValueError("Could not connect to source Redis instance.") 34 | 35 | dest_con: redis.Redis = redis.Redis(**dest.dict()) 36 | 37 | if await dest_con.ping() is False: 38 | raise ValueError("Could not connect to destination Redis instance.") 39 | 40 | # Copy all keys prefixed MOTION_STATE: and MOTION_VERSION: 41 | try: 42 | key_prefixes = ["MOTION_STATE:", "MOTION_VERSION:"] 43 | for key_prefix in key_prefixes: 44 | logger.info(f"Copying keys with prefix {key_prefix}") 45 | 46 | cursor = 0 47 | while True: 48 | cursor, keys = await src_con.scan( 49 | cursor=cursor, match=f"{key_prefix}*", count=1000 50 | ) 51 | if not keys: 52 | break 53 | 54 | # Pipeline to fetch all values in a single round trip 55 | pipeline = src_con.pipeline() 56 | for key in keys: 57 | pipeline.get(key) 58 | values = await pipeline.execute() 59 | 60 | # Pipeline to set all values in the destination Redis 61 | pipeline = dest_con.pipeline() 62 | for key, value in zip(keys, values): 63 | pipeline.set(key, value) 64 | await pipeline.execute() 65 | 66 | logger.info(f"Copied {len(keys)} keys with prefix {key_prefix}") 67 | 68 | # Make sure to convert cursor back to an integer if it's not already 69 | cursor = int(cursor) if cursor is not None else None 70 | if cursor == 0: 71 | break 72 | 73 | logger.info(f"Finished copying keys with prefix {key_prefix}.") 74 | 75 | except Exception as e: 76 | logger.error( 77 | "Error copying Motion database. Please try again. " 78 | + f"Partial results may have completed. {e}", 79 | exc_info=True, 80 | ) 81 | raise e 82 | 83 | finally: 84 | logger.info("Finished copying Motion database.") 85 | await src_con.close() 86 | await dest_con.close() 87 | -------------------------------------------------------------------------------- /motion/dashboard_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime, timedelta 3 | from typing import Any, Dict, List, Optional, Tuple 4 | 5 | import redis 6 | import requests 7 | 8 | from motion.utils import get_redis_params, loadState, saveState 9 | 10 | 11 | def query_victoriametrics(victoria_metrics_url: str, query: str) -> Any: 12 | """Query VictoriaMetrics using the given PromQL query.""" 13 | params = { 14 | "query": query, 15 | "time": "now", # Alternatively, you can specify a UNIX timestamp 16 | } 17 | response = requests.get(f"{victoria_metrics_url}/api/v1/query", params=params) 18 | return response.json() 19 | 20 | 21 | def query_victoriametrics_with_ts( 22 | victoria_metrics_url: str, query: str, start: int, end: int, step: int 23 | ) -> Any: 24 | """Query VictoriaMetrics using the given PromQL query.""" 25 | params = {"query": query, "start": start, "end": end, "step": step} 26 | response = requests.get(f"{victoria_metrics_url}/api/v1/query_range", params=params) # type: ignore 27 | return response.json() 28 | 29 | 30 | def calculate_percentage_change(old: int, new: int) -> float: 31 | if old == 0: 32 | return 0 if new == 0 else 100 33 | return ((new - old) / old) * 100 34 | 35 | 36 | def calculate_color_and_tooltip( 37 | success_count: float, total_count: float 38 | ) -> Tuple[str, str]: 39 | if total_count == 0: 40 | return "gray", "Inactive" 41 | success_rate = success_count / total_count 42 | if success_rate >= 0.95: 43 | return "emerald", "Operational" 44 | elif 0.80 <= success_rate < 0.95: 45 | return "yellow", "Degraded" 46 | else: 47 | return "rose", "Downtime" 48 | 49 | 50 | def get_interval_data( 51 | victoria_metrics_url: str, component_name: str, instance_id: Optional[str] = None 52 | ) -> Tuple[List[Dict[str, Any]], Optional[float]]: 53 | # Define time range 54 | end_time = datetime.now() 55 | start_time = end_time - timedelta(hours=24) 56 | start_ts = int(start_time.timestamp()) 57 | end_ts = int(end_time.timestamp()) 58 | step = 30 * 60 # 30 minutes in seconds 59 | 60 | # Query for counts over the last 24 hours in 30-minute intervals 61 | if instance_id: 62 | success_query = f'sum(sum_over_time(motion_operation_success_count_value{{component="{component_name}", instance="{instance_id}"}}[30m]))' # noqa: E501 63 | failure_query = f'sum(sum_over_time(motion_operation_failure_count_value{{component="{component_name}", instance="{instance_id}"}}[30m]))' # noqa: E501 64 | 65 | else: 66 | success_query = f'sum(sum_over_time(motion_operation_success_count_value{{component="{component_name}"}}[30m]))' # noqa: E501 67 | # Failure query 68 | failure_query = f'sum(sum_over_time(motion_operation_failure_count_value{{component="{component_name}"}}[30m]))' # noqa: E501 69 | 70 | response = query_victoriametrics_with_ts( 71 | victoria_metrics_url, success_query, start_ts, end_ts, step 72 | ) 73 | success_result = response.get("data", {}).get("result", []) 74 | 75 | response = query_victoriametrics_with_ts( 76 | victoria_metrics_url, failure_query, start_ts, end_ts, step 77 | ) 78 | failure_result = response.get("data", {}).get("result", []) 79 | 80 | success_interval_counts = { 81 | timestamp: 0.0 for timestamp in range(start_ts, end_ts, 1800) 82 | } 83 | failure_interval_counts = { 84 | timestamp: 0.0 for timestamp in range(start_ts, end_ts, 1800) 85 | } 86 | 87 | # Assuming there's only one series in the result 88 | if success_result: 89 | for value in success_result[0].get("values", []): 90 | timestamp = int(value[0]) 91 | count = float(value[1]) 92 | success_interval_counts[timestamp] = count 93 | 94 | if failure_result: 95 | for value in failure_result[0].get("values", []): 96 | timestamp = int(value[0]) 97 | count = float(value[1]) 98 | failure_interval_counts[timestamp] = count 99 | 100 | # Convert the dictionary to a list of counts in order 101 | success_counts = list(success_interval_counts.values()) 102 | failure_counts = list(failure_interval_counts.values()) 103 | 104 | # Turn it into bars 105 | bars = [] 106 | for success_count, failure_count in zip(success_counts, failure_counts): 107 | color, tooltip = calculate_color_and_tooltip( 108 | success_count, success_count + failure_count 109 | ) 110 | bars.append({"color": color, "tooltip": tooltip}) 111 | 112 | denominator = sum(success_counts) + sum(failure_counts) 113 | if denominator == 0: 114 | fraction_uptime = None 115 | else: 116 | fraction_uptime = sum(success_counts) / denominator * 100 117 | 118 | return bars, fraction_uptime 119 | 120 | 121 | def get_component_usage(component_name: str) -> Dict[str, Any]: 122 | # Retrieves the number of instances 123 | # and various statuses for a component 124 | rp = get_redis_params() 125 | redis_con = redis.Redis(**rp.dict()) 126 | 127 | # Count number of keys that match the component name 128 | instance_keys = redis_con.keys(f"MOTION_VERSION:{component_name}__*") 129 | instance_ids = [key.decode("utf-8").split("__")[-1] for key in instance_keys] 130 | 131 | redis_con.close() 132 | 133 | # Query victoria metrics if it exists 134 | flow_count_list = [] 135 | status_counts = {"success": 0, "failure": 0} 136 | prev_status_counts = {"success": 0, "failure": 0} 137 | status_changes = { 138 | "success": {"value": float("inf"), "deltaType": "increase"}, 139 | "failure": {"value": float("inf"), "deltaType": "increase"}, 140 | } 141 | status_bar_data: List[Dict[str, Any]] = [] 142 | fraction_uptime = None 143 | victoria_metrics_url = os.getenv("MOTION_VICTORIAMETRICS_URL") 144 | if victoria_metrics_url: 145 | # Count logs by flow 146 | promql_query = f'count(motion_operation_duration_seconds_value{{component="{component_name}"}}[24h]) by (flow)' # noqa: E501 147 | 148 | response = query_victoriametrics(victoria_metrics_url, promql_query) 149 | 150 | # Extract list of flows and their counts 151 | if response["status"] == "success": 152 | for result in response["data"]["result"]: 153 | flow = result["metric"]["flow"] 154 | count = result["value"][1] 155 | flow_count_list.append({"flow": flow, "count": int(count)}) 156 | 157 | # Count logs by success/failure 158 | status_promql_query = f'count(motion_operation_duration_seconds_value{{component="{component_name}"}}[24h]) by (status)' # noqa: E501 159 | 160 | response = query_victoriametrics(victoria_metrics_url, status_promql_query) 161 | 162 | # Extract success/failure counts 163 | if response["status"] == "success": 164 | for result in response["data"]["result"]: 165 | status = result["metric"]["status"] 166 | count = result["value"][1] 167 | status_counts[status] = int(count) 168 | 169 | # Previous period status counts 170 | prev_status_promql_query = f'count(motion_operation_duration_seconds_value{{component="{component_name}"}}[24h] offset 24h) by (status)' # noqa: E501 171 | prev_response = query_victoriametrics( 172 | victoria_metrics_url, prev_status_promql_query 173 | ) 174 | 175 | # Extract success/failure counts 176 | if prev_response["status"] == "success": 177 | for result in prev_response["data"]["result"]: 178 | status = result["metric"]["status"] 179 | count = result["value"][1] 180 | prev_status_counts[status] = int(count) 181 | 182 | # Calculate percentage changes 183 | for status, count in status_counts.items(): 184 | prev_count = prev_response.get(status, 0) 185 | change = calculate_percentage_change(prev_count, count) 186 | deltaType = "unchanged" 187 | if change > 0: 188 | deltaType = "increase" 189 | elif change < 0: 190 | deltaType = "decrease" 191 | status_changes[status] = { 192 | "value": f"{change:.2f}%", 193 | "deltaType": deltaType, 194 | } 195 | 196 | # Calculate interval data 197 | status_bar_data, fraction_uptime = get_interval_data( 198 | victoria_metrics_url, component_name 199 | ) 200 | 201 | # Sort the flow counts by count 202 | flow_count_list = sorted(flow_count_list, key=lambda x: x["count"], reverse=True) 203 | 204 | return { 205 | "numInstances": len(instance_ids), 206 | "instanceIds": instance_ids, 207 | "flowCounts": flow_count_list, 208 | "statusCounts": status_counts, 209 | "statusChanges": status_changes, 210 | "statusBarData": status_bar_data, 211 | "fractionUptime": fraction_uptime, 212 | } 213 | 214 | 215 | def get_component_instance_usage( 216 | component_name: str, instance_id: str, num_results: int = 100 217 | ) -> Dict[str, Any]: 218 | rp = get_redis_params() 219 | redis_con = redis.Redis(**rp.dict()) 220 | 221 | try: 222 | version = int(redis_con.get(f"MOTION_VERSION:{component_name}__{instance_id}")) # type: ignore 223 | except TypeError: 224 | raise ValueError(f"Instance {component_name}__{instance_id} does not exist.") 225 | 226 | redis_con.close() 227 | 228 | # Get the results by flow using the component name and instance id 229 | flowCounts = [] 230 | statusBarData: List[Dict[str, Any]] = [] 231 | fraction_uptime = None 232 | 233 | victoria_metrics_url = os.getenv("MOTION_VICTORIAMETRICS_URL") 234 | if victoria_metrics_url: 235 | # Count logs by flow 236 | promql_query = f'count(motion_operation_duration_seconds_value{{component="{component_name}",instance="{instance_id}"}}[24h]) by (flow)' # noqa: E501 237 | 238 | response = query_victoriametrics(victoria_metrics_url, promql_query) 239 | 240 | # Extract list of flows and their counts 241 | if response["status"] == "success": 242 | for result in response["data"]["result"]: 243 | flow = result["metric"]["flow"] 244 | count = result["value"][1] 245 | flowCounts.append({"flow": flow, "count": int(count)}) 246 | 247 | # Calculate interval data 248 | statusBarData, fraction_uptime = get_interval_data( 249 | victoria_metrics_url, component_name, instance_id 250 | ) 251 | 252 | # Sort the flow counts by count 253 | flowCounts = sorted(flowCounts, key=lambda x: x["count"], reverse=True) 254 | 255 | return { 256 | "version": version, 257 | "flowCounts": flowCounts, 258 | "statusBarData": statusBarData, 259 | "fractionUptime": fraction_uptime, 260 | } 261 | 262 | 263 | def writeState(instance_name: str, new_updates: Dict[str, Any]) -> None: 264 | # Load state and version from redis 265 | # Establish a connection to the Redis server 266 | rp = get_redis_params() 267 | redis_con = redis.Redis(**rp.dict()) 268 | 269 | state, version = loadState(redis_con, instance_name, None) 270 | if state is None: 271 | raise ValueError(f"Instance {instance_name} does not exist.") 272 | 273 | # Update the state 274 | state.update(new_updates) 275 | 276 | # Save the state 277 | saveState(state, version, redis_con, instance_name, None) 278 | 279 | # Close the connection to the Redis server 280 | redis_con.close() 281 | -------------------------------------------------------------------------------- /motion/df.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import pyarrow as pa 3 | 4 | 5 | class MDataFrame(pd.DataFrame): 6 | """Wrapper around pandas DataFrame that allows for pyarrow-based 7 | serialization. This is to be used in a motion component's state. 8 | 9 | Simply use this class instead of pandas DataFrame. For example: 10 | ```python 11 | from motion import MDataFrame, Component 12 | 13 | C = Component("MyDFComponent") 14 | 15 | @C.init_state 16 | def setUp(): 17 | df = MDataFrame({"value": [0, 1, 2]}) 18 | return {"df": df} 19 | ``` 20 | """ 21 | 22 | def __getstate__(self) -> dict: 23 | # Serialize with pyarrow 24 | table = pa.Table.from_pandas(self) 25 | # Convert the PyArrow Table to a PyArrow Buffer 26 | sink = pa.BufferOutputStream() 27 | writer = pa.ipc.new_stream(sink, table.schema) 28 | writer.write_table(table) 29 | writer.close() 30 | 31 | buffer = sink.getvalue() 32 | return {"table": buffer} 33 | 34 | def __setstate__(self, state: dict) -> None: 35 | # Convert the PyArrow Buffer to a PyArrow Table 36 | buf = state["table"] 37 | reader = pa.ipc.open_stream(buf) 38 | df = reader.read_pandas() 39 | self.__init__(df) # type: ignore 40 | -------------------------------------------------------------------------------- /motion/dicts.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains the props class, which is used to store 3 | properties of a flow. 4 | """ 5 | 6 | from typing import Any, Optional 7 | 8 | 9 | class CustomDict(dict): 10 | def __init__( 11 | self, 12 | component_name: str, 13 | dict_type: str, 14 | instance_id: Optional[str] = None, 15 | *args: Any, 16 | **kwargs: Any, 17 | ) -> None: 18 | self.component_name = component_name 19 | self.instance_id = instance_id 20 | self.dict_type = dict_type 21 | super().__init__(*args, **kwargs) 22 | 23 | def __getitem__(self, key: str) -> object: 24 | try: 25 | return super().__getitem__(key) 26 | except KeyError: 27 | raise KeyError( 28 | f"Key `{key}` not found in {self.dict_type} for " 29 | + f"instance {self.component_name}__{self.instance_id}." 30 | ) 31 | 32 | 33 | class Properties(dict): 34 | """Dictionary that stores properties of a flow. 35 | 36 | Example usage: 37 | 38 | ```python 39 | from motion import Component 40 | 41 | some_component = Component("SomeComponent") 42 | 43 | @some_component.init_state 44 | def setUp(): 45 | return {"model": ...} 46 | 47 | @some_component.serve("image") 48 | def predict_image(state, props): 49 | # props["image_embedding"] is passed in at runtime 50 | return state["model"](props["image_embedding"]) 51 | 52 | @some_component.update("image") 53 | def monitor_prediction(state, props): 54 | # props.serve_result is the result of the serve operation 55 | if props.serve_result > some_threshold: 56 | trigger_alert() 57 | 58 | if __name__ == "__main__": 59 | c = some_component() 60 | c.run("image", props={"image_embedding": ...}) 61 | ``` 62 | """ 63 | 64 | def __init__( 65 | self, 66 | *args: Any, 67 | **kwargs: Any, 68 | ) -> None: 69 | self._serve_result = None 70 | super().__init__(*args, **kwargs) 71 | 72 | def __getitem__(self, key: str) -> object: 73 | try: 74 | return super().__getitem__(key) 75 | except KeyError: 76 | raise KeyError(f"Key `{key}` not found in props. ") 77 | 78 | @property 79 | def serve_result(self) -> Any: 80 | """Stores the result of the serve operation. Can be accessed 81 | in the update operation, not the serve operation. 82 | 83 | Returns: 84 | Any: Result of the serve operation. 85 | """ 86 | return self._serve_result 87 | 88 | # def __getattr__(self, key: str) -> object: 89 | # return self.__getitem__(key) 90 | 91 | # def __setattr__(self, key: str, value: Any) -> None: 92 | # self[key] = value 93 | 94 | # def __getstate__(self) -> dict: 95 | # return dict(self) 96 | 97 | 98 | class State(dict): 99 | """Dictionary that stores state for a component instance. 100 | The instance id is stored in the `instance_id` attribute. 101 | 102 | Example usage: 103 | 104 | ```python 105 | from motion import Component 106 | 107 | some_component = Component("SomeComponent") 108 | 109 | @some_component.init_state 110 | def setUp(): 111 | return {"model": ...} 112 | 113 | @some_component.serve("retrieve") 114 | def retrieve_nn(state, props): 115 | # model can be accessed via state["model"] 116 | prediction = state["model"](props["image_embedding"]) 117 | # match the prediction to some other data to do a retrieval 118 | nn_component_instance = SomeOtherMotionComponent(state.instance_id) 119 | return nn_component_instance.run("retrieve", props={"prediction": prediction}) 120 | 121 | if __name__ == "__main__": 122 | c = some_component() 123 | nearest_neighbors = c.run("retrieve", props={"image_embedding": ...}) 124 | ``` 125 | """ 126 | 127 | def __init__( 128 | self, 129 | component_name: str, 130 | instance_id: str, 131 | *args: Any, 132 | **kwargs: Any, 133 | ) -> None: 134 | self.component_name = component_name 135 | self._instance_id = instance_id 136 | super().__init__(*args, **kwargs) 137 | 138 | @property 139 | def instance_id(self) -> str: 140 | """ 141 | Returns the instance id of the component. 142 | Useful if wanting to create other component instances 143 | within a serve or update operation. 144 | """ 145 | return self._instance_id 146 | 147 | def __getitem__(self, key: str) -> object: 148 | try: 149 | return super().__getitem__(key) 150 | except KeyError: 151 | raise KeyError( 152 | f"Key `{key}` not found in state for " 153 | + f"instance {self.component_name}__{self._instance_id}." 154 | ) 155 | 156 | 157 | class Params(dict): 158 | def __init__( 159 | self, 160 | *args: Any, 161 | **kwargs: Any, 162 | ) -> None: 163 | super().__init__(*args, **kwargs) 164 | 165 | def __getitem__(self, key: str) -> object: 166 | try: 167 | return super().__getitem__(key) 168 | except KeyError: 169 | raise KeyError(f"Key `{key}` not found in component params.") 170 | -------------------------------------------------------------------------------- /motion/discard_policy.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains discard policies for update queues. 3 | """ 4 | 5 | from enum import Enum 6 | from typing import Optional 7 | 8 | 9 | class DiscardPolicy(Enum): 10 | """ 11 | Defines the policy for discarding items in an update operation's queue. 12 | Each component instance has a queue for each update operation. Items in 13 | the queue are processed in first-in-first-out (FIFO) order, and items 14 | in the queue can delete based on the discard policy set by the 15 | developer. 16 | 17 | Attributes: 18 | NONE: Indicates no discard policy. Items in the queue do not delete. 19 | NUM_NEW_UPDATES: Items delete based on the number of new updates. Once 20 | the number of new updates exceeds a certain threshold, the oldest 21 | items are removed. 22 | SECONDS: Items delete based on time. Items older than a specified 23 | number of seconds at the time of processing are removed. 24 | 25 | Use the `discard_after` and `discard_policy` arguments in `Component.update` 26 | decorator to set the discard policy for an update operation. 27 | 28 | Example Usage: 29 | ```python 30 | from motion import Component, DiscardPolicy 31 | 32 | C = Component("C") 33 | 34 | @C.init_state 35 | def setup(): 36 | return {"default_value": 0, "some_value": 0, "another_value": 0} 37 | 38 | @C.update( 39 | "something", 40 | discard_after=10, 41 | discard_policy=DiscardPolicy.NUM_NEW_UPDATES 42 | ) 43 | def update_num_new(state, props): 44 | # Do an expensive operation that could take a while 45 | ... 46 | return {"some_value": state["some_value"] + props["value"]} 47 | 48 | @C.update("something", discard_after=1, discard_policy=DiscardPolicy.SECONDS) 49 | def update_seconds(state, props): 50 | # Do an expensive operation that could take a while 51 | ... 52 | return {"another_value": state["another_value"] + props["value"]} 53 | 54 | @C.update("something") 55 | def update_default(state, props): 56 | # Do an expensive operation that could take a while 57 | ... 58 | return {"default_value": state["default_value"] + props["value"]} 59 | 60 | if __name__ == "__main__": 61 | c = C() 62 | 63 | # If we do many runs of "something", the update queue will grow 64 | # and the policy will be automatically enforced by Motion. 65 | 66 | for i in range(100): 67 | c.run("something", props={"value": str(i)}) 68 | 69 | # Flush the update queue (i.e., wait for all updates to finish) 70 | c.flush_update("something") 71 | 72 | print(c.read_state("default_value")) # (1)! 73 | 74 | print(c.read_state("some_value")) # (2)! 75 | 76 | print(c.read_state("another_value")) # (3)! 77 | 78 | c.shutdown() 79 | ``` 80 | 81 | 1. The default policy is to not delete any items (DiscardPolicy.NONE), so 82 | the value of `default_value` will be the sum of all the values passed to 83 | `run` (i.e., `sum(range(100))`). 84 | 85 | 2. The NUM_NEW_UPDATES policy will delete items in the queue once the 86 | number of new updates exceeds a certain threshold. The threshold is set by 87 | the `discard_after` argument in the `update` decorator. So the result will 88 | be < 4950 because the NUM_NEW_UPDATES policy will have deleted some items. 89 | 90 | 3. This will be < 4950 because the SECONDS policy will have deleted some 91 | items (only whatever updates could have been processed in the second after 92 | they were added to the queue). 93 | """ 94 | 95 | NONE = 0 96 | """ No discard policy. Does not discard items in the queue. """ 97 | 98 | NUM_NEW_UPDATES = 1 99 | """ Delete items based on the number of new updates enqueued. """ 100 | 101 | SECONDS = 2 102 | """ Delete items based on time (in seconds). """ 103 | 104 | 105 | def validate_policy(policy: DiscardPolicy, discard_after: Optional[int]) -> None: 106 | if policy == DiscardPolicy.NONE: 107 | if discard_after is not None: 108 | raise ValueError("discard_after must be None for policy NONE") 109 | return 110 | 111 | if discard_after is None: 112 | raise ValueError("discard_after must be set for policy != NONE") 113 | 114 | if discard_after <= 0: 115 | raise ValueError("discard_after must be > 0") 116 | -------------------------------------------------------------------------------- /motion/migrate.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import logging 3 | import os 4 | from multiprocessing import Pool 5 | from typing import Callable, List, Optional, Tuple 6 | 7 | import redis 8 | from pydantic import BaseConfig, BaseModel, Field 9 | from tqdm import tqdm 10 | 11 | from motion.component import Component 12 | from motion.dicts import State 13 | from motion.utils import get_redis_params, loadState, saveState 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def process_migration( 19 | instance_name: str, 20 | migrate_func: Callable, 21 | load_state_fn: Callable, 22 | save_state_fn: Callable, 23 | ) -> Tuple[str, Optional[Exception]]: 24 | try: 25 | rp = get_redis_params() 26 | redis_con = redis.Redis( 27 | **rp.dict(), 28 | ) 29 | state, version = loadState(redis_con, instance_name, load_state_fn) 30 | 31 | new_state = migrate_func(state) 32 | assert isinstance(new_state, dict), ( 33 | "Migration function must return a dict." 34 | + " Warning: partial progress may have been made!" 35 | ) 36 | empty_state = State( 37 | instance_name.split("__")[0], 38 | instance_name.split("__")[1], 39 | {}, 40 | ) 41 | empty_state.update(new_state) 42 | success_indicator = saveState( 43 | empty_state, version, redis_con, instance_name, save_state_fn 44 | ) 45 | 46 | if success_indicator == -1: 47 | # Migration failed because the state was updated in the meantime 48 | raise RuntimeError( 49 | f"Migration failed for {instance_name} because the " 50 | + "state was updated by another process in the meantime." 51 | ) 52 | 53 | except Exception as e: 54 | if isinstance(e, AssertionError): 55 | raise e 56 | else: 57 | logger.error(e, exc_info=True) 58 | return instance_name, e 59 | 60 | redis_con.close() 61 | return instance_name, None 62 | 63 | 64 | class MigrationResult(BaseModel): 65 | instance_id: str = Field(..., description="Instance ID of the component") 66 | exception: Optional[Exception] = Field( 67 | None, description="Exception migration raised, if any" 68 | ) 69 | 70 | class Config(BaseConfig): 71 | arbitrary_types_allowed = True 72 | 73 | 74 | class StateMigrator: 75 | def __init__(self, component: Component, migrate_func: Callable) -> None: 76 | """Creates a StateMigrator object. 77 | 78 | Args: 79 | component (Component): Component to perform the migration for. 80 | migrate_func (Callable): Function to apply to the state of each 81 | instance of the component. 82 | 83 | Raises: 84 | TypeError: if component is not a valid Component 85 | ValueError: if migrate_func does not have exactly one parameter 86 | """ 87 | 88 | # Assert that we are in prod 89 | assert ( 90 | os.environ.get("MOTION_ENV") == "prod" 91 | ), "StateMigrator should only be used in prod." 92 | 93 | # Type check 94 | if not isinstance(component, Component): 95 | raise TypeError("component must be a valid Component") 96 | 97 | signature = inspect.signature(migrate_func) 98 | parameters = signature.parameters 99 | if len(parameters) != 1: 100 | raise ValueError("migrate_func must have exactly one parameter (`state`)") 101 | 102 | self.component = component 103 | self.migrate_func = migrate_func 104 | 105 | def migrate( 106 | self, instance_ids: List[str] = [], num_workers: int = 4 107 | ) -> List[MigrationResult]: 108 | """Performs the migrate_func for component instances' states. 109 | If instance_ids is empty, then migrate_func is performed for all 110 | instances of the component. 111 | 112 | Args: 113 | instance_ids (List[str], optional): 114 | List of instance ids to perform migration for. Defaults to 115 | empty list. 116 | num_workers (int, optional): 117 | Number of workers to use for parallel processing the migration. 118 | Defaults to 4. 119 | 120 | Returns: 121 | List[MigrationResult]: 122 | List of objects with instance_id and exception keys, where 123 | exception is None if the migration was successful for that 124 | instance name. 125 | """ 126 | 127 | # Read all the states 128 | 129 | rp = get_redis_params() 130 | redis_con = redis.Redis( 131 | **rp.dict(), 132 | ) 133 | instance_names = [ 134 | self.component.name + "__" + iid if "__" not in iid else iid 135 | for iid in instance_ids 136 | ] 137 | if not instance_names: 138 | instance_names = [ 139 | key.decode("utf-8").replace("MOTION_STATE:", "") # type: ignore 140 | for key in redis_con.keys(f"MOTION_STATE:{self.component.name}__*") 141 | ] 142 | 143 | if not instance_names: 144 | logger.warning(f"No instances for component {self.component.name} found.") 145 | 146 | # Create a process pool with 4 executors 147 | with Pool(num_workers) as executor: 148 | # Create a list of arguments for process_migration 149 | args_list = [ 150 | ( 151 | instance_name, 152 | self.migrate_func, 153 | self.component._load_state_func, 154 | self.component._save_state_func, 155 | ) 156 | for instance_name in instance_names 157 | ] 158 | 159 | # Initialize the progress bar 160 | progress_bar = tqdm( 161 | total=len(args_list), 162 | desc=f"Migrating state for {self.component.name}", 163 | unit="instance", 164 | ) 165 | 166 | # Process each key in parallel and update the progress bar 167 | # for each completed task 168 | results = [] 169 | for result in executor.starmap(process_migration, args_list): 170 | results.append(result) 171 | progress_bar.update(1) 172 | 173 | # Close the progress bar 174 | progress_bar.close() 175 | 176 | # Strip component name from instance names 177 | redis_con.close() 178 | mresults = [ 179 | MigrationResult(instance_id=instance_name.split("__")[-1], exception=e) 180 | for instance_name, e in results 181 | ] 182 | return mresults 183 | -------------------------------------------------------------------------------- /motion/route.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import logging 3 | from typing import Any, Callable, Dict 4 | 5 | from pydantic import BaseModel, Field, PrivateAttr 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class Route(BaseModel): 11 | key: str = Field(..., description="The keyword to which this route applies.") 12 | op: str = Field( 13 | ..., 14 | description="The operation to perform.", 15 | pattern="^(serve|update)$", 16 | ) 17 | udf: Callable = Field( 18 | ..., 19 | description="The udf to call for the op. The udf should have at least " 20 | + "a `state` argument.", 21 | ) 22 | _udf_params: Dict[str, Any] = PrivateAttr() 23 | 24 | def __init__(self, **data: Any) -> None: 25 | super().__init__(**data) 26 | udf_params = inspect.signature(self.udf).parameters 27 | self._udf_params = {param: udf_params[param].default for param in udf_params} 28 | 29 | def run(self, **kwargs: Any) -> Any: 30 | filtered_kwargs = { 31 | param: kwargs[param] for param in self._udf_params if param in kwargs 32 | } 33 | try: 34 | result = self.udf(**filtered_kwargs) 35 | except Exception as e: 36 | logger.error(f"Error in {self.key}, {self.op} flow: {e}", exc_info=True) 37 | raise e 38 | 39 | return result 40 | -------------------------------------------------------------------------------- /motion/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/motion/server/__init__.py -------------------------------------------------------------------------------- /motion/server/update_task.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | import traceback 4 | from multiprocessing import Process 5 | from threading import Thread 6 | from typing import Any, Callable, Dict, List, Optional 7 | 8 | import cloudpickle 9 | import redis 10 | import requests 11 | 12 | from motion.route import Route 13 | from motion.utils import FlowOpStatus, loadState, logger, saveState 14 | 15 | 16 | class BaseUpdateTask: 17 | def __init__( 18 | self, 19 | task_type: str, 20 | instance_name: str, 21 | routes: Dict[str, Route], 22 | save_state_func: Optional[Callable], 23 | load_state_func: Optional[Callable], 24 | queue_identifiers: List[str], 25 | channel_identifiers: Dict[str, str], 26 | lock_identifier: str, 27 | redis_params: Dict[str, Any], 28 | running: Any, 29 | victoria_metrics_url: Optional[str] = None, 30 | ): 31 | super().__init__() 32 | self.task_type = task_type 33 | self.name = f"UpdateTask-{task_type}-{instance_name}" 34 | self.instance_name = instance_name 35 | 36 | self._component_name, self._instance_id = instance_name.split("__") 37 | 38 | self.victoria_metrics_url = victoria_metrics_url 39 | 40 | self.save_state_func = save_state_func 41 | self.load_state_func = load_state_func 42 | 43 | self.routes = routes 44 | self.queue_identifiers = queue_identifiers 45 | self.channel_identifiers = channel_identifiers 46 | self.lock_identifier = lock_identifier 47 | 48 | self.running = running 49 | self.daemon = True 50 | 51 | self.redis_params = redis_params 52 | 53 | def _logMessage( 54 | self, 55 | flow_key: str, 56 | op_type: str, 57 | status: FlowOpStatus, 58 | duration: float, 59 | func_name: str, 60 | ) -> None: 61 | """Method to log a message directly to VictoriaMetrics using InfluxDB 62 | line protocol.""" 63 | if self.victoria_metrics_url: 64 | timestamp = int( 65 | time.time() * 1000000000 66 | ) # Nanoseconds for InfluxDB line protocol 67 | status_label = "success" if status == FlowOpStatus.SUCCESS else "failure" 68 | 69 | # Format the metric in InfluxDB line protocol 70 | metric_data = f"motion_operation_duration_seconds,component={self._component_name},instance={self._instance_id},flow={flow_key},op_type={op_type},status={status_label} value={duration} {timestamp}" # noqa: E501 71 | # Format the counter metrics for success and failure 72 | success_counter = f"motion_operation_success_count,component={self._component_name},instance={self._instance_id},flow={flow_key},op_type={op_type} value={1 if status_label == 'success' else 0} {timestamp}" # noqa: E501 73 | failure_counter = f"motion_operation_failure_count,component={self._component_name},instance={self._instance_id},flow={flow_key},op_type={op_type} value={1 if status_label == 'failure' else 0} {timestamp}" # noqa: E501 74 | 75 | # Combine metrics into a single payload with newline character 76 | payload = "\n".join([metric_data, success_counter, failure_counter]) 77 | 78 | try: 79 | # Send HTTP POST request with the combined metric data 80 | response = requests.post( 81 | self.victoria_metrics_url + "/write", data=payload 82 | ) 83 | response.raise_for_status() # Raise an exception for HTTP errors 84 | except requests.RequestException as e: 85 | logger.error(f"Failed to send metric to VictoriaMetrics: {e}") 86 | 87 | def custom_run(self) -> None: 88 | try: 89 | redis_con = None 90 | while self.running.value: 91 | if not redis_con: 92 | redis_con = redis.Redis(**self.redis_params) 93 | 94 | item: Dict[str, Any] = {} 95 | queue_name = "" 96 | try: 97 | # for _ in range(self.batch_size): 98 | full_item = redis_con.blpop(self.queue_identifiers, timeout=0.01) 99 | if full_item is None: 100 | if not self.running.value: 101 | break # no more items in the list 102 | else: 103 | continue 104 | 105 | queue_name = full_item[0].decode("utf-8") 106 | item = cloudpickle.loads(full_item[1]) 107 | # self.batch.append(item) 108 | # if flush_update: 109 | # break 110 | except redis.exceptions.ConnectionError: 111 | logger.error("Connection to redis lost.", exc_info=True) 112 | break 113 | 114 | # Check if we should stop 115 | if not self.running.value and not item: 116 | # self.cleanup() 117 | break 118 | 119 | # if not self.batch: 120 | # continue 121 | 122 | exception_str = "" 123 | # Check if it was a no op 124 | if item["identifier"].startswith("NOOP_"): 125 | redis_con.publish( 126 | self.channel_identifiers[queue_name], 127 | str( 128 | { 129 | "identifier": item["identifier"], 130 | "exception": exception_str, 131 | } 132 | ), 133 | ) 134 | continue 135 | 136 | # Check if item.get("expire_at") has passed 137 | expire_at = item.get("expire_at") 138 | if expire_at is not None: 139 | if expire_at < redis_con.time()[0]: 140 | redis_con.publish( 141 | self.channel_identifiers[queue_name], 142 | str( 143 | { 144 | "identifier": item["identifier"], 145 | "exception": "Expired", 146 | } 147 | ), 148 | ) 149 | continue 150 | 151 | # Run update op 152 | try: 153 | start_time = time.time() 154 | with redis_con.lock(self.lock_identifier, timeout=120): 155 | old_state, version = loadState( 156 | redis_con, 157 | self.instance_name, 158 | self.load_state_func, 159 | ) 160 | if old_state is None: 161 | # Create new state 162 | # If state does not exist, run setUp 163 | raise ValueError( 164 | f"State for {self.instance_name} not found." 165 | ) 166 | 167 | state_update = self.routes[queue_name].run( 168 | state=old_state, 169 | props=item["props"], 170 | ) 171 | # Await if state_update is a coroutine 172 | if asyncio.iscoroutine(state_update): 173 | state_update = asyncio.run(state_update) 174 | 175 | if not isinstance(state_update, dict): 176 | logger.error( 177 | "Update methods should return a dict of state updates.", 178 | exc_info=True, 179 | ) 180 | else: 181 | old_state.update(state_update) 182 | saveState( 183 | old_state, 184 | version, 185 | redis_con, 186 | self.instance_name, 187 | self.save_state_func, 188 | ) 189 | 190 | except Exception: 191 | logger.error(traceback.format_exc()) 192 | exception_str = str(traceback.format_exc()) 193 | 194 | duration = time.time() - start_time 195 | 196 | redis_con.publish( 197 | self.channel_identifiers[queue_name], 198 | str( 199 | { 200 | "identifier": item["identifier"], 201 | "exception": exception_str, 202 | } 203 | ), 204 | ) 205 | 206 | # Log to VictoriaMetrics 207 | if self.victoria_metrics_url: 208 | try: 209 | flow_key = queue_name.split("/")[-2] 210 | udf_name = queue_name.split("/")[-1] 211 | self._logMessage( 212 | flow_key, 213 | "update", 214 | ( 215 | FlowOpStatus.SUCCESS 216 | if not exception_str 217 | else FlowOpStatus.FAILURE 218 | ), 219 | duration, 220 | udf_name, 221 | ) 222 | except Exception as e: 223 | logger.error( 224 | f"Error logging to VictoriaMetrics: {e}", exc_info=True 225 | ) 226 | 227 | finally: 228 | if redis_con: 229 | redis_con.close() 230 | 231 | 232 | class UpdateProcess(Process): 233 | def __init__( 234 | self, 235 | **kwargs: Any, 236 | ) -> None: 237 | super().__init__() 238 | self.name = f"UpdateTask-{kwargs.get('instance_name', '')}" 239 | self.daemon = True 240 | self.but = BaseUpdateTask(task_type="process", **kwargs) 241 | 242 | def run(self) -> None: 243 | self.but.custom_run() 244 | 245 | 246 | class UpdateThread(Thread): 247 | def __init__( 248 | self, 249 | **kwargs: Any, 250 | ) -> None: 251 | super().__init__() 252 | self.name = f"UpdateTask-{kwargs.get('instance_name', '')}" 253 | self.daemon = True 254 | self.but = BaseUpdateTask(task_type="thread", **kwargs) 255 | 256 | def run(self) -> None: 257 | self.but.custom_run() 258 | -------------------------------------------------------------------------------- /motion/static/asset-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": { 3 | "main.css": "/static/css/main.4e812611.css", 4 | "main.js": "/static/js/main.532352eb.js", 5 | "static/js/808.653ad45b.chunk.js": "/static/js/808.653ad45b.chunk.js", 6 | "index.html": "/index.html", 7 | "main.4e812611.css.map": "/static/css/main.4e812611.css.map", 8 | "main.532352eb.js.map": "/static/js/main.532352eb.js.map", 9 | "808.653ad45b.chunk.js.map": "/static/js/808.653ad45b.chunk.js.map" 10 | }, 11 | "entrypoints": [ 12 | "static/css/main.4e812611.css", 13 | "static/js/main.532352eb.js" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /motion/static/index.html: -------------------------------------------------------------------------------- 1 | Motion Dashboard
2 | -------------------------------------------------------------------------------- /motion/static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/motion/static/logo.png -------------------------------------------------------------------------------- /motion/static/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "logo.png", 7 | "type": "image/png", 8 | "sizes": "192x192" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /motion/static/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /motion/static/static/js/808.653ad45b.chunk.js: -------------------------------------------------------------------------------- 1 | "use strict";(self.webpackChunkui=self.webpackChunkui||[]).push([[808],{8808:(e,n,t)=>{t.r(n),t.d(n,{CLSThresholds:()=>I,FCPThresholds:()=>S,FIDThresholds:()=>N,INPThresholds:()=>G,LCPThresholds:()=>X,TTFBThresholds:()=>$,getCLS:()=>F,getFCP:()=>P,getFID:()=>R,getINP:()=>W,getLCP:()=>Z,getTTFB:()=>ne,onCLS:()=>F,onFCP:()=>P,onFID:()=>R,onINP:()=>W,onLCP:()=>Z,onTTFB:()=>ne});var r,i,o,a,c,u=-1,s=function(e){addEventListener("pageshow",(function(n){n.persisted&&(u=n.timeStamp,e(n))}),!0)},f=function(){return window.performance&&performance.getEntriesByType&&performance.getEntriesByType("navigation")[0]},d=function(){var e=f();return e&&e.activationStart||0},l=function(e,n){var t=f(),r="navigate";return u>=0?r="back-forward-cache":t&&(document.prerendering||d()>0?r="prerender":document.wasDiscarded?r="restore":t.type&&(r=t.type.replace(/_/g,"-"))),{name:e,value:void 0===n?-1:n,rating:"good",delta:0,entries:[],id:"v3-".concat(Date.now(),"-").concat(Math.floor(8999999999999*Math.random())+1e12),navigationType:r}},p=function(e,n,t){try{if(PerformanceObserver.supportedEntryTypes.includes(e)){var r=new PerformanceObserver((function(e){Promise.resolve().then((function(){n(e.getEntries())}))}));return r.observe(Object.assign({type:e,buffered:!0},t||{})),r}}catch(e){}},v=function(e,n,t,r){var i,o;return function(a){n.value>=0&&(a||r)&&((o=n.value-(i||0))||void 0===i)&&(i=n.value,n.delta=o,n.rating=function(e,n){return e>n[1]?"poor":e>n[0]?"needs-improvement":"good"}(n.value,t),e(n))}},m=function(e){requestAnimationFrame((function(){return requestAnimationFrame((function(){return e()}))}))},h=function(e){var n=function(n){"pagehide"!==n.type&&"hidden"!==document.visibilityState||e(n)};addEventListener("visibilitychange",n,!0),addEventListener("pagehide",n,!0)},g=function(e){var n=!1;return function(t){n||(e(t),n=!0)}},T=-1,y=function(){return"hidden"!==document.visibilityState||document.prerendering?1/0:0},C=function(e){"hidden"===document.visibilityState&&T>-1&&(T="visibilitychange"===e.type?e.timeStamp:0,L())},E=function(){addEventListener("visibilitychange",C,!0),addEventListener("prerenderingchange",C,!0)},L=function(){removeEventListener("visibilitychange",C,!0),removeEventListener("prerenderingchange",C,!0)},w=function(){return T<0&&(T=y(),E(),s((function(){setTimeout((function(){T=y(),E()}),0)}))),{get firstHiddenTime(){return T}}},b=function(e){document.prerendering?addEventListener("prerenderingchange",(function(){return e()}),!0):e()},S=[1800,3e3],P=function(e,n){n=n||{},b((function(){var t,r=w(),i=l("FCP"),o=p("paint",(function(e){e.forEach((function(e){"first-contentful-paint"===e.name&&(o.disconnect(),e.startTimer.value&&(r.value=i,r.entries=o,t())},c=p("layout-shift",a);c&&(t=v(e,r,I,n.reportAllChanges),h((function(){a(c.takeRecords()),t(!0)})),s((function(){i=0,r=l("CLS",0),t=v(e,r,I,n.reportAllChanges),m((function(){return t()}))})),setTimeout(t,0))})))},A={passive:!0,capture:!0},k=new Date,D=function(e,n){r||(r=n,i=e,o=new Date,x(removeEventListener),M())},M=function(){if(i>=0&&i1e12?new Date:performance.now())-e.timeStamp;"pointerdown"==e.type?function(e,n){var t=function(){D(e,n),i()},r=function(){i()},i=function(){removeEventListener("pointerup",t,A),removeEventListener("pointercancel",r,A)};addEventListener("pointerup",t,A),addEventListener("pointercancel",r,A)}(n,e):D(n,e)}},x=function(e){["mousedown","keydown","touchstart","pointerdown"].forEach((function(n){return e(n,B,A)}))},N=[100,300],R=function(e,n){n=n||{},b((function(){var t,o=w(),c=l("FID"),u=function(e){e.startTimen.latency){if(t)t.entries.push(e),t.latency=Math.max(t.latency,e.duration);else{var r={id:e.interactionId,latency:e.duration,entries:[e]};U[r.id]=r,Q.push(r)}Q.sort((function(e,n){return n.latency-e.latency})),Q.splice(10).forEach((function(e){delete U[e.id]}))}},W=function(e,n){n=n||{},b((function(){var t;z();var r,i=l("INP"),o=function(e){e.forEach((function(e){e.interactionId&&V(e),"first-input"===e.entryType&&!Q.some((function(n){return n.entries.some((function(n){return e.duration===n.duration&&e.startTime===n.startTime}))}))&&V(e)}));var n,t=(n=Math.min(Q.length-1,Math.floor(K()/50)),Q[n]);t&&t.latency!==i.value&&(i.value=t.latency,i.entries=t.entries,r())},a=p("event",o,{durationThreshold:null!==(t=n.durationThreshold)&&void 0!==t?t:40});r=v(e,i,G,n.reportAllChanges),a&&("PerformanceEventTiming"in window&&"interactionId"in PerformanceEventTiming.prototype&&a.observe({type:"first-input",buffered:!0}),h((function(){o(a.takeRecords()),i.value<0&&K()>0&&(i.value=0,i.entries=[]),r(!0)})),s((function(){Q=[],J=_(),i=l("INP"),r=v(e,i,G,n.reportAllChanges)})))}))},X=[2500,4e3],Y={},Z=function(e,n){n=n||{},b((function(){var t,r=w(),i=l("LCP"),o=function(e){var n=e[e.length-1];n&&n.startTimeperformance.now())return;t.value=Math.max(o-d(),0),t.entries=[i],r(!0),s((function(){t=l("TTFB",0),(r=v(e,t,$,n.reportAllChanges))(!0)}))}}))}}}]); 2 | //# sourceMappingURL=808.653ad45b.chunk.js.map 3 | -------------------------------------------------------------------------------- /motion/static/static/js/main.532352eb.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /*! decimal.js-light v2.5.1 https://github.com/MikeMcl/decimal.js-light/LICENCE */ 2 | 3 | /** 4 | * @license React 5 | * react-dom.production.min.js 6 | * 7 | * Copyright (c) Facebook, Inc. and its affiliates. 8 | * 9 | * This source code is licensed under the MIT license found in the 10 | * LICENSE file in the root directory of this source tree. 11 | */ 12 | 13 | /** 14 | * @license React 15 | * react-jsx-runtime.production.min.js 16 | * 17 | * Copyright (c) Facebook, Inc. and its affiliates. 18 | * 19 | * This source code is licensed under the MIT license found in the 20 | * LICENSE file in the root directory of this source tree. 21 | */ 22 | 23 | /** 24 | * @license React 25 | * react.production.min.js 26 | * 27 | * Copyright (c) Facebook, Inc. and its affiliates. 28 | * 29 | * This source code is licensed under the MIT license found in the 30 | * LICENSE file in the root directory of this source tree. 31 | */ 32 | 33 | /** 34 | * @license React 35 | * scheduler.production.min.js 36 | * 37 | * Copyright (c) Facebook, Inc. and its affiliates. 38 | * 39 | * This source code is licensed under the MIT license found in the 40 | * LICENSE file in the root directory of this source tree. 41 | */ 42 | 43 | /** 44 | * @remix-run/router v1.15.0 45 | * 46 | * Copyright (c) Remix Software Inc. 47 | * 48 | * This source code is licensed under the MIT license found in the 49 | * LICENSE.md file in the root directory of this source tree. 50 | * 51 | * @license MIT 52 | */ 53 | 54 | /** 55 | * React Router DOM v6.22.0 56 | * 57 | * Copyright (c) Remix Software Inc. 58 | * 59 | * This source code is licensed under the MIT license found in the 60 | * LICENSE.md file in the root directory of this source tree. 61 | * 62 | * @license MIT 63 | */ 64 | 65 | /** 66 | * React Router v6.22.0 67 | * 68 | * Copyright (c) Remix Software Inc. 69 | * 70 | * This source code is licensed under the MIT license found in the 71 | * LICENSE.md file in the root directory of this source tree. 72 | * 73 | * @license MIT 74 | */ 75 | 76 | /** 77 | * react-virtual 78 | * 79 | * Copyright (c) TanStack 80 | * 81 | * This source code is licensed under the MIT license found in the 82 | * LICENSE.md file in the root directory of this source tree. 83 | * 84 | * @license MIT 85 | */ 86 | 87 | /** 88 | * virtual-core 89 | * 90 | * Copyright (c) TanStack 91 | * 92 | * This source code is licensed under the MIT license found in the 93 | * LICENSE.md file in the root directory of this source tree. 94 | * 95 | * @license MIT 96 | */ 97 | 98 | /** @license React v16.13.1 99 | * react-is.production.min.js 100 | * 101 | * Copyright (c) Facebook, Inc. and its affiliates. 102 | * 103 | * This source code is licensed under the MIT license found in the 104 | * LICENSE file in the root directory of this source tree. 105 | */ 106 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "motion-python" 3 | version = "0.1.132" 4 | description = "A framework for building and maintaining self-updating prompts for LLMs." 5 | authors = ["Shreya Shankar "] 6 | readme = "README.md" 7 | packages = [{include = "motion"}] 8 | include = ["static/**/*"] 9 | 10 | [tool.poetry.dependencies] 11 | python = "^3.9" 12 | click = "^8.1.3" 13 | colorlog = "^6.7.0" 14 | cloudpickle = "^2.0" 15 | redis = "^4.5.5" 16 | psutil = "^5.9.5" 17 | rich = "^13.4.1" 18 | pyyaml = "^6.0.1" 19 | tqdm = "^4.66.1" 20 | fastvs = { version = "^0.1.7", optional = true } 21 | pyarrow = { version = "^14.0.1", optional = true } 22 | pandas = { version = "^2.1.0", optional = true } 23 | pyjwt = { version = "^2.8.0", optional = true } 24 | fastapi = { version = "^0.108.0", optional = true } 25 | pydantic = "^2.5.3" 26 | 27 | [tool.poetry.extras] 28 | application = ["pyjwt", "fastapi"] 29 | table = ["fastvs", "pyarrow", "pandas"] 30 | all = ["pyjwt", "fastapi", "fastvs", "pyarrow", "pandas"] 31 | 32 | [tool.poetry.group.dev.dependencies] 33 | poetry-types = "^0.3.5" 34 | pytest = "^7.2.2" 35 | mypy = "^1.1.1" 36 | coverage = {extras = ["toml"], version = "^7.2.3"} 37 | pre-commit = "^3.2.1" 38 | types-requests = "^2.28.11.16" 39 | types-croniter = "^1.3.2.7" 40 | mkdocs = "^1.4.2" 41 | mkdocs-terminal = "^4.2.0" 42 | mkdocs-material = "^9.1.5" 43 | mkdocstrings = {version="^0.20.0", extras = ["python"] } 44 | pytkdocs = "^0.16.1" 45 | linkchecker = "^10.2.1" 46 | maturin = "^0.14.17" 47 | mike = "^1.1.2" 48 | scikit-learn = "^1.2.2" 49 | types-redis = "^4.5.5.2" 50 | httpx = "^0.24.1" 51 | pytest-asyncio = "^0.21.0" 52 | pytest-timeout = "^2.1.0" 53 | types-pyyaml = "^6.0.12.10" 54 | ruff = "^0.1.14" 55 | 56 | [tool.poetry.scripts] 57 | motion = "motion.cli:motioncli" 58 | 59 | [tool.pytest.ini_options] 60 | testpaths = ["tests"] 61 | addopts = "--basetemp=/tmp/pytest" 62 | filterwarnings = [ 63 | "ignore::DeprecationWarning", 64 | "ignore::UserWarning", 65 | "ignore::RuntimeWarning" 66 | ] 67 | 68 | [tool.mypy] 69 | files = "motion" 70 | mypy_path = "motion" 71 | warn_return_any = true 72 | warn_unused_configs = true 73 | disallow_untyped_defs = true 74 | exclude = ['motion/tests*'] 75 | ignore_missing_imports = true 76 | show_error_codes = true 77 | 78 | [tool.coverage.run] 79 | omit = [".*", "*/site-packages/*"] 80 | 81 | [build-system] 82 | requires = ["poetry-core"] 83 | build-backend = "poetry.core.masonry.api" 84 | 85 | [tool.coverage.report] 86 | exclude_lines = [ 87 | "pragma: no cover", 88 | "if TYPE_CHECKING:" 89 | ] 90 | fail_under = 100 91 | -------------------------------------------------------------------------------- /tests/.motionrc.yml: -------------------------------------------------------------------------------- 1 | MOTION_REDIS_HOST: "localhost" 2 | MOTION_REDIS_PORT: "6381" 3 | MOTION_REDIS_PASSWORD: null 4 | MOTION_REDIS_DB: "0" 5 | MOTION_ENV: "prod" 6 | MOTION_VICTORIAMETRICS_URL: "http://localhost:8428" 7 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/tests/__init__.py -------------------------------------------------------------------------------- /tests/component/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/tests/component/__init__.py -------------------------------------------------------------------------------- /tests/component/test_bad_components.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | import pytest 4 | 5 | 6 | def test_bad_serve_component(): 7 | with pytest.raises(ValueError): 8 | c = Component("BadserveComponent") 9 | 10 | @c.init_state 11 | def setUp(): 12 | return {"value": 0} 13 | 14 | @c.serve("add") 15 | def plus(state, props): 16 | return state["value"] + props["value"] 17 | 18 | @c.serve("add") 19 | def plus2(state, props): 20 | return state["value"] + props["value"] 21 | 22 | 23 | c = Component(name="DoubleFit", params={}) 24 | 25 | 26 | @c.init_state 27 | def setUp(): 28 | return {"value": 0} 29 | 30 | 31 | @c.update("add") 32 | def plus(state, props): 33 | return {"value": state["value"] + props["value"]} 34 | 35 | 36 | @c.update("add") 37 | def plus2(state, props): 38 | return {"value": state["value"] + props["value"]} 39 | 40 | 41 | @c.serve("read") 42 | def read(state, props): 43 | return props["value"] 44 | 45 | 46 | def test_double_fit_component(): 47 | c_instance = c() 48 | 49 | c_instance.run("add", props={"value": 1}, flush_update=True) 50 | 51 | assert c_instance.run("read", props={"value": 2}) == 2 52 | c_instance.shutdown() 53 | -------------------------------------------------------------------------------- /tests/component/test_context_manager.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | import pytest 3 | 4 | CounterCM = Component("CounterCM") 5 | 6 | 7 | @CounterCM.init_state 8 | def setUp(): 9 | return {"value": 0, "list_val": [1, 2, 3], "dict_val": {"a": 1, "b": 2}} 10 | 11 | 12 | @CounterCM.serve("number") 13 | def noop(state, props): 14 | return state["value"], props["value"] 15 | 16 | 17 | @CounterCM.update("number") 18 | def increment(state, props): 19 | return {"value": state["value"] + props["value"]} 20 | 21 | 22 | def test_context_manager(): 23 | with CounterCM() as c: 24 | assert c.read_state("value") == 0 25 | 26 | assert c.run("number", props={"value": 1}, flush_update=True)[1] == 1 27 | c.run("number", props={"value": 2}, flush_update=True) 28 | assert c.run("number", props={"value": 3}, flush_update=True)[1] == 3 29 | assert c.run("number", props={"value": 4}, flush_update=True)[0] == 6 30 | -------------------------------------------------------------------------------- /tests/component/test_counter.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | import pytest 3 | 4 | Counter = Component("Counter") 5 | 6 | 7 | @Counter.init_state 8 | def setUp(): 9 | return {"value": 0} 10 | 11 | 12 | @Counter.serve("number") 13 | def noop(state, props): 14 | return state["value"], props["value"] 15 | 16 | 17 | @Counter.update("number") 18 | def increment(state, props): 19 | return {"value": state["value"] + props["value"]} 20 | 21 | 22 | def test_create(): 23 | c = Counter(disable_update_task=True) 24 | 25 | assert c.read_state("value") == 0 26 | 27 | assert c.run("number", props={"value": 1}, flush_update=True)[1] == 1 28 | c.run("number", props={"value": 2}, flush_update=True) 29 | assert c.run("number", props={"value": 3}, flush_update=True)[1] == 3 30 | assert c.run("number", props={"value": 4}, flush_update=True)[0] == 6 31 | 32 | # Should raise errors 33 | with pytest.raises(KeyError): 34 | c.run(6) 35 | 36 | # Get value 37 | assert c.read_state("value") == 10 38 | assert c.read_state("DNE") is None 39 | 40 | 41 | def test_fit_error(): 42 | c = Counter() 43 | 44 | # Should raise error bc update op won't work 45 | with pytest.raises(RuntimeError): 46 | c.run("number", props={"value": [1]}, flush_update=True) 47 | -------------------------------------------------------------------------------- /tests/component/test_disabled_instance.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | import pytest 3 | import multiprocessing 4 | 5 | from motion.utils import get_redis_params 6 | import redis 7 | 8 | Counter = Component("Counter", cache_ttl=0) 9 | 10 | 11 | @Counter.init_state 12 | def setUp(): 13 | return {"value": 0} 14 | 15 | 16 | @Counter.serve("number") 17 | def noop(state, props): 18 | return state["value"], props["value"] 19 | 20 | 21 | @Counter.update("number") 22 | def increment(state, props): 23 | return {"value": state["value"] + props["value"]} 24 | 25 | 26 | # Create enabled component in a subprocess 27 | def counter_process(): 28 | c = Counter() 29 | assert c.run("number", props={"value": 1}) == (0, 1) 30 | 31 | 32 | def test_disabled(): 33 | # Create disabled component 34 | c = Counter(disable_update_task=True) 35 | with pytest.raises(RuntimeError): 36 | c.run("number", props={"value": 1}) 37 | 38 | # Make sure this can run successfully 39 | process = multiprocessing.Process(target=counter_process) 40 | process.start() 41 | process.join() 42 | 43 | 44 | def test_no_caching(): 45 | # Create component with no caching 46 | c = Counter("no_cache_test") 47 | assert c._cache_ttl == 0 48 | assert c._executor._cache_ttl == 0 49 | c.run("number", props={"value": 1}, flush_update=True) 50 | 51 | # Check that the result is not in the cache 52 | rp = get_redis_params() 53 | r = redis.Redis( 54 | host=rp.host, 55 | port=rp.port, 56 | password=rp.password, 57 | db=rp.db, 58 | ) 59 | # Define the prefix 60 | prefix = "MOTION_RESULT:Counter__no_cache_test/" 61 | 62 | # Initialize the count 63 | count = 0 64 | 65 | # Iterate over keys with the prefix 66 | for _ in r.scan_iter(f"{prefix}*"): 67 | count += 1 68 | 69 | # Check that there are no keys with the prefix 70 | assert count == 0 71 | -------------------------------------------------------------------------------- /tests/component/test_multiple_routes.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | import pytest 3 | 4 | Calculator = Component("Calculator") 5 | 6 | 7 | @Calculator.init_state 8 | def setUp(): 9 | return {"value": 0} 10 | 11 | 12 | @Calculator.serve("add") 13 | def plus(state, props): 14 | return state["value"] + props["value"] 15 | 16 | 17 | @Calculator.serve("subtract") 18 | def minus(state, props): 19 | return state["value"] - props["value"] 20 | 21 | 22 | @Calculator.update(["add", "subtract"]) 23 | def decrement(state, props): 24 | return {"value": props.serve_result} 25 | 26 | 27 | @Calculator.serve("identity") 28 | def noop(state, props): 29 | return props["value"] 30 | 31 | 32 | @Calculator.update("reset") 33 | def reset(state, props): 34 | return {"value": 0} 35 | 36 | 37 | def test_multiple_routes(): 38 | c = Calculator() 39 | assert c.run("add", props={"value": 1}, flush_update=True) == 1 40 | assert c.run("add", props={"value": 2}, flush_update=True) == 3 41 | assert c.run("subtract", props={"value": 1}, flush_update=True) == 2 42 | assert c.run("identity", props={"value": 1}) == 1 43 | 44 | # Force update doesn't do anything 45 | c.run("identity", props={"value": 1}, flush_update=True) 46 | 47 | c.run("reset", flush_update=True) 48 | assert c.run("add", props={"value": 1}, flush_update=True) == 1 49 | -------------------------------------------------------------------------------- /tests/component/test_params.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | import pytest 4 | 5 | c = Component("ComponentWithParams", params={"multiplier": 2}) 6 | 7 | 8 | @c.init_state 9 | def setUp(): 10 | return {"value": 0} 11 | 12 | 13 | @c.serve("add") 14 | def plus(state, props): 15 | return c.params["multiplier"] * (state["value"] + props["value"]) 16 | 17 | 18 | @c.update("add") 19 | def increment(state, props): 20 | return {"value": state["value"] + props["value"]} 21 | 22 | 23 | def test_params(): 24 | c_instance = c() 25 | assert c_instance.run("add", props={"value": 1}, flush_update=True) == 2 26 | assert c_instance.run("add", props={"value": 2}, flush_update=True) == 6 27 | 28 | 29 | cwp = Component("ComponentWithoutParams") 30 | 31 | 32 | @cwp.init_state 33 | def setUp2(): 34 | return {"value": 0} 35 | 36 | 37 | @cwp.serve("add") 38 | def plus2(state, props): 39 | return cwp.params["multiplier"] * (state["value"] + props["value"]) 40 | 41 | 42 | @cwp.update("add") 43 | def increment2(state, props): 44 | return {"value": state["value"] + props["value"]} 45 | 46 | 47 | def test_params_not_exist(): 48 | c_instance = cwp() 49 | with pytest.raises(KeyError): 50 | assert ( 51 | c_instance.run("add", props={"value": 1}, flush_update=True) == 2 52 | ) 53 | 54 | c_instance.shutdown() 55 | -------------------------------------------------------------------------------- /tests/component/test_process.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | import time 3 | import pytest 4 | 5 | Spinner = Component("Spinner") 6 | 7 | 8 | def fibonacci(n): 9 | if n <= 1: 10 | return n 11 | else: 12 | return fibonacci(n - 1) + fibonacci(n - 2) 13 | 14 | 15 | @Spinner.update("spin") 16 | def spin(state, props): 17 | fib = fibonacci(props["value"]) 18 | return {"fib": fib} 19 | 20 | 21 | def test_process(): 22 | num = 11 23 | res = 89 24 | 25 | rounds = 10 26 | 27 | # Commented this out because it takes a large number of rounds for process to be better than thread 28 | # inst1 = Spinner(instance_id="thread", update_task_type="thread") 29 | # start = time.time() 30 | 31 | # for i in range(rounds): 32 | # inst1.run( 33 | # "spin", 34 | # props={"value": num}, 35 | # flush_update=i == rounds - 1, 36 | # ignore_cache=True, 37 | # ) 38 | 39 | # thread_time = time.time() - start 40 | # assert inst1.read_state("fib") == res 41 | # inst1.shutdown() 42 | 43 | inst2 = Spinner(instance_id="process", update_task_type="process") 44 | start = time.time() 45 | 46 | for i in range(rounds): 47 | inst2.run( 48 | "spin", 49 | props={"value": num}, 50 | flush_update=i == rounds - 1, 51 | ignore_cache=True, 52 | ) 53 | 54 | process_time = time.time() - start 55 | assert inst2.read_state("fib") == res 56 | 57 | # assert thread_time > process_time 58 | 59 | inst2.shutdown() 60 | 61 | with pytest.raises(ValueError): 62 | Spinner(update_task_type="hehehe") 63 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import psutil 3 | import pytest 4 | import os 5 | import logging 6 | 7 | 8 | @pytest.fixture(scope="session", autouse=True) 9 | def redis_fixture(): 10 | """Set up redis as a pytest fixture.""" 11 | 12 | # Change dir to the root of tests 13 | os.chdir(os.path.dirname(os.path.abspath(__file__))) 14 | 15 | from motion import RedisParams 16 | from motion.utils import import_config 17 | 18 | import_config() 19 | 20 | rp = RedisParams() 21 | r = redis.Redis( 22 | host=rp.host, 23 | port=rp.port, 24 | password=rp.password, 25 | db=rp.db, 26 | ) 27 | assert os.getenv("MOTION_ENV") == "prod", "MOTION_ENV must be set to prod." 28 | 29 | try: 30 | r.ping() 31 | except redis.exceptions.ConnectionError: 32 | raise ConnectionError("Make sure you are running Redis on localhost:6381.") 33 | 34 | r.flushdb() 35 | assert len(r.keys()) == 0 36 | 37 | yield r 38 | 39 | # Delete any parquet files that were created 40 | home = os.path.expanduser("~") 41 | parquet_dir = f"{home}/.motion" 42 | 43 | # Delete any files in the parquet directory 44 | for filename in os.listdir(parquet_dir): 45 | file_path = os.path.join(parquet_dir, filename) 46 | # Delete the file 47 | os.remove(file_path) 48 | 49 | 50 | @pytest.hookimpl(tryfirst=True) 51 | def pytest_collection_modifyitems(config, items): 52 | # Print process count before test execution starts 53 | print_process_count() 54 | 55 | 56 | @pytest.hookimpl(trylast=True) 57 | def pytest_unconfigure(config): 58 | # Print process count after test execution ends 59 | print_process_count() 60 | 61 | 62 | def print_process_count(): 63 | count = sum(1 for _ in psutil.process_iter()) 64 | print(f"Number of processes: {count}") 65 | -------------------------------------------------------------------------------- /tests/parallel/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/tests/parallel/__init__.py -------------------------------------------------------------------------------- /tests/parallel/test_async.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | import asyncio 4 | import pytest 5 | 6 | Counter = Component("Counter") 7 | 8 | 9 | @Counter.init_state 10 | def setup(): 11 | return {"value": 1} 12 | 13 | 14 | @Counter.serve("multiply") 15 | async def noop(state, props): 16 | await asyncio.sleep(0.01) 17 | return state["value"] * props["value"] 18 | 19 | 20 | @Counter.serve("sync_multiply") 21 | def sync_noop(state, props): 22 | return state["value"] * props["value"] 23 | 24 | 25 | @Counter.update("multiply") 26 | async def increment(state, props): 27 | return {"value": state["value"] + 1} 28 | 29 | 30 | @pytest.mark.asyncio 31 | async def test_async_serve(): 32 | c = Counter() 33 | assert await c.arun("multiply", props={"value": 2}) == 2 34 | 35 | # Test that the user can't call run 36 | with pytest.raises(TypeError): 37 | c.run("multiply", props={"value": 2}, force_refresh=True) 38 | 39 | # Test that the user can call arun for regular functions 40 | result = await c.arun("sync_multiply", props={"value": 2}) 41 | assert result == 4 42 | 43 | 44 | @pytest.mark.asyncio 45 | async def test_async_update(): 46 | c = Counter(disable_update_task=True) 47 | 48 | await c.arun("multiply", props={"value": 2}, flush_update=True) 49 | assert c.read_state("value") == 2 50 | 51 | 52 | @pytest.mark.asyncio 53 | @pytest.mark.timeout(1) # This test should take less than 3 seconds 54 | async def test_gather(): 55 | c = Counter(disable_update_task=True) 56 | 57 | tasks = [ 58 | c.arun("multiply", props={"value": i}, flush_update=True) for i in range(100) 59 | ] 60 | # Run all tasks at the same time 61 | await asyncio.gather(*tasks) 62 | 63 | # Assert new state 64 | assert c.read_state("value") == 101 65 | -------------------------------------------------------------------------------- /tests/parallel/test_broken_fit.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | import pytest 4 | 5 | Counter = Component("Counter") 6 | 7 | 8 | @Counter.init_state 9 | def setup(): 10 | return {"value": 1} 11 | 12 | 13 | @Counter.serve("multiply") 14 | def noop(state, props): 15 | return state["value"] * props["value"] 16 | 17 | 18 | @Counter.update("multiply") 19 | def increment(state, props): 20 | print(state["does_not_exist"]) # This should break thread 21 | return {"value": state["value"] + 1} 22 | 23 | 24 | def test_release_lock_on_broken_update(): 25 | c = Counter("same_id") 26 | with pytest.raises(RuntimeError): 27 | c.run("multiply", props={"value": 2}, flush_update=True) 28 | c.shutdown() 29 | 30 | # Should be able to run again 31 | c2 = Counter("same_id") 32 | c2.run("multiply", props={"value": 2}) 33 | c2.shutdown() 34 | -------------------------------------------------------------------------------- /tests/parallel/test_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file tests the generator and async generator functionality. 3 | """ 4 | import pytest 5 | from motion import Component 6 | import random 7 | 8 | C = Component("GeneratorComponent") 9 | 10 | 11 | @C.init_state 12 | def setUp(): 13 | return {} 14 | 15 | 16 | @C.serve("identity") 17 | def identity(state, props): 18 | k = props["k"] 19 | 20 | # Randomly generate a list of k values 21 | values = [random.randint(0, 100) for _ in range(k)] 22 | 23 | # Return an iterator over the props values 24 | for v in values: 25 | yield v 26 | 27 | 28 | @C.update("identity") 29 | def assert_list(state, props): 30 | # Add serve result to state 31 | return {"sync_serve_result": props.serve_result} 32 | 33 | 34 | @C.serve("async_identity") 35 | async def async_identity(state, props): 36 | k = props["k"] 37 | 38 | # Randomly generate a list of k values 39 | values = [random.randint(0, 100) for _ in range(k)] 40 | 41 | # Return an iterator over the props values 42 | for v in values: 43 | yield v 44 | 45 | 46 | @C.update("async_identity") 47 | async def async_assert_list(state, props): 48 | # Add serve result to state 49 | return {"async_serve_result": props.serve_result} 50 | 51 | 52 | def test_regular_generator(): 53 | c = C("some_instance") 54 | 55 | serve_values = [] 56 | for v in c.gen("identity", props={"k": 3}, flush_update=True): 57 | serve_values.append(v) 58 | assert len(serve_values) == 3 59 | 60 | assert c.read_state("sync_serve_result") == serve_values 61 | c.shutdown() 62 | 63 | # Open the instance again and read the cached result 64 | c = C("some_instance") 65 | assert c.read_state("sync_serve_result") == serve_values 66 | 67 | new_serve_values = [] 68 | for v in c.gen("identity", props={"k": 3}): 69 | new_serve_values.append(v) 70 | assert new_serve_values == serve_values, "Cached result should be returned" 71 | 72 | c.shutdown() 73 | 74 | 75 | @pytest.mark.asyncio 76 | async def test_async_generator(): 77 | c = C("some_async_instance") 78 | 79 | serve_values = [] 80 | async for v in c.agen("async_identity", props={"k": 3}, flush_update=True): 81 | serve_values.append(v) 82 | assert len(serve_values) == 3 83 | 84 | assert c.read_state("async_serve_result") == serve_values 85 | c.shutdown() 86 | 87 | # Open the instance again and read the cached result 88 | c = C("some_async_instance") 89 | assert c.read_state("async_serve_result") == serve_values 90 | 91 | new_serve_values = [] 92 | async for v in c.agen("async_identity", props={"k": 3}): 93 | new_serve_values.append(v) 94 | assert new_serve_values == serve_values, "Cached result should be returned" 95 | 96 | c.shutdown() 97 | -------------------------------------------------------------------------------- /tests/parallel/test_many_keys.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file is used to test functionality of running a flow for many keys. 3 | """ 4 | from motion import Component 5 | 6 | import time 7 | 8 | Counter = Component("Counter") 9 | 10 | 11 | @Counter.init_state 12 | def setup(): 13 | return {"value": 1, "multifit": []} 14 | 15 | 16 | @Counter.serve(["increment", "decrement"]) 17 | def noop(state, props): 18 | return props["value"] 19 | 20 | 21 | @Counter.update("increment") 22 | def increment(state, props): 23 | return {"value": state["value"] + 1} 24 | 25 | 26 | @Counter.update("decrement") 27 | def nothing(state, props): 28 | return {"value": state["value"] - 1} 29 | 30 | 31 | @Counter.update(["accumulate", "something_else"]) 32 | def multiupdate(state, props): 33 | return {"multifit": state["multifit"] + [props["value"]]} 34 | 35 | 36 | def test_many_keys(): 37 | c = Counter() 38 | 39 | c.run("increment", props={"value": 1}, flush_update=True) 40 | assert c.read_state("value") == 2 41 | c.run("decrement", props={"value": 1}, flush_update=True) 42 | assert c.read_state("value") == 1 43 | 44 | # Test multifit 45 | c.run("accumulate", props={"value": 1}) 46 | c.run("something_else", props={"value": 2}) 47 | 48 | c.flush_update("accumulate") 49 | c.flush_update("something_else") 50 | 51 | assert c.read_state("multifit") == [1, 2] 52 | -------------------------------------------------------------------------------- /tests/parallel/test_two_instance.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | Counter = Component("Counter") 4 | 5 | 6 | @Counter.init_state 7 | def setup(): 8 | return {"value": 1} 9 | 10 | 11 | @Counter.serve("multiply") 12 | def noop(state, props): 13 | return state["value"] * props["value"] 14 | 15 | 16 | @Counter.update("multiply") 17 | def increment(state, props): 18 | return {"value": state["value"] + 1} 19 | 20 | 21 | def test_redis_saving(): 22 | inst1 = Counter(instance_id="test") 23 | assert inst1.run("multiply", props={"value": 2}, flush_update=True) == 2 24 | assert inst1.read_state("value") == 2 25 | inst1.shutdown() 26 | 27 | print("Starting second instance") 28 | 29 | inst2 = Counter(instance_id="test") 30 | assert inst2.read_state("value") == 2 31 | assert inst2.run("multiply", props={"value": 3}, flush_update=True) == 6 32 | assert inst2.read_state("value") == 3 33 | inst2.shutdown() 34 | -------------------------------------------------------------------------------- /tests/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/tests/server/__init__.py -------------------------------------------------------------------------------- /tests/server/test_dashboard.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from fastapi.testclient import TestClient 3 | 4 | from motion.dashboard import dashboard_app 5 | 6 | 7 | @pytest.fixture 8 | def client(): 9 | # Create application 10 | 11 | return TestClient(dashboard_app) 12 | 13 | 14 | def test_dashboard(client): 15 | # make sure a get request to / returns a 200 16 | response = client.get("/") 17 | assert response.status_code == 200 18 | -------------------------------------------------------------------------------- /tests/server/test_simple_server.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from fastapi.testclient import TestClient 3 | 4 | from motion import Component 5 | from motion import Application 6 | 7 | # Create some components 8 | 9 | Counter = Component("Counter") 10 | 11 | 12 | @Counter.init_state 13 | def setup(): 14 | return {"multiplier": 1} 15 | 16 | 17 | @Counter.serve("sum") 18 | def compute_sum(state, props): 19 | return sum(props["values"]) * state["multiplier"] 20 | 21 | 22 | @Counter.update("sum") 23 | def update_sum(state, props): 24 | return {"multiplier": state["multiplier"] + 1} 25 | 26 | 27 | @pytest.fixture 28 | def client(): 29 | # Create application 30 | motion_app = Application(components=[Counter]) 31 | credentials = motion_app.get_credentials() 32 | app = motion_app.get_app() 33 | 34 | return credentials, TestClient(app) 35 | 36 | 37 | def test_endpoint(client): 38 | credentials, app_client = client # Unpack 39 | 40 | # Get a token for the instance id 41 | instance_id = "testid" 42 | response = app_client.post( 43 | "/auth", 44 | json={ 45 | "instance_id": instance_id, 46 | }, 47 | headers={"X-Api-Key": credentials["api_key"]}, 48 | ) 49 | assert response.status_code == 200 50 | jwt_token = response.json()["token"] 51 | 52 | # Test the run endpoint 53 | response = app_client.post( 54 | "/Counter", 55 | json={ 56 | "instance_id": instance_id, 57 | "flow_key": "sum", 58 | "is_async": False, 59 | "props": {"values": [1, 2, 3]}, 60 | }, 61 | headers={ 62 | "Authorization": f"Bearer {jwt_token}", 63 | "X-Api-Key": credentials["api_key"], 64 | }, 65 | ) 66 | 67 | assert response.status_code == 200 68 | assert response.json() == 6 69 | 70 | # Read the state and check that it was updated 71 | response = app_client.get( 72 | "/Counter/read", 73 | params={ 74 | "instance_id": instance_id, 75 | "key": "multiplier", 76 | }, 77 | headers={ 78 | "Authorization": f"Bearer {jwt_token}", 79 | "X-Api-Key": credentials["api_key"], 80 | }, 81 | ) 82 | 83 | assert response.status_code == 200 84 | assert response.json() == {"multiplier": 2} 85 | -------------------------------------------------------------------------------- /tests/state/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/tests/state/__init__.py -------------------------------------------------------------------------------- /tests/state/test_dataframe.py: -------------------------------------------------------------------------------- 1 | from motion import Component, MDataFrame 2 | 3 | C = Component("MyDFComponent") 4 | 5 | 6 | @C.init_state 7 | def setUp(): 8 | return {"df": MDataFrame({"value": [0, 1, 2]})} 9 | 10 | 11 | def test_read_and_write_to_df(): 12 | c_instance = C() 13 | df = c_instance.read_state("df") 14 | assert df.to_dict(orient="list") == {"value": [0, 1, 2]} 15 | 16 | # Add a column 17 | df["value2"] = [3, 4, 5] 18 | c_instance.write_state({"df": df}) 19 | 20 | # Check that the column was added 21 | df_2 = c_instance.read_state("df") 22 | assert df_2.to_dict(orient="list") == { 23 | "value": [0, 1, 2], 24 | "value2": [3, 4, 5], 25 | } 26 | -------------------------------------------------------------------------------- /tests/state/test_db_conn.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | import sqlite3 4 | 5 | c = Component("DBComponent") 6 | 7 | 8 | @c.init_state 9 | def setUp(): 10 | # Create in-memory sqlite database 11 | conn = sqlite3.connect(":memory:") 12 | cursor = conn.cursor() 13 | cursor.execute( 14 | """CREATE TABLE IF NOT EXISTS users 15 | (id INTEGER PRIMARY KEY AUTOINCREMENT, 16 | name TEXT, 17 | age INTEGER)""" 18 | ) 19 | 20 | cursor.execute( 21 | "INSERT INTO users (name, age) VALUES (?, ?)", ("John Doe", 25) 22 | ) 23 | cursor.execute( 24 | "INSERT INTO users (name, age) VALUES (?, ?)", ("Jane Smith", 30) 25 | ) 26 | conn.commit() 27 | 28 | return {"cursor": cursor, "fit_count": 0} 29 | 30 | 31 | @c.save_state 32 | def save(state): 33 | return {"fit_count": state["fit_count"]} 34 | 35 | 36 | @c.load_state 37 | def load(state): 38 | conn = sqlite3.connect(":memory:") 39 | cursor = conn.cursor() 40 | return {"cursor": cursor, "fit_count": state["fit_count"]} 41 | 42 | 43 | @c.serve("count") 44 | def execute_fn(state, props): 45 | return state["cursor"].execute("SELECT COUNT(*) FROM users").fetchall() 46 | 47 | 48 | @c.serve("something") 49 | def noop(state, props): 50 | return state["fit_count"] 51 | 52 | 53 | @c.update("something") 54 | def increment(state, props): 55 | return {"fit_count": state["fit_count"] + 1} 56 | 57 | 58 | def test_db_component(): 59 | c_instance = c() 60 | assert c_instance.run("count", props={"value": 1}) == [(2,)] 61 | c_instance.run("something", props={"value": 1}, flush_update=True) 62 | assert c_instance.run("something", props={"value": 5}) == 1 63 | -------------------------------------------------------------------------------- /tests/state/test_flush_fit.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | Counter = Component("Counter") 4 | 5 | 6 | @Counter.init_state 7 | def setUp(): 8 | return {"value": 0, "values": []} 9 | 10 | 11 | @Counter.serve("number") 12 | def noop(state, props): 13 | return props["value"] 14 | 15 | 16 | # Arbitrarily large batch 17 | @Counter.update("number") 18 | def increment(state, props): 19 | values = state["values"] 20 | values.append(props.serve_result) 21 | 22 | if len(values) == 10: 23 | return {"values": [], "value": sum(values) + state["value"]} 24 | 25 | return {"values": values} 26 | 27 | 28 | def test_flush_instance(): 29 | counter = Counter() 30 | 31 | init_value = counter.read_state("value") 32 | 33 | for i in range(10): 34 | counter.run("number", props={"value": i}) 35 | 36 | # Flush instance 37 | counter.flush_update("number") 38 | 39 | # Assert new state is different from old state 40 | assert counter.get_version() > 1 41 | assert counter.read_state("value") != init_value 42 | assert counter.read_state("value") == sum(range(10)) 43 | 44 | # If I flush again, nothing should happen since 45 | # there are no elements in the update queue 46 | counter.flush_update("number") 47 | assert counter.get_version() > 1 48 | 49 | counter.shutdown() 50 | 51 | 52 | def test_fit_daemon(): 53 | counter = Counter() 54 | 55 | for i in range(10): 56 | counter.run("number", props={"value": i}) 57 | 58 | # Flush instance 59 | counter.flush_update("number") 60 | 61 | # Assert new state is different from old state 62 | assert counter.get_version() > 1 63 | 64 | # Don't shutdown 65 | -------------------------------------------------------------------------------- /tests/state/test_instance_cli.py: -------------------------------------------------------------------------------- 1 | from motion import Component, clear_instance, inspect_state, get_instances 2 | 3 | import pytest 4 | import os 5 | 6 | C = Component("MyComponent") 7 | 8 | 9 | @C.init_state 10 | def setUp(): 11 | return {"value": 0} 12 | 13 | 14 | def test_instance_clear(): 15 | c_instance = C() 16 | instance_name = c_instance.instance_name 17 | assert c_instance.read_state("value") == 0 18 | assert c_instance.get_version() == 1 19 | c_instance.shutdown() 20 | 21 | # Clear instance 22 | cleared = clear_instance(instance_name) 23 | assert cleared 24 | 25 | # Assert new state is different from old state 26 | new_instance = C(instance_name.strip("__")[1]) 27 | assert new_instance.read_state("value") == 0 28 | assert new_instance.get_version() == 1 29 | 30 | # Make sure there are no cached resuts 31 | assert ( 32 | len(new_instance._executor._redis_con.keys(f"MOTION_RESULT:{instance_name}")) 33 | == 0 34 | ) 35 | 36 | new_instance.shutdown() 37 | 38 | # Clear something that doesn't exist 39 | cleared = clear_instance("DoesNotExist__somename") 40 | assert not cleared 41 | 42 | # Clear something of the wrong type 43 | with pytest.raises(ValueError): 44 | clear_instance("DoesNotExist") 45 | 46 | 47 | def test_instance_inspect(): 48 | c_instance = C() 49 | instance_name = c_instance.instance_name 50 | 51 | # Inspect instance 52 | state = inspect_state(instance_name) 53 | 54 | assert state == {"value": 0} 55 | 56 | 57 | def test_list_instances(): 58 | instance_ids = get_instances(C.name) 59 | 60 | assert len(instance_ids) >= 1 61 | -------------------------------------------------------------------------------- /tests/state/test_model.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | 4 | from sklearn.datasets import make_regression 5 | from sklearn.linear_model import LinearRegression 6 | 7 | c = Component("ModelComponent") 8 | 9 | 10 | @c.init_state 11 | def setUp(): 12 | # Generate a sample dataset for training 13 | X, y = make_regression(n_samples=100, n_features=1, noise=0.1) 14 | 15 | # Train a linear regression model on the sample dataset 16 | model = LinearRegression() 17 | model.fit(X, y) 18 | 19 | return {"model": model, "training_batch": []} 20 | 21 | 22 | @c.serve("value") 23 | def predict(state, props): 24 | return state["model"].predict([[props["value"]]])[0] 25 | 26 | 27 | @c.update("value") 28 | def finetune(state, props): 29 | training_batch = state["training_batch"] 30 | training_batch.append((props["value"], props.serve_result)) 31 | if len(training_batch) < 2: 32 | return {"training_batch": training_batch} 33 | 34 | # Perform training on the batch of data 35 | # Example training logic: 36 | model = state["model"] 37 | X = [[v[0]] for v in training_batch] 38 | y = [r[1] + 0.02 for r in training_batch] 39 | model.fit(X, y) 40 | 41 | # Return updated state if needed 42 | return {"model": model} 43 | 44 | 45 | def test_model_component(): 46 | c_instance = c() 47 | first_run = c_instance.run("value", props={"value": 1}, flush_update=True) 48 | assert first_run == c_instance.run("value", props={"value": 1}) 49 | 50 | second_run = c_instance.run("value", props={"value": 1}, force_refresh=True) 51 | 52 | # The model should have been updated 53 | assert second_run != first_run 54 | 55 | 56 | def test_ignore_cache(): 57 | c_instance = c() 58 | first_run = c_instance.run("value", props={"value": 1}, flush_update=True) 59 | assert first_run == c_instance.run("value", props={"value": 1}, flush_update=True) 60 | 61 | second_run = c_instance.run("value", props={"value": 1}, ignore_cache=True) 62 | 63 | # The model should have been updated 64 | assert second_run != first_run 65 | -------------------------------------------------------------------------------- /tests/state/test_mtable_integration.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file stores an MTable in a Motion component. 3 | """ 4 | 5 | from motion import Component, MTable 6 | import pandas as pd 7 | import pyarrow as pa 8 | import numpy as np 9 | 10 | C = Component("MyMTableComponent") 11 | 12 | 13 | @C.init_state 14 | def setUp(): 15 | df = pd.DataFrame( 16 | { 17 | "vector": [ 18 | np.array([1.0, 2.0], dtype=np.float64), 19 | np.array([2.0, 3.0], dtype=np.float64), 20 | np.array([3.0, 4.0], dtype=np.float64), 21 | ], 22 | "label": ["a", "b", "c"], 23 | } 24 | ) 25 | table = MTable.from_pandas(df) 26 | return {"data": table} 27 | 28 | 29 | D = Component("MyMTableComponent2") 30 | 31 | 32 | @D.init_state 33 | def setUp(): 34 | df = pd.DataFrame({"value": [0, 1, 2]}) 35 | table = MTable.from_pandas(df) 36 | 37 | # Set the filesystem 38 | fs = pa.fs.LocalFileSystem() 39 | table.filesystem = fs 40 | return {"data": table} 41 | 42 | 43 | @C.serve("search") 44 | def search(state, props): 45 | # Do nearest neighbor search 46 | vector = props["vector"] 47 | table = state["data"] 48 | result = table.knn("vector", vector, 2, "euclidean") 49 | return result 50 | 51 | 52 | def test_read_and_write_data(): 53 | c_instance = C() 54 | pyarrow_table = c_instance.read_state("data") 55 | assert pyarrow_table.data.to_pandas()["label"].tolist() == ["a", "b", "c"] 56 | 57 | # Add a column 58 | pyarrow_table.append_column("label2", pa.array(["d", "e", "f"])) 59 | c_instance.write_state({"data": pyarrow_table}) 60 | 61 | # Check that the column was added 62 | pyarrow_table_2 = c_instance.read_state("data") 63 | assert pyarrow_table_2.data.to_pandas()[["label", "label2"]].to_dict( 64 | orient="list" 65 | ) == { 66 | "label": ["a", "b", "c"], 67 | "label2": ["d", "e", "f"], 68 | } 69 | 70 | 71 | def test_search(): 72 | c_instance = C() 73 | 74 | query_vector = np.array([1.0, 2.0], dtype=np.float64) 75 | result = c_instance.run("search", props={"vector": query_vector}) 76 | labels = result.to_pandas()["label"].tolist() 77 | assert sorted(labels) == ["a", "b"] 78 | 79 | 80 | def test_filesystem_write(): 81 | d_instance = D("instance_1") 82 | # Assert we can close the instance and reread the state 83 | 84 | d_instance.shutdown() 85 | 86 | d_instance = D("instance_1") 87 | pyarrow_table = d_instance.read_state("data") 88 | 89 | assert pyarrow_table.data.to_pandas()["value"].tolist() == [0, 1, 2] 90 | -------------------------------------------------------------------------------- /tests/state/test_mtable_unit.py: -------------------------------------------------------------------------------- 1 | # import pytest 2 | import pyarrow as pa 3 | import pandas as pd 4 | import numpy as np 5 | from motion import MTable 6 | 7 | 8 | class TestMTable: 9 | def test_create_from_pandas(self): 10 | df = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) 11 | table = MTable.from_pandas(df) 12 | assert table.data.num_rows == 3 13 | assert table.data.num_columns == 2 14 | 15 | def test_create_from_arrow(self): 16 | arrow_table = pa.Table.from_pandas( 17 | pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) 18 | ) 19 | table = MTable.from_arrow(arrow_table) 20 | assert table.data.num_rows == 3 21 | assert table.data.num_columns == 2 22 | 23 | def test_add_row(self): 24 | table = MTable.from_schema( 25 | pa.schema([pa.field("a", pa.int32()), pa.field("b", pa.string())]) 26 | ) 27 | table.add_row({"a": 1, "b": "x"}) 28 | assert table.data.num_rows == 1 29 | 30 | def test_remove_row(self): 31 | df = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) 32 | table = MTable.from_pandas(df) 33 | table.data = table.remove_row(1) 34 | assert table.data.num_rows == 2 35 | 36 | def test_add_and_remove_column(self): 37 | df = pd.DataFrame({"a": [1, 2, 3]}) 38 | table = MTable.from_pandas(df) 39 | table.append_column("b", pa.array(["x", "y", "z"])) 40 | assert table.data.num_columns == 2 41 | table.remove_column_by_name("b") 42 | assert table.data.num_columns == 1 43 | 44 | def test_knn(self): 45 | # Modified test with vectors of type float64 46 | df = pd.DataFrame( 47 | { 48 | "vector": [ 49 | np.array([1.0, 2.0], dtype=np.float64), 50 | np.array([2.0, 3.0], dtype=np.float64), 51 | np.array([3.0, 4.0], dtype=np.float64), 52 | ], 53 | "label": ["a", "b", "c"], 54 | } 55 | ) 56 | table = MTable.from_pandas(df) 57 | print(table.data) 58 | 59 | result = table.knn( 60 | "vector", np.array([1.0, 2.0], dtype=np.float64), 2, "euclidean" 61 | ) 62 | assert result.num_rows == 2 63 | assert "distances" in result.column_names 64 | 65 | def test_apply_distance(self): 66 | # Modified test with vectors of type float64 67 | df = pd.DataFrame( 68 | { 69 | "vector": [ 70 | np.array([1.0, 2.0], dtype=np.float64), 71 | np.array([2.0, 3.0], dtype=np.float64), 72 | np.array([3.0, 4.0], dtype=np.float64), 73 | ], 74 | "label": ["a", "b", "c"], 75 | } 76 | ) 77 | table = MTable.from_pandas(df) 78 | 79 | # Apply the distance calculation 80 | query_point = np.array([1.0, 2.0], dtype=np.float64) 81 | metric = "euclidean" # or any other metric your function supports 82 | result_table = table.apply_distance("vector", query_point, metric) 83 | 84 | # Check if the distances column is added 85 | assert "distances" in result_table.schema.names 86 | 87 | # Check the number of rows remains the same 88 | assert result_table.num_rows == table.data.num_rows 89 | -------------------------------------------------------------------------------- /tests/state/test_write.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | 3 | import pytest 4 | 5 | C = Component("MyComponent") 6 | 7 | 8 | @C.init_state 9 | def setUp(): 10 | return {"value": 0} 11 | 12 | 13 | @C.serve("my_key") 14 | def serve(state, props): 15 | return state.instance_id 16 | 17 | 18 | def test_write_state(): 19 | c_instance = C() 20 | assert c_instance.read_state("value") == 0 21 | c_instance.write_state({"value": 1, "value2": 2}) 22 | assert c_instance.read_state("value") == 1 23 | assert c_instance.read_state("value2") == 2 24 | 25 | # Test something that should fail 26 | with pytest.raises(TypeError): 27 | c_instance.write_state(1) 28 | 29 | with pytest.raises(TypeError): 30 | c_instance.write_state("Hello") 31 | 32 | # Should do nothing 33 | c_instance.write_state({}) 34 | 35 | 36 | def test_read_instance_id(): 37 | c_instance = C("some_id") 38 | assert c_instance.run("my_key", ignore_cache=True) == "some_id" 39 | -------------------------------------------------------------------------------- /tests/test_dev_box.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import pytest 3 | import os 4 | import redis 5 | 6 | from motion.utils import RedisParams 7 | 8 | 9 | def test_dev_instance_cleanup(): 10 | code_to_execute = """ 11 | from motion import Component 12 | 13 | testc = Component("testc") 14 | 15 | @testc.init_state 16 | def setup(): 17 | return {"value": 1} 18 | 19 | @testc.serve("multiply") 20 | def noop(state, props): 21 | return state["value"] * props["value"] 22 | 23 | if __name__ == "__main__": 24 | i = testc("hello") 25 | res = i.run("multiply", props={"value": 2}, ignore_cache=True) 26 | """ 27 | 28 | # Copy env vars from the current process 29 | env = os.environ.copy() 30 | 31 | # Set the MOTION_ENV variable to dev 32 | env["MOTION_ENV"] = "dev" 33 | 34 | # Run the code in a separate Python process 35 | subprocess.run(["python", "-c", code_to_execute], env=env) 36 | 37 | # Check that "hello" instance does not exist 38 | rp = RedisParams() 39 | r = redis.Redis( 40 | host=rp.host, 41 | port=rp.port, 42 | password=rp.password, 43 | db=rp.db, 44 | ) 45 | assert r.get("MOTION_VERSION:DEV:testc__hello") is None, "Instance was not deleted." 46 | -------------------------------------------------------------------------------- /tests/test_expire_policies.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file tests the functionality to set discard policies for update queues. 3 | We will test two discard policies: NUM_NEW_UPDATES and SECONDS. 4 | We will test them in both sync and async functions. 5 | """ 6 | from motion import Component, DiscardPolicy 7 | import time 8 | import asyncio 9 | import pytest 10 | 11 | C = Component("C") 12 | 13 | 14 | @C.init_state 15 | def setup(): 16 | return {"num_new_update_value": 0, "regular_value": 0, "seconds_value": 0} 17 | 18 | 19 | @C.update("sum", discard_after=10, discard_policy=DiscardPolicy.NUM_NEW_UPDATES) 20 | def update_sum_num_new(state, props): 21 | # Sleep for a bit so there's a real bottleneck in the update queue 22 | time.sleep(0.02) 23 | return {"num_new_update_value": state["num_new_update_value"] + props["value"]} 24 | 25 | 26 | @C.update("sum") 27 | def update_sum_default(state, props): 28 | return {"regular_value": state["regular_value"] + props["value"]} 29 | 30 | 31 | @C.update("sum", discard_after=1, discard_policy=DiscardPolicy.SECONDS) 32 | def update_sum_seconds(state, props): 33 | time.sleep(0.05) 34 | return {"seconds_value": state["seconds_value"] + props["value"]} 35 | 36 | 37 | @C.update("asum", discard_after=10, discard_policy=DiscardPolicy.NUM_NEW_UPDATES) 38 | async def aupdate_sum_num_new(state, props): 39 | # Sleep for a bit so there's a real bottleneck in the update queue 40 | await asyncio.sleep(0.01) 41 | return {"num_new_update_value": state["num_new_update_value"] + props["value"]} 42 | 43 | 44 | @C.update("asum") 45 | async def aupdate_sum_default(state, props): 46 | return {"regular_value": state["regular_value"] + props["value"]} 47 | 48 | 49 | @C.update("asum", discard_after=1, discard_policy=DiscardPolicy.SECONDS) 50 | async def aupdate_sum_seconds(state, props): 51 | await asyncio.sleep(0.05) 52 | return {"seconds_value": state["seconds_value"] + props["value"]} 53 | 54 | 55 | def test_sync_num_new_updates(): 56 | c = C() 57 | 58 | for i in range(50): 59 | c.run("sum", props={"value": i}) 60 | 61 | # Flush instance 62 | c.flush_update("sum") 63 | 64 | # Assert new state is different from old state 65 | assert c.get_version() > 1 66 | assert c.read_state("num_new_update_value") != 0 67 | assert c.read_state("seconds_value") != 0 68 | 69 | # Assert that the num_new_update_value is not sum of 1..50 because there was the discard policy 70 | assert c.read_state("num_new_update_value") != sum(range(50)) 71 | assert c.read_state("seconds_value") != sum(range(50)) 72 | 73 | # Assert that regular_value is sum of 1..50 because there was no discard policy 74 | assert c.read_state("regular_value") == sum(range(50)) 75 | 76 | c.shutdown() 77 | 78 | 79 | @pytest.mark.asyncio 80 | async def test_async_num_new_updates(): 81 | c = C() 82 | 83 | async_tasks = [] 84 | for i in range(50): 85 | async_tasks.append(c.arun("asum", props={"value": i})) 86 | 87 | await asyncio.gather(*async_tasks) 88 | 89 | # Flush instance 90 | c.flush_update("asum") 91 | 92 | # Assert new state is different from old state 93 | assert c.get_version() > 1 94 | assert c.read_state("num_new_update_value") != 0 95 | assert c.read_state("seconds_value") != 0 96 | 97 | # Assert that the num_new_update_value is not sum of 1..50 because there was the discard policy 98 | assert c.read_state("num_new_update_value") != sum(range(50)) 99 | assert c.read_state("seconds_value") != sum(range(50)) 100 | 101 | # Assert that regular_value is sum of 1..50 because there was no discard policy 102 | assert c.read_state("regular_value") == sum(range(50)) 103 | 104 | c.shutdown() 105 | -------------------------------------------------------------------------------- /tests/test_fastapi.py: -------------------------------------------------------------------------------- 1 | """This creates a mock fastapi instance and a motion 2 | component instance within the fastapi endpoint. 3 | 4 | Using a web app framework like fastapi runs motion 5 | in some thread that isn't the main thread of the 6 | main interpreter, so this is a good test to have. 7 | """ 8 | from fastapi import FastAPI 9 | from fastapi.testclient import TestClient 10 | import pytest 11 | 12 | from motion import Component 13 | 14 | Test = Component("Test") 15 | 16 | 17 | @Test.init_state 18 | def setup(): 19 | return {"count": 0} 20 | 21 | 22 | @Test.serve("noop") 23 | async def noop(state, props): 24 | return props["value"] 25 | 26 | 27 | @Test.update("increment") 28 | def noopupdate(state, props): 29 | return {"count": state["count"] + 1} 30 | 31 | 32 | app = FastAPI() 33 | 34 | 35 | @app.get("/sync_endpoint") 36 | def read_endpoint(): 37 | # Create some instance of a component 38 | t = Test("testid") 39 | t.run("increment") 40 | t.flush_update("increment") 41 | 42 | return {"value": t.read_state("count")} 43 | 44 | 45 | @app.get("/async_endpoint") 46 | async def read_noop(): 47 | t = Test("testid") 48 | return {"value": await t.arun("noop", props={"value": 1})} 49 | 50 | 51 | @pytest.fixture 52 | def client(): 53 | return TestClient(app) 54 | 55 | 56 | def test_endpoint(client): 57 | response = client.get("/sync_endpoint") 58 | assert response.status_code == 200 59 | assert response.json() == {"value": 1} 60 | 61 | # Do it again! 62 | response = client.get("/sync_endpoint") 63 | assert response.status_code == 200 64 | assert response.json() == {"value": 2} 65 | 66 | # Try the async endpoint 67 | response = client.get("/async_endpoint") 68 | assert response.status_code == 200 69 | assert response.json() == {"value": 1} 70 | -------------------------------------------------------------------------------- /tests/test_migrate.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file tests the state migration functionality of motion. 3 | """ 4 | 5 | from motion import Component, StateMigrator 6 | 7 | import pytest 8 | 9 | Something = Component("Something") 10 | 11 | 12 | @Something.init_state 13 | def setup(): 14 | return {"state_val": 0} 15 | 16 | 17 | def bad_migrator(state, something_else): 18 | return state 19 | 20 | 21 | def migrator_not_returning_dict(state): 22 | return "this isn't a dict" 23 | 24 | 25 | def good_migrator(state): 26 | state.update({"another_val": 0}) 27 | return state 28 | 29 | 30 | def test_state_migration(): 31 | # Create a bunch of instances 32 | instance_ids = [] 33 | for _ in range(10): 34 | s = Something() 35 | instance_ids.append(s.instance_id) 36 | 37 | # Run bad migrators 38 | with pytest.raises(TypeError): 39 | sm = StateMigrator("helloworld", bad_migrator) 40 | 41 | with pytest.raises(AssertionError): 42 | sm = StateMigrator(Something, migrator_not_returning_dict) 43 | sm.migrate() 44 | 45 | # Run good migrator 46 | sm = StateMigrator(Something, good_migrator) 47 | result = sm.migrate([instance_ids[0]]) 48 | assert len(result) == 1 49 | assert result[0].instance_id == instance_ids[0] 50 | assert result[0].exception is None 51 | 52 | # Run good migrator on all instances 53 | results = sm.migrate() 54 | assert len(results) == 10 55 | for result in results: 56 | assert result.instance_id in instance_ids 57 | assert result.exception is None 58 | 59 | # Assert the instances have the new state 60 | for instance_id in instance_ids: 61 | s = Something(instance_id) 62 | # Load state 63 | s._executor._loadState() 64 | assert s._executor._state == {"state_val": 0, "another_val": 0} 65 | -------------------------------------------------------------------------------- /tests/test_simple_pipeline.py: -------------------------------------------------------------------------------- 1 | from motion import Component 2 | from motion.dashboard_utils import get_component_instance_usage, get_component_usage 3 | import os 4 | 5 | # Test a pipeline with multiple components 6 | a = Component("ComponentA") 7 | 8 | 9 | @a.init_state 10 | def setUp(): 11 | return {"value": 0} 12 | 13 | 14 | @a.serve("add") 15 | def plus(state, props): 16 | props["something_else"] = "Hello" 17 | return state["value"] + props["value"] 18 | 19 | 20 | @a.update("add") 21 | def increment(state, props): 22 | assert "something_else" in props 23 | return {"value": state["value"] + props["value"]} 24 | 25 | 26 | b = Component("ComponentB") 27 | 28 | 29 | @b.init_state 30 | def setUp(): 31 | return {"message": ""} 32 | 33 | 34 | @b.serve("concat") 35 | def concat_message(state, props): 36 | return state["message"] + " " + props["str_to_concat"] 37 | 38 | 39 | @b.update("concat") 40 | def update_message(state, props): 41 | return {"message": state["message"] + " " + props["str_to_concat"]} 42 | 43 | 44 | def test_simple_pipeline(): 45 | a_instance = a("my_instance_a") 46 | b_instance = b("my_instance_b") 47 | add_result = a_instance.run("add", props={"value": 1}, flush_update=True) 48 | assert add_result == 1 49 | 50 | concat_result = b_instance.run( 51 | "concat", props={"str_to_concat": str(add_result)}, flush_update=True 52 | ) 53 | assert concat_result == " 1" 54 | 55 | add_result_2 = a_instance.run("add", props={"value": 2}) 56 | assert add_result_2 == 3 57 | concat_result_2 = b_instance.run( 58 | "concat", props={"str_to_concat": str(add_result_2)} 59 | ) 60 | assert concat_result_2 == " 1 3" 61 | 62 | # Check that the logs show results 63 | a_instance.shutdown(wait_for_logging_threads=True) 64 | b_instance.shutdown(wait_for_logging_threads=True) 65 | 66 | component_usage = get_component_usage("ComponentA") 67 | assert component_usage.keys() == { 68 | "numInstances", 69 | "instanceIds", 70 | "flowCounts", 71 | "statusCounts", 72 | "statusChanges", 73 | "statusBarData", 74 | "fractionUptime", 75 | } 76 | 77 | # Assert that flowCounts statusCounts statusBarData fractionUptime are not empty 78 | # TODO: make this pass 79 | # assert len(component_usage["flowCounts"]) > 0 80 | # assert len(component_usage["statusCounts"]) > 0 81 | # assert len(component_usage["statusBarData"]) > 0 82 | # assert component_usage["fractionUptime"] is not None 83 | 84 | usage = get_component_instance_usage("ComponentA", "my_instance_a") 85 | assert usage.keys() == {"version", "flowCounts", "statusBarData", "fractionUptime"} 86 | 87 | # Assert that the flowCounts are not empty 88 | # TODO: make this pass 89 | assert usage["version"] > 0 90 | # assert len(usage["flowCounts"]) > 0 91 | # assert len(usage["statusBarData"]) > 0 92 | # assert usage["fractionUptime"] is not None 93 | 94 | 95 | def test_without_victoriametrics(): 96 | # No victoriametrics 97 | old_url = os.environ["MOTION_VICTORIAMETRICS_URL"] 98 | del os.environ["MOTION_VICTORIAMETRICS_URL"] 99 | 100 | a_instance = a("my_instance_a_no_vm") 101 | add_result = a_instance.run("add", props={"value": 1}, flush_update=True) 102 | assert add_result == 1 103 | 104 | add_result_2 = a_instance.run("add", props={"value": 2}) 105 | assert add_result_2 == 3 106 | 107 | # Check that the logs for this instance are empty 108 | a_instance.shutdown(wait_for_logging_threads=True) 109 | usage = get_component_instance_usage("ComponentA", "my_instance_a_no_vm") 110 | assert usage.keys() == {"version", "flowCounts", "statusBarData", "fractionUptime"} 111 | 112 | # Assert that the flowCounts are not empty 113 | assert usage["version"] > 0 114 | assert len(usage["flowCounts"]) == 0 115 | assert len(usage["statusBarData"]) == 0 116 | assert usage["fractionUptime"] is None 117 | 118 | os.environ["MOTION_VICTORIAMETRICS_URL"] = old_url 119 | -------------------------------------------------------------------------------- /ui/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /ui/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Create React App 2 | 3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 4 | 5 | ## Available Scripts 6 | 7 | In the project directory, you can run: 8 | 9 | ### `npm start` 10 | 11 | Runs the app in the development mode.\ 12 | Open [http://localhost:3000](http://localhost:3000) to view it in your browser. 13 | 14 | The page will reload when you make changes.\ 15 | You may also see any lint errors in the console. 16 | 17 | ### `npm test` 18 | 19 | Launches the test runner in the interactive watch mode.\ 20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 21 | 22 | ### `npm run build` 23 | 24 | Builds the app for production to the `build` folder.\ 25 | It correctly bundles React in production mode and optimizes the build for the best performance. 26 | 27 | The build is minified and the filenames include the hashes.\ 28 | Your app is ready to be deployed! 29 | 30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 31 | 32 | ### `npm run eject` 33 | 34 | **Note: this is a one-way operation. Once you `eject`, you can't go back!** 35 | 36 | If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 37 | 38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own. 39 | 40 | You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it. 41 | 42 | ## Learn More 43 | 44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 45 | 46 | To learn React, check out the [React documentation](https://reactjs.org/). 47 | 48 | ### Code Splitting 49 | 50 | This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting) 51 | 52 | ### Analyzing the Bundle Size 53 | 54 | This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size) 55 | 56 | ### Making a Progressive Web App 57 | 58 | This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app) 59 | 60 | ### Advanced Configuration 61 | 62 | This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration) 63 | 64 | ### Deployment 65 | 66 | This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment) 67 | 68 | ### `npm run build` fails to minify 69 | 70 | This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify) 71 | -------------------------------------------------------------------------------- /ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ui", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@emotion/react": "^11.11.3", 7 | "@emotion/styled": "^11.11.0", 8 | "@fontsource/inter": "^5.0.16", 9 | "@mui/icons-material": "^5.15.9", 10 | "@mui/joy": "^5.0.0-beta.27", 11 | "@testing-library/jest-dom": "^6.4.2", 12 | "@testing-library/react": "^14.2.1", 13 | "@testing-library/user-event": "^14.5.2", 14 | "@tremor/react": "^3.13.4", 15 | "axios": "^1.6.7", 16 | "react": "^18.2.0", 17 | "react-dom": "^18.2.0", 18 | "react-router-dom": "^6.22.0", 19 | "react-scripts": "^5.0.1", 20 | "web-vitals": "^3.5.2" 21 | }, 22 | "scripts": { 23 | "start": "react-scripts start", 24 | "build": "react-scripts build", 25 | "test": "react-scripts test", 26 | "eject": "react-scripts eject" 27 | }, 28 | "eslintConfig": { 29 | "extends": [ 30 | "react-app", 31 | "react-app/jest" 32 | ] 33 | }, 34 | "browserslist": { 35 | "production": [ 36 | ">0.2%", 37 | "not dead", 38 | "not op_mini all" 39 | ], 40 | "development": [ 41 | "last 1 chrome version", 42 | "last 1 firefox version", 43 | "last 1 safari version" 44 | ] 45 | }, 46 | "devDependencies": { 47 | "@babel/plugin-proposal-private-property-in-object": "^7.21.11", 48 | "@headlessui/tailwindcss": "^0.2.0", 49 | "@tailwindcss/forms": "^0.5.7", 50 | "tailwindcss": "^3.4.1" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ui/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 14 | 15 | 24 | Motion Dashboard 25 | 26 | 27 | 28 |
29 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /ui/public/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/ui/public/logo.png -------------------------------------------------------------------------------- /ui/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "logo.png", 7 | "type": "image/png", 8 | "sizes": "192x192" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /ui/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /ui/src/App.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dm4ml/motion/f7b4cfa08016a5ac11181c6a22fd4ce0df363b6e/ui/src/App.css -------------------------------------------------------------------------------- /ui/src/App.js: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import { BrowserRouter as Router, Routes, Route } from "react-router-dom"; 3 | import { Box, CssVarsProvider, Sheet } from "@mui/joy"; 4 | import Sidebar from "./components/Sidebar"; 5 | import MotionHeader from "./components/MotionHeader"; 6 | import MainContent from "./components/MainContent"; 7 | import axios from "axios"; 8 | 9 | axios.defaults.baseURL = "http://localhost:8000"; 10 | 11 | function App() { 12 | const [components, setComponents] = useState([]); 13 | 14 | useEffect(() => { 15 | axios 16 | .get("/components") // Assuming the endpoint is '/components' 17 | .then((response) => { 18 | // Add id to each component in response.data 19 | const componentsWithIds = response.data.map((component, index) => { 20 | return { name: component, key: `component${index + 1}` }; 21 | }); 22 | setComponents(componentsWithIds); 23 | }) 24 | .catch((error) => { 25 | console.error("There was an error fetching the components", error); 26 | }); 27 | }, []); 28 | 29 | return ( 30 | 35 | 36 | 44 | 45 | 46 | 47 | 48 | {components.map((component) => ( 49 | } 53 | /> 54 | ))} 55 | 56 | 57 | 58 | 59 | 60 | ); 61 | } 62 | 63 | export default App; 64 | -------------------------------------------------------------------------------- /ui/src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from "@testing-library/react"; 2 | import App from "./App"; 3 | 4 | test("renders learn react link", () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /ui/src/components/ComponentInfoCard.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Card, Typography } from "@mui/joy"; 3 | import { 4 | Tracker, 5 | BarList, 6 | Title, 7 | Flex, 8 | Text, 9 | Metric, 10 | BadgeDelta, 11 | Card as TremorCard, 12 | Grid, 13 | } from "@tremor/react"; 14 | 15 | const ComponentInfoCard = ({ 16 | componentName, 17 | numInstances, 18 | statusCounts, 19 | statusChanges, 20 | fractionUptime, 21 | flowCounts, 22 | statusBars, 23 | }) => { 24 | return ( 25 | 26 | {componentName} 27 | 28 | 29 | 30 |
31 | Success (Last 24 Hr) 32 | {statusCounts["success"]} 33 |
34 | 35 | {statusChanges?.success?.value} 36 | 37 | 38 | 39 | 40 | 41 |
42 | Failure (Last 24 Hr) 43 | {statusCounts["failure"]} 44 |
45 | 46 | {statusChanges?.failure?.value} 47 | 48 | 49 | 50 | 51 | 52 |
53 | Instances 54 | {numInstances} 55 |
56 | 57 | 58 | 59 | 60 | Status 61 | 62 | Uptime {fractionUptime}% 63 | 64 | 65 | 66 | 24 hours ago 67 | Now 68 | 69 | 70 | 71 | Distribution 72 | Last 24 Hr 73 | 74 | Flow 75 | # Runs 76 | 77 | 78 | 79 | 80 | ); 81 | }; 82 | 83 | export default ComponentInfoCard; 84 | -------------------------------------------------------------------------------- /ui/src/components/DetailComponent.js: -------------------------------------------------------------------------------- 1 | import React, { useState } from "react"; 2 | import { Typography, IconButton, Box, Snackbar } from "@mui/joy"; 3 | import ContentCopyIcon from "@mui/icons-material/ContentCopy"; 4 | import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; 5 | import ExpandLessIcon from "@mui/icons-material/ExpandLess"; 6 | 7 | export default function DetailComponent({ detail }) { 8 | const [isExpanded, setIsExpanded] = useState(false); 9 | const [open, setOpen] = useState(false); 10 | 11 | const truncateText = (text, maxLength) => { 12 | if (text.length > maxLength) { 13 | return `${text.substring(0, maxLength)}...`; 14 | } 15 | return text; 16 | }; 17 | 18 | const handleCopy = () => { 19 | navigator.clipboard.writeText(detail.value); 20 | // Add snackbar to show that the text was copied 21 | setOpen(true); 22 | }; 23 | 24 | const handleExpandToggle = () => { 25 | setIsExpanded(!isExpanded); 26 | }; 27 | 28 | return ( 29 | 30 | 39 | {isExpanded ? detail.value : truncateText(detail.value, 100)} 40 | 41 | 47 | {isExpanded ? : } 48 | 49 | 50 | 51 | 52 | { 59 | if (reason === "clickaway") { 60 | return; 61 | } 62 | setOpen(false); 63 | }} 64 | > 65 | {"Copied " + detail.key + " value to clipboard"} 66 | 67 | 68 | ); 69 | } 70 | -------------------------------------------------------------------------------- /ui/src/components/DynamicTable.js: -------------------------------------------------------------------------------- 1 | import Table from "@mui/joy/Table"; 2 | 3 | export default function DynamicTable({ tableData }) { 4 | const headers = Object.keys(tableData[0]); 5 | const data = tableData.map((row) => Object.values(row)); 6 | 7 | return ( 8 | 9 | 10 | 11 | {headers.map((header) => ( 12 | 13 | ))} 14 | 15 | 16 | 17 | {data.map((row, index) => ( 18 | 19 | {row.map((cell, index) => ( 20 | 21 | ))} 22 | 23 | ))} 24 | 25 |
{header}
{cell}
26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /ui/src/components/MotionHeader.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Box, Typography, IconButton, Sheet } from "@mui/joy"; 3 | import GitHubIcon from "@mui/icons-material/GitHub"; 4 | import ArticleIcon from "@mui/icons-material/Article"; 5 | import DarkModeIcon from "@mui/icons-material/DarkMode"; 6 | import LightModeIcon from "@mui/icons-material/LightMode"; 7 | import FastForwardOutlined from "@mui/icons-material/FastForwardOutlined"; 8 | import { useColorScheme } from "@mui/joy/styles"; 9 | 10 | function ModeSwitcher() { 11 | const { mode, setMode } = useColorScheme(); 12 | const [mounted, setMounted] = React.useState(false); 13 | 14 | React.useEffect(() => { 15 | setMounted(true); 16 | }, []); 17 | 18 | React.useEffect(() => { 19 | // Add or remove the 'dark' class on the body element 20 | if (mode === "dark") { 21 | document.body.classList.add("dark"); 22 | } else { 23 | document.body.classList.remove("dark"); 24 | } 25 | }, [mode]); 26 | 27 | if (!mounted) { 28 | return null; 29 | } 30 | 31 | const Icon = mode === "dark" ? LightModeIcon : DarkModeIcon; 32 | 33 | return ( 34 | setMode(mode === "dark" ? "light" : "dark")} 37 | > 38 | 39 | 40 | ); 41 | } 42 | 43 | const MotionHeader = ({ title }) => { 44 | return ( 45 | 59 | 60 | 61 | {title} 62 | 63 | 64 | 65 | 70 | 71 | 72 | 77 | 78 | 79 | 80 | 81 | ); 82 | }; 83 | 84 | export default MotionHeader; 85 | -------------------------------------------------------------------------------- /ui/src/components/Sidebar.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { NavLink, useLocation } from "react-router-dom"; 3 | import { Box, List, ListItemButton, ListSubheader, Typography } from "@mui/joy"; 4 | import Divider from "@mui/joy/Divider"; 5 | 6 | const Sidebar = ({ components }) => { 7 | const location = useLocation(); 8 | 9 | return ( 10 | 18 | 19 | Components 20 | {components.map((component) => ( 21 | 22 | 27 | {component.name} 28 | 29 | 30 | 31 | ))} 32 | 33 | 34 | ); 35 | }; 36 | 37 | export default Sidebar; 38 | -------------------------------------------------------------------------------- /ui/src/customTheme.js: -------------------------------------------------------------------------------- 1 | import { extendTheme } from "@mui/joy/styles"; 2 | 3 | const theme = extendTheme({ 4 | // colorSchemes: { 5 | // light: { 6 | // palette: { 7 | // primary: { 8 | // 50: "#eef2ff", 9 | // 100: "#e0e7ff", 10 | // 200: "#c7d2fe", 11 | // 300: "#a5b4fc", 12 | // 400: "#818cf8", 13 | // 500: "#6366f1", 14 | // 600: "#4f46e5", 15 | // 700: "#4338ca", 16 | // 800: "#3730a3", 17 | // 900: "#312e81", 18 | // }, 19 | // neutral: { 20 | // 50: "#f9fafb", 21 | // 100: "#f3f4f6", 22 | // 200: "#e5e7eb", 23 | // 300: "#d1d5db", 24 | // 400: "#9ca3af", 25 | // 500: "#6b7280", 26 | // 600: "#4b5563", 27 | // 700: "#374151", 28 | // 800: "#1f2937", 29 | // 900: "#111827", 30 | // }, 31 | // danger: { 32 | // 50: "#fef2f2", 33 | // 100: "#fee2e2", 34 | // 200: "#fecaca", 35 | // 300: "#fca5a5", 36 | // 400: "#f87171", 37 | // 500: "#ef4444", 38 | // 600: "#dc2626", 39 | // 700: "#b91c1c", 40 | // 800: "#991b1b", 41 | // 900: "#7f1d1d", 42 | // }, 43 | // success: { 44 | // 50: "#f0fdf4", 45 | // 100: "#dcfce7", 46 | // 200: "#bbf7d0", 47 | // 300: "#86efac", 48 | // 400: "#4ade80", 49 | // 500: "#22c55e", 50 | // 600: "#16a34a", 51 | // 700: "#15803d", 52 | // 800: "#166534", 53 | // 900: "#14532d", 54 | // }, 55 | // warning: { 56 | // 50: "#fffbeb", 57 | // 100: "#fef3c7", 58 | // 200: "#fde68a", 59 | // 300: "#fcd34d", 60 | // 400: "#fbbf24", 61 | // 500: "#f59e0b", 62 | // 600: "#d97706", 63 | // 700: "#b45309", 64 | // 800: "#92400e", 65 | // 900: "#78350f", 66 | // }, 67 | // }, 68 | // }, 69 | // dark: { 70 | // palette: { 71 | // primary: { 72 | // 50: "#eef2ff", 73 | // 100: "#e0e7ff", 74 | // 200: "#c7d2fe", 75 | // 300: "#a5b4fc", 76 | // 400: "#818cf8", 77 | // 500: "#6366f1", 78 | // 600: "#4f46e5", 79 | // 700: "#4338ca", 80 | // 800: "#3730a3", 81 | // 900: "#312e81", 82 | // }, 83 | // neutral: { 84 | // 50: "#fafafa", 85 | // 100: "#f5f5f5", 86 | // 200: "#e5e5e5", 87 | // 300: "#d4d4d4", 88 | // 400: "#a3a3a3", 89 | // 500: "#737373", 90 | // 600: "#525252", 91 | // 700: "#404040", 92 | // 800: "#262626", 93 | // 900: "#171717", 94 | // }, 95 | // danger: { 96 | // 50: "#fef2f2", 97 | // 100: "#fee2e2", 98 | // 200: "#fecaca", 99 | // 300: "#fca5a5", 100 | // 400: "#f87171", 101 | // 500: "#ef4444", 102 | // 600: "#dc2626", 103 | // 700: "#b91c1c", 104 | // 800: "#991b1b", 105 | // 900: "#7f1d1d", 106 | // }, 107 | // warning: { 108 | // 50: "#fffbeb", 109 | // 100: "#fef3c7", 110 | // 200: "#fde68a", 111 | // 300: "#fcd34d", 112 | // 400: "#fbbf24", 113 | // 500: "#f59e0b", 114 | // 600: "#d97706", 115 | // 700: "#b45309", 116 | // 800: "#92400e", 117 | // 900: "#78350f", 118 | // }, 119 | // success: { 120 | // 50: "#f0fdf4", 121 | // 100: "#dcfce7", 122 | // 200: "#bbf7d0", 123 | // 300: "#86efac", 124 | // 400: "#4ade80", 125 | // 500: "#22c55e", 126 | // 600: "#16a34a", 127 | // 700: "#15803d", 128 | // 800: "#166534", 129 | // 900: "#14532d", 130 | // }, 131 | // }, 132 | // }, 133 | // }, 134 | }); 135 | 136 | export default theme; 137 | -------------------------------------------------------------------------------- /ui/src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | body { 6 | margin: 0; 7 | font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen", 8 | "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", 9 | sans-serif; 10 | -webkit-font-smoothing: antialiased; 11 | -moz-osx-font-smoothing: grayscale; 12 | } 13 | 14 | code { 15 | font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", 16 | monospace; 17 | } 18 | -------------------------------------------------------------------------------- /ui/src/index.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom/client"; 3 | import "./index.css"; 4 | import App from "./App"; 5 | import reportWebVitals from "./reportWebVitals"; 6 | 7 | const root = ReactDOM.createRoot(document.getElementById("root")); 8 | root.render( 9 | 10 | 11 | , 12 | ); 13 | 14 | // If you want to start measuring performance in your app, pass a function 15 | // to log results (for example: reportWebVitals(console.log)) 16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 17 | reportWebVitals(); 18 | -------------------------------------------------------------------------------- /ui/src/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /ui/src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = (onPerfEntry) => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import("web-vitals").then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /ui/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import "@testing-library/jest-dom"; 6 | -------------------------------------------------------------------------------- /ui/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | 3 | const colors = require("tailwindcss/colors"); 4 | module.exports = { 5 | content: [ 6 | "./src/**/*.{js,ts,jsx,tsx}", 7 | "./node_modules/@tremor/**/*.{js,ts,jsx,tsx}", 8 | ], 9 | darkMode: "class", 10 | theme: { 11 | transparent: "transparent", 12 | current: "currentColor", 13 | extend: { 14 | colors: { 15 | // light mode 16 | tremor: { 17 | brand: { 18 | faint: colors.blue[50], 19 | muted: colors.blue[200], 20 | subtle: colors.blue[400], 21 | DEFAULT: colors.blue[500], 22 | emphasis: colors.blue[700], 23 | inverted: colors.white, 24 | }, 25 | background: { 26 | muted: colors.gray[50], 27 | subtle: colors.gray[100], 28 | DEFAULT: colors.white, 29 | emphasis: colors.gray[700], 30 | }, 31 | border: { 32 | DEFAULT: colors.gray[200], 33 | }, 34 | ring: { 35 | DEFAULT: colors.gray[200], 36 | }, 37 | content: { 38 | subtle: colors.gray[400], 39 | DEFAULT: colors.gray[500], 40 | emphasis: colors.gray[700], 41 | strong: colors.gray[900], 42 | inverted: colors.white, 43 | }, 44 | }, 45 | // dark mode 46 | "dark-tremor": { 47 | brand: { 48 | faint: "#0B1229", 49 | muted: colors.blue[950], 50 | subtle: colors.blue[800], 51 | DEFAULT: colors.blue[500], 52 | emphasis: colors.blue[400], 53 | inverted: colors.blue[950], 54 | }, 55 | background: { 56 | muted: "#131A2B", 57 | subtle: colors.gray[800], 58 | DEFAULT: colors.gray[900], 59 | emphasis: colors.gray[300], 60 | }, 61 | border: { 62 | DEFAULT: colors.gray[800], 63 | }, 64 | ring: { 65 | DEFAULT: colors.gray[800], 66 | }, 67 | content: { 68 | subtle: colors.gray[600], 69 | DEFAULT: colors.gray[500], 70 | emphasis: colors.gray[200], 71 | strong: colors.gray[50], 72 | inverted: colors.gray[950], 73 | }, 74 | }, 75 | }, 76 | boxShadow: { 77 | // light 78 | "tremor-input": "0 1px 2px 0 rgb(0 0 0 / 0.05)", 79 | "tremor-card": 80 | "0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1)", 81 | "tremor-dropdown": 82 | "0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1)", 83 | // dark 84 | "dark-tremor-input": "0 1px 2px 0 rgb(0 0 0 / 0.05)", 85 | "dark-tremor-card": 86 | "0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1)", 87 | "dark-tremor-dropdown": 88 | "0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1)", 89 | }, 90 | borderRadius: { 91 | "tremor-small": "0.375rem", 92 | "tremor-default": "0.5rem", 93 | "tremor-full": "9999px", 94 | }, 95 | fontSize: { 96 | "tremor-label": ["0.75rem", { lineHeight: "1rem" }], 97 | "tremor-default": ["0.875rem", { lineHeight: "1.25rem" }], 98 | "tremor-title": ["1.125rem", { lineHeight: "1.75rem" }], 99 | "tremor-metric": ["1.875rem", { lineHeight: "2.25rem" }], 100 | }, 101 | }, 102 | }, 103 | safelist: [ 104 | { 105 | pattern: 106 | /^(bg-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 107 | variants: ["hover", "ui-selected"], 108 | }, 109 | { 110 | pattern: 111 | /^(text-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 112 | variants: ["hover", "ui-selected"], 113 | }, 114 | { 115 | pattern: 116 | /^(border-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 117 | variants: ["hover", "ui-selected"], 118 | }, 119 | { 120 | pattern: 121 | /^(ring-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 122 | }, 123 | { 124 | pattern: 125 | /^(stroke-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 126 | }, 127 | { 128 | pattern: 129 | /^(fill-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, 130 | }, 131 | ], 132 | plugins: [require("@headlessui/tailwindcss"), require("@tailwindcss/forms")], 133 | }; 134 | --------------------------------------------------------------------------------