├── .github ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── build.yml │ └── wiki.yml ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── Makefile ├── README.md ├── csp_gateway ├── __init__.py ├── benchmarks │ ├── asv.conf.jsonc │ └── benchmarks │ │ ├── __init__.py │ │ ├── server │ │ ├── __init__.py │ │ └── gateway │ │ │ ├── __init__.py │ │ │ └── csp │ │ │ ├── __init__.py │ │ │ └── state.py │ │ └── utils │ │ ├── __init__.py │ │ └── id_generator.py ├── client │ ├── __init__.py │ └── client.py ├── server │ ├── __init__.py │ ├── build │ │ └── favicon.png │ ├── cli.py │ ├── config │ │ ├── __init__.py │ │ ├── base.yaml │ │ ├── gateway │ │ │ ├── demo.yaml │ │ │ └── omnibus.yaml │ │ ├── hydra │ │ │ └── job_logging │ │ │ │ └── custom.yaml │ │ └── modules.yaml │ ├── demo │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── config │ │ │ ├── demo.yaml │ │ │ └── omnibus.yaml │ │ ├── omnibus.py │ │ └── simple.py │ ├── gateway │ │ ├── __init__.py │ │ ├── csp │ │ │ ├── __init__.py │ │ │ ├── channels.py │ │ │ ├── factory.py │ │ │ ├── futures │ │ │ │ ├── __init__.py │ │ │ │ └── adapter.py │ │ │ ├── module.py │ │ │ └── state.py │ │ └── gateway.py │ ├── middleware │ │ ├── __init__.py │ │ ├── api_key.py │ │ └── hacks │ │ │ ├── __init__.py │ │ │ └── api_key_middleware_websocket_fix │ │ │ ├── __init__.py │ │ │ ├── api_key.py │ │ │ ├── http.py │ │ │ └── utils.py │ ├── modules │ │ ├── __init__.py │ │ ├── controls │ │ │ ├── __init__.py │ │ │ └── controls.py │ │ ├── initializer.py │ │ ├── io │ │ │ ├── __init__.py │ │ │ ├── graph_output.py │ │ │ ├── json.py │ │ │ └── json_pull_adapter.py │ │ ├── kafka │ │ │ ├── __init__.py │ │ │ ├── kafka.py │ │ │ └── utils.py │ │ ├── logging │ │ │ ├── __init__.py │ │ │ ├── datadog.py │ │ │ ├── logging.py │ │ │ ├── opsgenie.py │ │ │ ├── printing.py │ │ │ ├── symphony.py │ │ │ └── util.py │ │ ├── mirror.py │ │ ├── sql.py │ │ └── web │ │ │ ├── __init__.py │ │ │ ├── channels_graph.py │ │ │ ├── mount.py │ │ │ ├── mount_fields.py │ │ │ ├── outputs.py │ │ │ ├── perspective.py │ │ │ └── websocket.py │ ├── settings.py │ ├── shared │ │ ├── __init__.py │ │ ├── adapters.py │ │ ├── channel_selection.py │ │ ├── engine_replay.py │ │ └── json_converter.py │ └── web │ │ ├── __init__.py │ │ ├── app.py │ │ ├── routes │ │ ├── __init__.py │ │ ├── controls.py │ │ ├── last.py │ │ ├── lookup.py │ │ ├── next.py │ │ ├── send.py │ │ ├── shared.py │ │ └── state.py │ │ ├── templates │ │ ├── channels_graph.html.j2 │ │ ├── css │ │ │ └── common.css │ │ ├── files.html.j2 │ │ ├── js │ │ │ └── common.js │ │ ├── login.html.j2 │ │ └── logout.html.j2 │ │ └── utils.py ├── testing │ ├── __init__.py │ ├── harness.py │ ├── shared_helpful_classes.py │ └── web.py ├── tests │ ├── __init__.py │ ├── client │ │ └── test_client.py │ ├── config │ │ ├── test_load.py │ │ └── user_config │ │ │ └── sample_config.yaml │ ├── conftest.py │ ├── server │ │ ├── __init__.py │ │ ├── config │ │ │ ├── __init__.py │ │ │ └── test_load_config.py │ │ ├── gateway │ │ │ ├── __init__.py │ │ │ ├── csp │ │ │ │ ├── __init__.py │ │ │ │ ├── test_channels.py │ │ │ │ └── test_state.py │ │ │ ├── test_gateway.py │ │ │ ├── test_gateway_dicts_and_feedbacks.py │ │ │ └── test_gateway_start_stop.py │ │ ├── modules │ │ │ ├── __init__.py │ │ │ ├── controls │ │ │ │ └── __init__.py │ │ │ ├── io │ │ │ │ ├── __init__.py │ │ │ │ ├── test_graph_output.py │ │ │ │ ├── test_json.py │ │ │ │ ├── test_json_converter.py │ │ │ │ └── test_json_pull_adapter.py │ │ │ ├── kafka │ │ │ │ ├── __init__.py │ │ │ │ └── test_kafka.py │ │ │ ├── logging │ │ │ │ ├── __init__.py │ │ │ │ ├── test_datadog.py │ │ │ │ ├── test_logging.py │ │ │ │ ├── test_opsgenie.py │ │ │ │ ├── test_printing.py │ │ │ │ └── test_util.py │ │ │ ├── test_initializer.py │ │ │ ├── test_mirror.py │ │ │ ├── test_sql.py │ │ │ └── web │ │ │ │ ├── __init__.py │ │ │ │ └── test_perspective.py │ │ ├── shared │ │ │ ├── __init__.py │ │ │ ├── test_adapters.py │ │ │ └── test_channel_selection.py │ │ └── web │ │ │ ├── __init__.py │ │ │ ├── test_shared.py │ │ │ └── test_webserver.py │ ├── test_harness.py │ └── utils │ │ ├── __init__.py │ │ ├── struct │ │ ├── __init__.py │ │ ├── test_base.py │ │ └── test_lookup.py │ │ ├── test_csp.py │ │ ├── test_picklable_queue.py │ │ └── web │ │ ├── __init__.py │ │ └── test_query.py └── utils │ ├── __init__.py │ ├── csp.py │ ├── enums.py │ ├── exceptions.py │ ├── fastapi.py │ ├── id_generator.py │ ├── picklable_queue.py │ ├── struct │ ├── __init__.py │ ├── base.py │ └── psp.py │ └── web │ ├── __init__.py │ ├── controls.py │ ├── filter.py │ └── query.py ├── docs ├── img │ ├── demo.gif │ ├── logo-name-dark.png │ └── logo-name.png └── wiki │ ├── API.md │ ├── CSP-Notes.md │ ├── Client.md │ ├── Configuration.md │ ├── Develop.md │ ├── Installation.md │ ├── Modules.md │ ├── Overview.md │ ├── UI.md │ ├── _Footer.md │ ├── _Sidebar.md │ └── contribute │ ├── Build-from-Source.md │ ├── Contribute.md │ └── Local-Development-Setup.md ├── examples ├── Client.ipynb ├── kafka_example.py ├── multiprocessing_example.py ├── panel_example.py └── websocket_client.py ├── js ├── README.md ├── build.mjs ├── package.json ├── pnpm-lock.yaml └── src │ ├── common.js │ ├── components │ ├── footer.jsx │ ├── header.jsx │ ├── index.js │ ├── logo.jsx │ ├── perspective │ │ ├── gateway.js │ │ ├── index.js │ │ ├── layout.js │ │ ├── tables.js │ │ ├── theme.js │ │ └── workspace.jsx │ └── settings.jsx │ ├── html │ └── index.html │ ├── index.jsx │ └── style │ ├── base.css │ ├── favicon.png │ ├── header_footer.css │ ├── index.css │ ├── nord.css │ ├── perspective.css │ └── settings.css └── pyproject.toml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @arhamchopra @emilybarrettp72 @neejweej @ptomecek @feussy @hintse @timkpaine @vstolin 2 | 3 | # Administrative 4 | .github/CODEOWNERS @ptomecek @timkpaine 5 | LICENSE @ptomecek @timkpaine 6 | NOTICE @ptomecek @timkpaine 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 15 | 16 | **Describe the bug** 17 | 18 | 19 | **To Reproduce** 20 | 30 | 31 | 32 | 33 | **Expected behavior** 34 | 35 | 36 | **Error Message** 37 | 39 | 40 | **Runtime Environment** 41 | 45 | 46 | **Additional context** 47 | 50 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | labels: 8 | - "part: github_actions" 9 | 10 | - package-ecosystem: "pip" 11 | directory: "/" 12 | schedule: 13 | interval: "monthly" 14 | labels: 15 | - "lang: python" 16 | - "part: dependencies" 17 | 18 | - package-ecosystem: "npm" 19 | directory: "/js" 20 | schedule: 21 | interval: "monthly" 22 | labels: 23 | - "lang: javascript" 24 | - "part: dependencies" 25 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build Status 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - v* 9 | paths-ignore: 10 | - LICENSE 11 | - README.md 12 | pull_request: 13 | branches: 14 | - main 15 | workflow_dispatch: 16 | 17 | concurrency: 18 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 19 | cancel-in-progress: true 20 | 21 | permissions: 22 | contents: read 23 | checks: write 24 | pull-requests: write 25 | 26 | jobs: 27 | build: 28 | runs-on: ${{ matrix.os }} 29 | 30 | strategy: 31 | matrix: 32 | os: [ubuntu-latest, macos-latest] 33 | python-version: ["3.9", "3.10", "3.11"] 34 | node-version: [20.x] 35 | exclude: 36 | # No CSP builds for python 3.9 / macos arm64 37 | - python-version: "3.9" 38 | os: macos-latest 39 | 40 | steps: 41 | - uses: actions/checkout@v4 42 | 43 | - name: Set up Python ${{ matrix.python-version }} 44 | uses: actions/setup-python@v5 45 | with: 46 | python-version: ${{ matrix.python-version }} 47 | cache: 'pip' 48 | cache-dependency-path: 'pyproject.toml' 49 | 50 | - name: Use Node.js ${{ matrix.node-version }} 51 | uses: actions/setup-node@v4 52 | with: 53 | node-version: ${{ matrix.node-version }} 54 | 55 | - name: Install pnpm 56 | uses: pnpm/action-setup@v4 57 | with: 58 | version: 9 59 | package_json_file: js/package.json 60 | 61 | - name: Install dependencies 62 | run: make develop 63 | 64 | - name: Lint 65 | run: make lint 66 | if: matrix.os == 'ubuntu-latest' 67 | 68 | - name: Checks 69 | run: make checks 70 | if: matrix.os == 'ubuntu-latest' 71 | 72 | - name: Build 73 | run: make build 74 | 75 | - name: Test 76 | run: make coverage 77 | if: matrix.os != 'windows-latest' 78 | 79 | - name: Upload test results (Python) 80 | uses: actions/upload-artifact@v4 81 | with: 82 | name: test-results-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.node-version }} 83 | path: '**/junit.xml' 84 | if: ${{ always() }} 85 | 86 | - name: Publish Unit Test Results 87 | uses: EnricoMi/publish-unit-test-result-action@v2 88 | with: 89 | files: '**/junit.xml' 90 | if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' 91 | 92 | - name: Upload coverage 93 | uses: codecov/codecov-action@v5 94 | with: 95 | token: ${{ secrets.CODECOV_TOKEN }} 96 | 97 | - name: Make dist 98 | run: make dist 99 | if: matrix.os == 'ubuntu-latest' 100 | 101 | - uses: actions/upload-artifact@v4 102 | with: 103 | name: dist-${{matrix.os}} 104 | path: dist 105 | if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' 106 | -------------------------------------------------------------------------------- /.github/workflows/wiki.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - "docs/**" 9 | - "README.md" 10 | workflow_dispatch: 11 | 12 | concurrency: 13 | group: docs 14 | cancel-in-progress: true 15 | 16 | permissions: 17 | contents: write 18 | 19 | jobs: 20 | deploy: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Make home file 27 | run: cp README.md docs/wiki/Home.md 28 | 29 | - name: Install Python 30 | uses: actions/setup-python@v5 31 | with: 32 | python-version: 3.11 33 | 34 | - name: Install NBConvert 35 | run: pip install nbconvert 36 | 37 | - name: Convert Example Notebooks to Markdown 38 | run: jupyter nbconvert examples/Client.ipynb --to markdown --output ../docs/wiki/Client.md 39 | 40 | - name: Upload Documentation to Wiki 41 | uses: Andrew-Chen-Wang/github-wiki-action@v4 42 | with: 43 | path: docs/wiki 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | all/ 11 | all/** 12 | dist/ 13 | dist/** 14 | package/ 15 | package/** 16 | .Python 17 | env/ 18 | build/ 19 | build/** 20 | **/build 21 | flake8/ 22 | flake8/** 23 | develop-eggs/ 24 | downloads/ 25 | eggs/ 26 | .eggs/ 27 | lib/ 28 | lib64/ 29 | node_modules/ 30 | parts/ 31 | sdist/ 32 | var/ 33 | wheels/ 34 | *.egg-info/ 35 | .installed.cfg 36 | *.egg 37 | .asv 38 | 39 | # PyInstaller 40 | # Usually these files are written by a python script from a template 41 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 42 | *.manifest 43 | *.spec 44 | 45 | # Installer logs 46 | pip-log.txt 47 | pip-delete-this-directory.txt 48 | 49 | # Unit test / coverage reports 50 | reports/** 51 | htmlcov/ 52 | .tox/ 53 | .coverage 54 | .coverage.* 55 | .cache 56 | coverage.xml 57 | junit.xml 58 | nosetests.xml 59 | *.cover 60 | .hypothesis/ 61 | .pytest_cache/ 62 | 63 | # Translations 64 | *.mo 65 | *.pot 66 | 67 | # Django stuff: 68 | *.log 69 | local_settings.py 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | docs/api 81 | docs/html/ 82 | docs/jupyter_execute 83 | index.md 84 | # auto-generated files 85 | docs/wiki/components/Client.md 86 | 87 | # PyBuilder 88 | target/ 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # celery beat schedule file 94 | celerybeat-schedule 95 | 96 | # SageMath parsed files 97 | *.sage.py 98 | 99 | # dotenv 100 | .env 101 | 102 | # virtualenv 103 | .venv 104 | venv/ 105 | ENV/ 106 | 107 | # env used by requirements.sh 108 | .freeze_env 109 | 110 | # Spyder project settings 111 | .spyderproject 112 | .spyproject 113 | 114 | # Rope project settings 115 | .ropeproject 116 | 117 | # mkdocs documentation 118 | /site 119 | 120 | # mypy 121 | .mypy_cache/ 122 | 123 | # IDE settings 124 | .idea/ 125 | 126 | # CICD 127 | requirements.txt 128 | !ci/ 129 | ci/benchmarks/* 130 | !ci/benchmarks/benchmarks.json 131 | 132 | # Byte-compiled / optimized / DLL files 133 | __pycache__/ 134 | *.py[cod] 135 | *$py.class 136 | 137 | # OSX useful to ignore 138 | *.DS_Store 139 | .AppleDouble 140 | .LSOverride 141 | 142 | # Thumbnails 143 | ._* 144 | 145 | # Files that might appear in the root of a volume 146 | .DocumentRevisions-V100 147 | .fseventsd 148 | .Spotlight-V100 149 | .TemporaryItems 150 | .Trashes 151 | .VolumeIcon.icns 152 | .com.apple.timemachine.donotpresent 153 | 154 | # Directories potentially created on remote AFP share 155 | .AppleDB 156 | .AppleDesktop 157 | Network Trash Folder 158 | Temporary Items 159 | .apdisk 160 | 161 | # C extensions 162 | *.so 163 | 164 | # Jupyter 165 | .ipynb_checkpoints 166 | Untitled*.ipynb 167 | 168 | # Other 169 | tmp.py 170 | tmp.txt 171 | tmp.png 172 | repro*.py 173 | univ.txt.* 174 | .universe_data 175 | outputs/ 176 | 177 | .autoversion 178 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "eslint.workingDirectories": ["./js"] 3 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | csp-gateway logo, overlapping blue chevrons facing right 5 | 6 | 7 | 8 |
9 | 10 | [![Build Status](https://github.com/Point72/csp-gateway/actions/workflows/build.yml/badge.svg)](https://github.com/Point72/csp-gateway/actions/workflows/build.yml) 11 | [![License](https://img.shields.io/badge/license-Apache--2.0-green)](https://github.com/Point72/csp-gateway/LICENSE) 12 | [![PyPI](https://img.shields.io/pypi/v/csp-gateway.svg?style=flat)](https://pypi.python.org/pypi/csp-gateway) 13 | 14 | ## Overview 15 | 16 | `csp-gateway` is a framework for building high-performance streaming applications. 17 | It is is composed of four major components: 18 | 19 | - Engine: [csp](https://github.com/point72/csp), a streaming, complex event processor core 20 | - API: [FastAPI](https://fastapi.tiangolo.com) REST/WebSocket API 21 | - UI: [Perspective](https://perspective.finos.org) and React based frontend with automatic table and chart visualizations 22 | - Configuration: [ccflow](https://github.com/point72/ccflow), a [Pydantic](https://docs.pydantic.dev/latest/)/[Hydra](https://hydra.cc) based extensible, composeable dependency injection and configuration framework 23 | 24 | For a detailed overview, see our [Documentation](https://github.com/Point72/csp-gateway/wiki/Overview). 25 | 26 | ![A brief demo gif of csp-gateway ui, graph viewer, rest api docs, and rest api](https://raw.githubusercontent.com/point72/csp-gateway/main/docs/img/demo.gif) 27 | 28 | ## Installation 29 | 30 | `csp-gateway` can be installed via [pip](https://pip.pypa.io) or [conda](https://docs.conda.io/en/latest/), the two primary package managers for the Python ecosystem. 31 | 32 | To install `csp-gateway` via **pip**, run this command in your terminal: 33 | 34 | ```bash 35 | pip install csp-gateway 36 | ``` 37 | 38 | To install `csp-gateway` via **conda**, run this command in your terminal: 39 | 40 | ```bash 41 | conda install csp-gateway -c conda-forge 42 | ``` 43 | 44 | ## Getting Started 45 | 46 | See [our wiki!](https://github.com/Point72/csp-gateway/wiki) 47 | 48 | ## Development 49 | 50 | Check out the [contribution guide](https://github.com/Point72/csp-gateway/wiki/Contribute) for more information. 51 | 52 | ## License 53 | 54 | This software is licensed under the Apache 2.0 license. See the [LICENSE](https://github.com/Point72/csp-gateway/blob/main/LICENSE) file for details. 55 | -------------------------------------------------------------------------------- /csp_gateway/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "2.1.0" 2 | 3 | from .client import * 4 | from .server import * 5 | from .utils import * 6 | -------------------------------------------------------------------------------- /csp_gateway/benchmarks/asv.conf.jsonc: -------------------------------------------------------------------------------- 1 | { 2 | // The version of the config file format. Do not change, unless 3 | // you know what you are doing. 4 | "version": 1, 5 | 6 | // The name of the project being benchmarked 7 | "project": "csp_gateway", 8 | 9 | // The project's homepage 10 | "project_url": "https://github.prod.devops.point72.com/9951-CubistResearchTech/csp-gateway/", 11 | 12 | // The URL or local path of the source code repository for the 13 | // project being benchmarked 14 | "repo": "../..", 15 | 16 | // Customizable commands for building the project. 17 | // See asv.conf.json documentation. 18 | "build_command": [ 19 | "python -m pip install build", 20 | "python -m build", 21 | "python -m build --wheel -o {build_cache_dir} {build_dir}" 22 | ], 23 | 24 | // Customizable commands for installing and uninstalling the project. 25 | // See asv.conf.json documentation. 26 | "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], 27 | "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], 28 | 29 | // List of branches to benchmark. If not provided, defaults to "main" 30 | // (for git) or "default" (for mercurial). 31 | "branches": ["develop"], // Our development is on the `develop` branch, `main` is used for releases 32 | 33 | // The tool to use to create environments. May be "conda", 34 | // "virtualenv", "mamba" (above 3.8) 35 | // or other value depending on the plugins in use. 36 | "environment_type": "virtualenv", 37 | 38 | // timeout in seconds for installing any dependencies in environment 39 | // defaults to 10 min 40 | "install_timeout": 600, 41 | 42 | // the base URL to show a commit for the project. 43 | "show_commit_url": "https://github.prod.devops.point72.com/9951-CubistResearchTech/csp-gateway/commit/", 44 | 45 | // The Pythons you'd like to test against. If not provided, defaults 46 | // to the current version of Python used to run `asv`. 47 | // "pythons": ["3.9", "3.10", "3.11", "3.12"], 48 | "pythons": ["3.9"], 49 | 50 | // The directory (relative to the current directory) that benchmarks are 51 | // stored in. If not provided, defaults to "benchmarks" 52 | "benchmark_dir": "../../csp_gateway/benchmarks/benchmarks", 53 | 54 | // The directory (relative to the current directory) to cache the Python 55 | // environments in. If not provided, defaults to "env" 56 | "env_dir": "../../.asv/env", 57 | 58 | // The directory (relative to the current directory) that raw benchmark 59 | // results are stored in. If not provided, defaults to "results". 60 | "results_dir": "../../ci/benchmarks", 61 | 62 | // The directory (relative to the current directory) that the html tree 63 | // should be written to. If not provided, defaults to "html". 64 | "html_dir": "../../.asv/html", 65 | 66 | // The number of characters to retain in the commit hashes. 67 | "hash_length": 8, 68 | } 69 | -------------------------------------------------------------------------------- /csp_gateway/benchmarks/benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/benchmarks/benchmarks/__init__.py -------------------------------------------------------------------------------- /csp_gateway/benchmarks/benchmarks/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/benchmarks/benchmarks/server/__init__.py -------------------------------------------------------------------------------- /csp_gateway/benchmarks/benchmarks/server/gateway/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/benchmarks/benchmarks/server/gateway/__init__.py -------------------------------------------------------------------------------- /csp_gateway/benchmarks/benchmarks/server/gateway/csp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/benchmarks/benchmarks/server/gateway/csp/__init__.py -------------------------------------------------------------------------------- /csp_gateway/benchmarks/benchmarks/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/benchmarks/benchmarks/utils/__init__.py -------------------------------------------------------------------------------- /csp_gateway/benchmarks/benchmarks/utils/id_generator.py: -------------------------------------------------------------------------------- 1 | from csp_gateway import GatewayModule, get_counter 2 | 3 | 4 | class MyGatewayModule(GatewayModule): 5 | def connect(self, channels): 6 | pass 7 | 8 | def shutdown(self): 9 | pass 10 | 11 | 12 | class IDGenerator: 13 | def time_get_counter(self): 14 | _ = get_counter(MyGatewayModule()) 15 | 16 | 17 | class Counter: 18 | def setup(self): 19 | self.counter = get_counter(MyGatewayModule()) 20 | 21 | def time_counter_next(self): 22 | self.counter.next() 23 | -------------------------------------------------------------------------------- /csp_gateway/client/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import ( 2 | AsyncGatewayClient, 3 | AsyncGatewayClientMixin, 4 | BaseGatewayClient, 5 | GatewayClient, 6 | GatewayClientConfig, 7 | ResponseWrapper, 8 | SyncGatewayClient, 9 | SyncGatewayClientMixin, 10 | ) 11 | -------------------------------------------------------------------------------- /csp_gateway/server/__init__.py: -------------------------------------------------------------------------------- 1 | # isort: skip_file 2 | 3 | from .config import load_config, load_gateway 4 | from .gateway import * 5 | from .shared import * 6 | from .settings import Settings as GatewaySettings 7 | from .modules import * 8 | from .web import * 9 | from .middleware import * 10 | -------------------------------------------------------------------------------- /csp_gateway/server/build/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/server/build/favicon.png -------------------------------------------------------------------------------- /csp_gateway/server/cli.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pprint import pprint 3 | 4 | import hydra 5 | from ccflow import ModelRegistry 6 | 7 | from csp_gateway import __version__ 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | @hydra.main(config_path="config", config_name="base", version_base=None) 13 | def main(cfg): 14 | log.info("Loading csp-gateway config...") 15 | registry = ModelRegistry.root() 16 | registry.load_config(cfg=cfg, overwrite=True) 17 | gateway = registry["gateway"] 18 | 19 | log.info(f"Starting csp_gateway version {__version__}") 20 | kwargs = cfg["start"] 21 | if kwargs: # i.e. start=False override on command line 22 | log.info(f"Starting gateway with arguments: {kwargs}") 23 | gateway.start(**kwargs) 24 | else: 25 | pprint(gateway.model_dump(by_alias=True)) 26 | 27 | 28 | if __name__ == "__main__": 29 | main() 30 | -------------------------------------------------------------------------------- /csp_gateway/server/config/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import wraps 3 | from pathlib import Path 4 | from typing import TYPE_CHECKING, List, Optional 5 | 6 | from ccflow import RootModelRegistry, load_config as load_config_base 7 | 8 | if TYPE_CHECKING: 9 | from csp_gateway import Gateway 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | __all__ = ( 14 | "load_config", 15 | "load_gateway", 16 | ) 17 | 18 | 19 | def load_config( 20 | config_dir: str = "", 21 | config_name: str = "", 22 | overrides: Optional[List[str]] = None, 23 | *, 24 | overwrite: bool = True, 25 | basepath: str = "", 26 | ) -> RootModelRegistry: 27 | log.info("Loading csp-gateway config...") 28 | return load_config_base( 29 | root_config_dir=str(Path(__file__).resolve().parent), 30 | root_config_name="base", 31 | config_dir=config_dir, 32 | config_name=config_name, 33 | overrides=overrides, 34 | overwrite=overwrite, 35 | basepath=basepath, 36 | ) 37 | 38 | 39 | @wraps(load_config) 40 | def load_gateway(*args, **kwargs) -> "Gateway": 41 | log.info("Loading csp-gateway config...") 42 | return load_config(*args, **kwargs)["gateway"] 43 | -------------------------------------------------------------------------------- /csp_gateway/server/config/base.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - modules@modules 3 | - override hydra/job_logging: custom 4 | - _self_ 5 | 6 | start: 7 | # Options passed to gateway.start function 8 | realtime: True 9 | block: False 10 | show: False 11 | rest: True 12 | ui: True 13 | 14 | 15 | # Hydra config 16 | # See https://hydra.cc/docs/tutorials/basic/running_your_app/working_directory 17 | # https://hydra.cc/docs/configure_hydra/job/ 18 | hydra: 19 | run: 20 | dir: outputs/${oc.env:HOSTNAME,localhost}_${hydra.job.name}/${now:%Y-%m-%d}/${now:%H-%M-%S} 21 | -------------------------------------------------------------------------------- /csp_gateway/server/config/gateway/demo.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | defaults: 3 | - _self_ 4 | 5 | port: ??? 6 | 7 | modules: 8 | example_module: 9 | _target_: csp_gateway.server.demo.simple.ExampleModule 10 | mount_perspective_tables: 11 | _target_: csp_gateway.MountPerspectiveTables 12 | update_interval: 00:00:01 13 | mount_rest_routes: 14 | _target_: csp_gateway.MountRestRoutes 15 | force_mount_all: True 16 | 17 | gateway: 18 | _target_: csp_gateway.Gateway 19 | settings: 20 | PORT: ${port} 21 | UI: True 22 | modules: 23 | - /modules/example_module 24 | - /modules/mount_perspective_tables 25 | - /modules/mount_rest_routes 26 | channels: 27 | _target_: csp_gateway.server.demo.simple.ExampleGatewayChannels 28 | 29 | hydra: 30 | run: 31 | dir: outputs/${oc.env:HOSTNAME,localhost}_${hydra.job.name}/${now:%Y-%m-%d}/${now:%H-%M-%S} 32 | -------------------------------------------------------------------------------- /csp_gateway/server/config/gateway/omnibus.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | defaults: 3 | - _self_ 4 | 5 | authenticate: ??? 6 | port: ??? 7 | 8 | modules: 9 | example_module: 10 | _target_: csp_gateway.server.demo.ExampleModule 11 | example_module_feedback: 12 | _target_: csp_gateway.server.demo.ExampleModuleFeedback 13 | example_custom_table: 14 | _target_: csp_gateway.server.demo.ExampleModuleCustomTable 15 | mount_channels_graph: 16 | _target_: csp_gateway.MountChannelsGraph 17 | mount_controls: 18 | _target_: csp_gateway.MountControls 19 | mount_outputs: 20 | _target_: csp_gateway.MountOutputsFolder 21 | mount_perspective_tables: 22 | _target_: csp_gateway.MountPerspectiveTables 23 | perspective_field: "perspective" 24 | layouts: 25 | Server Defined Layout: '{"sizes":[1],"detail":{"main":{"type":"split-area","orientation":"vertical","children":[{"type":"split-area","orientation":"horizontal","children":[{"type":"tab-area","widgets":["EXAMPLE_LIST_GENERATED_4"],"currentIndex":0},{"type":"tab-area","widgets":["PERSPECTIVE_GENERATED_ID_1"],"currentIndex":0}],"sizes":[0.3,0.7]},{"type":"split-area","orientation":"horizontal","children":[{"type":"tab-area","widgets":["EXAMPLE_GENERATED_3"],"currentIndex":0},{"type":"tab-area","widgets":["PERSPECTIVE_GENERATED_ID_0"],"currentIndex":0}],"sizes":[0.3,0.7]}],"sizes":[0.5,0.5]}},"viewers":{"EXAMPLE_LIST_GENERATED_4":{"version":"3.3.4","plugin":"Datagrid","plugin_config":{"columns":{},"edit_mode":"READ_ONLY","scroll_lock":false},"columns_config":{},"title":"example_list","group_by":[],"split_by":[],"columns":["timestamp","x","y","data","mapping","dt","d","internal_csp_struct.z"],"filter":[],"sort":[["timestamp","desc"]],"expressions":{},"aggregates":{},"table":"example_list","settings":false},"PERSPECTIVE_GENERATED_ID_1":{"version":"3.3.4","plugin":"X Bar","plugin_config":{},"columns_config":{},"title":"example_list (*)","group_by":["x"],"split_by":[],"columns":["y"],"filter":[],"sort":[["x","asc"]],"expressions":{},"aggregates":{"y":"median"},"table":"example_list","settings":false},"EXAMPLE_GENERATED_3":{"version":"3.3.4","plugin":"Datagrid","plugin_config":{"columns":{},"edit_mode":"READ_ONLY","scroll_lock":false},"columns_config":{},"title":"example","group_by":[],"split_by":[],"columns":["timestamp","x","y","data","mapping","dt","d","internal_csp_struct.z"],"filter":[],"sort":[["timestamp","desc"]],"expressions":{},"aggregates":{},"table":"example","settings":false},"PERSPECTIVE_GENERATED_ID_0":{"version":"3.3.4","plugin":"Treemap","plugin_config":{},"columns_config":{},"title":"example (*)","group_by":["x"],"split_by":[],"columns":["y","x",null],"filter":[],"sort":[["timestamp","desc"]],"expressions":{},"aggregates":{},"table":"example","settings":false}}}' 26 | mount_rest_routes: 27 | _target_: csp_gateway.MountRestRoutes 28 | force_mount_all: True 29 | mount_websocket_routes: 30 | _target_: csp_gateway.MountWebSocketRoutes 31 | mount_api_key_middleware: 32 | _target_: csp_gateway.MountAPIKeyMiddleware 33 | 34 | gateway: 35 | _target_: csp_gateway.Gateway 36 | settings: 37 | PORT: ${port} 38 | AUTHENTICATE: ${authenticate} 39 | UI: True 40 | API_KEY: "12345" 41 | modules: 42 | - /modules/example_module 43 | - /modules/example_module_feedback 44 | - /modules/example_custom_table 45 | - /modules/mount_channels_graph 46 | - /modules/mount_controls 47 | - /modules/mount_outputs 48 | - /modules/mount_perspective_tables 49 | - /modules/mount_rest_routes 50 | - /modules/mount_websocket_routes 51 | - /modules/mount_api_key_middleware 52 | channels: 53 | _target_: csp_gateway.server.demo.ExampleGatewayChannels 54 | 55 | hydra: 56 | # searchpath: 57 | run: 58 | dir: outputs/${oc.env:HOSTNAME,localhost}_${hydra.job.name}/${now:%Y-%m-%d}/${now:%H-%M-%S} 59 | -------------------------------------------------------------------------------- /csp_gateway/server/config/hydra/job_logging/custom.yaml: -------------------------------------------------------------------------------- 1 | # See https://hydra.cc/docs/configure_hydra/logging/ 2 | version: 1 3 | disable_existing_loggers: False 4 | formatters: 5 | simple: 6 | format: '[%(asctime)s][%(threadName)s][%(name)s][%(levelname)s]: %(message)s' 7 | colorlog: 8 | '()': 'colorlog.ColoredFormatter' 9 | format: '[%(cyan)s%(asctime)s%(reset)s][%(threadName)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]: %(message)s' 10 | log_colors: 11 | DEBUG: purple 12 | INFO: green 13 | WARNING: yellow 14 | ERROR: red 15 | CRITICAL: red 16 | whenAndWhere: 17 | format: '[%(asctime)s][%(threadName)s][%(name)s][%(filename)s:%(lineno)s][%(levelname)s]: %(message)s' 18 | handlers: 19 | console: 20 | level: INFO 21 | class: logging.StreamHandler 22 | formatter: colorlog 23 | stream: ext://sys.stdout 24 | file: 25 | level: DEBUG 26 | class: logging.FileHandler 27 | formatter: whenAndWhere 28 | filename: ${hydra.runtime.output_dir}/csp-gateway.log 29 | root: 30 | handlers: [console, file] 31 | level: DEBUG 32 | loggers: 33 | uvicorn.error: 34 | level: CRITICAL 35 | -------------------------------------------------------------------------------- /csp_gateway/server/config/modules.yaml: -------------------------------------------------------------------------------- 1 | # Examples 2 | example_module: 3 | _target_: csp_gateway.server.demo.ExampleModule 4 | 5 | example_custom_table: 6 | _target_: csp_gateway.server.demo.ExampleModuleCustomTable 7 | 8 | 9 | # Controls 10 | mount_controls: 11 | _target_: csp_gateway.MountControls 12 | 13 | 14 | # Web Extras 15 | mount_outputs: 16 | _target_: csp_gateway.MountOutputsFolder 17 | 18 | mount_outputs_folder: 19 | _target_: csp_gateway.MountOutputsFolder 20 | 21 | mount_channels_graph: 22 | _target_: csp_gateway.MountChannelsGraph 23 | 24 | mount_perspective_tables: 25 | _target_: csp_gateway.MountPerspectiveTables 26 | 27 | 28 | # REST/WS/Auth 29 | mount_rest_routes: 30 | _target_: csp_gateway.MountRestRoutes 31 | 32 | mount_websocket_routes: 33 | _target_: csp_gateway.MountWebSocketRoutes 34 | 35 | 36 | mount_api_key_middleware: 37 | _target_: csp_gateway.MountAPIKeyMiddleware 38 | -------------------------------------------------------------------------------- /csp_gateway/server/demo/__init__.py: -------------------------------------------------------------------------------- 1 | from .omnibus import * 2 | -------------------------------------------------------------------------------- /csp_gateway/server/demo/__main__.py: -------------------------------------------------------------------------------- 1 | from logging import INFO, basicConfig 2 | from pathlib import Path 3 | 4 | from csp_gateway import ( 5 | Gateway, 6 | GatewaySettings, 7 | MountPerspectiveTables, 8 | MountRestRoutes, 9 | ) 10 | from csp_gateway.server.config import load_gateway 11 | 12 | from .simple import ExampleGatewayChannels, ExampleModule 13 | 14 | # csp-gateway is configured as a hydra application, 15 | # but it can also be instantiated directly as we do so here: 16 | 17 | # instantiate gateway 18 | gateway = Gateway( 19 | settings=GatewaySettings(), 20 | modules=[ 21 | ExampleModule(), 22 | MountPerspectiveTables(), 23 | MountRestRoutes(force_mount_all=True), 24 | ], 25 | channels=ExampleGatewayChannels(), 26 | ) 27 | 28 | if __name__ == "__main__": 29 | # To run, we could run our object directly: 30 | # gateway.start(rest=True, ui=True) 31 | 32 | # But instead, lets run the same code via hydra 33 | # We can use our own custom config, in config/demo.yaml 34 | # which inherits from csp-gateway's example config. 35 | # 36 | # With hydra, we can easily construct hierarchichal, 37 | # extensible configurations for all our modules! 38 | gateway = load_gateway( 39 | overrides=["+config=demo"], 40 | config_dir=Path(__file__).parent, 41 | ) 42 | 43 | # Set our log level to info. If using hydra, 44 | # we have even more configuration options at our disposal 45 | basicConfig(level=INFO) 46 | 47 | # Start the gateway 48 | gateway.start(rest=True, ui=True) 49 | 50 | # You can also run this directly via cli 51 | # > pip install csp-gateway 52 | # > csp-gateway-start --config-dir=csp_gateway/server/demo +config=demo 53 | 54 | # For more a more complicated example, see `./omnibus.py` 55 | -------------------------------------------------------------------------------- /csp_gateway/server/demo/config/demo.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | defaults: 3 | - /gateway: demo 4 | - _self_ 5 | 6 | # csp-gateway-start --config-dir=csp_gateway/server/demo +config=demo 7 | 8 | port: 8000 9 | -------------------------------------------------------------------------------- /csp_gateway/server/demo/config/omnibus.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | defaults: 3 | - /gateway: omnibus 4 | - _self_ 5 | 6 | # csp-gateway-start --config-dir=csp_gateway/server/omnibus +config=omnibus 7 | 8 | authenticate: False 9 | port: 8000 10 | -------------------------------------------------------------------------------- /csp_gateway/server/demo/simple.py: -------------------------------------------------------------------------------- 1 | """ 2 | A Demo CSP Gateway application 3 | """ 4 | 5 | from datetime import timedelta 6 | 7 | import csp 8 | from csp import ts 9 | 10 | from csp_gateway import GatewayChannels, GatewayModule, GatewayStruct 11 | 12 | # Let's walk through a worked example. 13 | 14 | # First, we want to define the edges that will be available to our modules. 15 | # In `csp-gateway` dialect, these deferred edges are called `channels`. 16 | # Let's define a few `csp.Struct` to go with them. 17 | 18 | # NOTE: See __main__.py for the entry point to run this example 19 | 20 | __all__ = ( 21 | "ExampleData", 22 | "ExampleGatewayChannels", 23 | "ExampleModule", 24 | ) 25 | 26 | 27 | class ExampleData(GatewayStruct): 28 | x: int 29 | 30 | 31 | # `MyGatewayChannels` is the collection of lazily-connected `csp` edges that will be provided to all modules in our graph. 32 | # The modules in our graph are `pydantic`-wrapped `csp` modules, that use the `channel` APIs to connect to edges. 33 | # Let's define a simple one. 34 | 35 | 36 | class ExampleGatewayChannels(GatewayChannels): 37 | example: ts[ExampleData] = None 38 | 39 | 40 | # A `GatewayModule` has two important features. 41 | # First, it is a typed `pydantic` model, so you can define attributes in the usual `pydantic` way. 42 | # Second, it has a `connect` method that will be provided the `GatewayChannels` instance when the graph is eventually wired together. 43 | # You can use `get_channel` and `set_channel` to read-from and publish-to `csp` edges, respectively. 44 | 45 | 46 | class ExampleModule(GatewayModule): 47 | interval: timedelta = timedelta(seconds=1) 48 | 49 | # An example module that ticks some data in a struct 50 | @csp.node 51 | def subscribe(self, trigger: ts[bool]) -> ts[ExampleData]: 52 | with csp.state(): 53 | last_x = 0 54 | if csp.ticked(trigger): 55 | last_x += 1 56 | return ExampleData(x=last_x) 57 | 58 | def connect(self, channels: ExampleGatewayChannels): 59 | # Create some CSP data streams 60 | data = self.subscribe(csp.timer(interval=self.interval, value=True)) 61 | 62 | # Channels set via `set_channel` 63 | channels.set_channel(ExampleGatewayChannels.example, data) 64 | -------------------------------------------------------------------------------- /csp_gateway/server/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | from .csp import * 2 | from .gateway import * 3 | -------------------------------------------------------------------------------- /csp_gateway/server/gateway/csp/__init__.py: -------------------------------------------------------------------------------- 1 | from .channels import Channels, ChannelsType 2 | from .factory import ChannelsFactory 3 | from .futures import ( 4 | AsyncFutureAdapter, 5 | ConcurrentFutureAdapter, 6 | FutureAdapter, 7 | named_on_request_node, 8 | named_on_request_node_dict_basket, 9 | on_request_node, 10 | on_request_node_dict_basket, 11 | ) 12 | from .module import Module 13 | from .state import * 14 | -------------------------------------------------------------------------------- /csp_gateway/server/gateway/csp/factory.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Generic, List, Optional, Type 3 | 4 | from ccflow import BaseModel 5 | from csp.impl.enum import Enum 6 | from csp.impl.genericpushadapter import GenericPushAdapter 7 | from pydantic import Field 8 | 9 | from csp_gateway.utils import get_dict_basket_key_type 10 | 11 | from .channels import ChannelsType 12 | from .module import Module 13 | 14 | 15 | class ChannelsFactory(BaseModel, Generic[ChannelsType]): 16 | model_config = dict(arbitrary_types_allowed=True) # (for FeedbackOutputDef) 17 | 18 | modules: List[Module[ChannelsType]] = Field( 19 | default_factory=list, description="The list of modules that will operate on the channels to build the csp graph." 20 | ) 21 | channels: ChannelsType = Field( 22 | default=None, 23 | description="An instance of the Channels to be provided by the user for the Gateway. One can think of this as the internal message bus topics for the Gateway, " 24 | "even though there is no message bus, just named edges in a csp graph.", 25 | ) 26 | channels_model: Type[ChannelsType] = Field( # type: ignore[misc] 27 | default=ChannelsType, 28 | description="The type of the channels. Users of a `Gateway` are expected to pass `channels`, and `channels_model` will" 29 | "be automatically inferred from the type. Developers can subclass `Gateway` and set the default value of" 30 | "`channels_model` to be the specific type of channels that users must provide.", 31 | ) 32 | block_set_channels_until: Optional[datetime] = Field( 33 | default=None, 34 | description=""" 35 | This determines the csp time at which modules can start sending data to channels. 36 | This can be overriden on a per module basis, to allow some modules to send data to channels. 37 | """, 38 | ) 39 | 40 | def build(self, channels: ChannelsType) -> ChannelsType: 41 | # First update the channels model type so we use the correct type, 42 | # important because we'll actually want to subclass the base GatewayChannels 43 | self.channels_model = channels.__class__ 44 | 45 | # Collect dynamic keys from the channels 46 | if dynamic_keys := channels.dynamic_keys(): 47 | for field, keys in dynamic_keys.items(): 48 | channels._dynamic_keys[field].update((k, None) for k in keys) 49 | 50 | enabled_modules = [node for node in self.modules if not node.disable] 51 | # Collect dynamic keys for each node 52 | for node in enabled_modules: 53 | if dynamic_keys := node.dynamic_keys(): 54 | for field, keys in dynamic_keys.items(): 55 | channels._dynamic_keys[field].update((k, None) for k in keys) 56 | 57 | if self.block_set_channels_until is not None: 58 | channels._block_set_channels_until = self.block_set_channels_until 59 | 60 | # Wire in each edge Provider. 61 | # The implementation of set_channel will handle multiplexing streams 62 | for node in enabled_modules: 63 | with channels._connection_context(node): 64 | node.connect(channels) 65 | 66 | # Connect to web app if it exists 67 | if self.web_app: # type: ignore[attr-defined] 68 | node.rest(self.web_app) # type: ignore[attr-defined] 69 | 70 | # Now wire in the signals 71 | # first pass is for any baskets 72 | for (field, _indexer), push_adapter in channels._send_channels.items(): 73 | if isinstance(push_adapter, tuple): 74 | # dict basket, plug in now that we should know all the 75 | # possible keys 76 | key_type = get_dict_basket_key_type(channels.get_outer_type(field)) 77 | if isinstance(key_type, type) and issubclass(key_type, Enum): 78 | for enumfield in key_type: 79 | channels.add_send_channel(field, enumfield) 80 | 81 | # add for whole basket ticks 82 | channels._add_send_channel_dict_basket(field, key_type) 83 | else: 84 | for key in channels._dynamic_keys.get(field, []): 85 | channels.add_send_channel(field, key) 86 | 87 | # add for whole basket ticks 88 | channels._add_send_channel_dict_basket(field, channels._dynamic_keys.get(field, [])) 89 | 90 | # second pass to finish connecting in wires 91 | for (field, indexer), push_adapter in channels._send_channels.items(): 92 | with channels._connection_context(f"Send[{field}]{f'<{indexer}>' if indexer else ''}"): 93 | # Add it as an edge on the StreamGroup 94 | if isinstance(push_adapter, GenericPushAdapter): 95 | channels.set_channel(field, push_adapter.out(), indexer=indexer) 96 | else: 97 | # basket, first item of tuple is generic channel, 98 | # second item is output 99 | channels.set_channel(field, push_adapter[1], indexer=indexer) 100 | 101 | # Do any post work thats necessary 102 | channels._finalize() 103 | return channels 104 | -------------------------------------------------------------------------------- /csp_gateway/server/gateway/csp/futures/__init__.py: -------------------------------------------------------------------------------- 1 | from .adapter import ( 2 | AsyncFutureAdapter, 3 | ConcurrentFutureAdapter, 4 | FutureAdapter, 5 | named_on_request_node, 6 | named_on_request_node_dict_basket, 7 | named_wait_for_next_node, 8 | named_wait_for_next_node_dict_basket, 9 | on_request_node, 10 | on_request_node_dict_basket, 11 | wait_for_next_dict_basket, 12 | wait_for_next_node, 13 | ) 14 | -------------------------------------------------------------------------------- /csp_gateway/server/gateway/csp/module.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import typing 3 | from datetime import datetime 4 | from typing import Any, Dict, Generic, List, Optional, Set, Type, Union 5 | 6 | from ccflow import BaseModel 7 | from pydantic import Field, TypeAdapter, model_validator 8 | 9 | from csp_gateway.server.shared import ChannelSelection 10 | from csp_gateway.utils import GatewayStruct 11 | 12 | from .channels import ChannelsType 13 | 14 | if typing.TYPE_CHECKING: 15 | from csp_gateway.server import GatewayWebApp 16 | 17 | 18 | class Module(BaseModel, Generic[ChannelsType], abc.ABC): 19 | model_config = {"arbitrary_types_allowed": True} 20 | 21 | requires: Optional[ChannelSelection] = None 22 | disable: bool = False 23 | block_set_channels_until: Optional[datetime] = Field( 24 | default=None, 25 | description=""" 26 | This determines the csp time at which this module can start sending data to channels. 27 | This value overrides any gateway-level blocks imposed. 28 | """, 29 | ) 30 | 31 | @abc.abstractmethod 32 | def connect(self, Channels: ChannelsType) -> None: ... 33 | 34 | def rest(self, app: "GatewayWebApp") -> None: ... 35 | 36 | @abc.abstractmethod 37 | def shutdown(self) -> None: ... 38 | 39 | def dynamic_keys(self) -> Optional[Dict[str, List[Any]]]: ... 40 | 41 | def dynamic_channels(self) -> Optional[Dict[str, Union[Type[GatewayStruct], Type[List[GatewayStruct]]]]]: 42 | """ 43 | Channels that this module dynamically adds to the gateway channels when this module is included into the gateway. 44 | 45 | Returns: 46 | Dictionary keyed by channel name and type of the timeseries of the channel as values. 47 | """ 48 | ... 49 | 50 | def dynamic_state_channels(self) -> Optional[Set[str]]: 51 | """ 52 | The set of dynamic channels that have state. 53 | """ 54 | ... 55 | 56 | # @abc.abstractmethod 57 | # def subscribe(self): 58 | # ... 59 | 60 | def __eq__(self, other): 61 | # Override equality because occasionally, Modules will contain fields with non-standard equality methods 62 | # i.e. numpy arrays or csp edges. 63 | # Without overriding, these types will prevent the modules from being compared with each other 64 | # which is needed for the dependency resolutions 65 | return id(self) == id(other) 66 | 67 | # See https://docs.pydantic.dev/latest/concepts/validators/#validation-of-default-values 68 | @model_validator(mode="before") 69 | def validate_requires(cls, v): 70 | requires = v.get("requires", cls.model_fields["requires"].default) 71 | v["requires"] = TypeAdapter(ChannelSelection).validate_python(requires) 72 | return v 73 | -------------------------------------------------------------------------------- /csp_gateway/server/middleware/__init__.py: -------------------------------------------------------------------------------- 1 | from .api_key import MountAPIKeyMiddleware 2 | -------------------------------------------------------------------------------- /csp_gateway/server/middleware/hacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/server/middleware/hacks/__init__.py -------------------------------------------------------------------------------- /csp_gateway/server/middleware/hacks/api_key_middleware_websocket_fix/__init__.py: -------------------------------------------------------------------------------- 1 | # https://github.com/tiangolo/fastapi/pull/10147 2 | -------------------------------------------------------------------------------- /csp_gateway/server/middleware/hacks/api_key_middleware_websocket_fix/api_key.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from fastapi.openapi.models import APIKey, APIKeyIn 4 | from fastapi.security.base import SecurityBase 5 | from starlette.exceptions import HTTPException 6 | from starlette.requests import HTTPConnection 7 | from starlette.status import HTTP_403_FORBIDDEN 8 | 9 | from .utils import handle_exc_for_ws 10 | 11 | 12 | class APIKeyBase(SecurityBase): 13 | pass 14 | 15 | 16 | class APIKeyQuery(APIKeyBase): 17 | def __init__( 18 | self, 19 | *, 20 | name: str, 21 | scheme_name: Optional[str] = None, 22 | description: Optional[str] = None, 23 | auto_error: bool = True, 24 | ): 25 | self.model: APIKey = APIKey( 26 | **{"in": APIKeyIn.query}, # type: ignore[arg-type] 27 | name=name, 28 | description=description, 29 | ) 30 | self.scheme_name = scheme_name or self.__class__.__name__ 31 | self.auto_error = auto_error 32 | 33 | @handle_exc_for_ws 34 | async def __call__(self, request: HTTPConnection) -> Optional[str]: 35 | api_key = request.query_params.get(self.model.name) 36 | if not api_key: 37 | if self.auto_error: 38 | raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Not authenticated") 39 | else: 40 | return None 41 | return api_key 42 | 43 | 44 | class APIKeyHeader(APIKeyBase): 45 | def __init__( 46 | self, 47 | *, 48 | name: str, 49 | scheme_name: Optional[str] = None, 50 | description: Optional[str] = None, 51 | auto_error: bool = True, 52 | ): 53 | self.model: APIKey = APIKey( 54 | **{"in": APIKeyIn.header}, # type: ignore[arg-type] 55 | name=name, 56 | description=description, 57 | ) 58 | self.scheme_name = scheme_name or self.__class__.__name__ 59 | self.auto_error = auto_error 60 | 61 | @handle_exc_for_ws 62 | async def __call__(self, request: HTTPConnection) -> Optional[str]: 63 | api_key = request.headers.get(self.model.name) 64 | if not api_key: 65 | if self.auto_error: 66 | raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Not authenticated") 67 | else: 68 | return None 69 | return api_key 70 | 71 | 72 | class APIKeyCookie(APIKeyBase): 73 | def __init__( 74 | self, 75 | *, 76 | name: str, 77 | scheme_name: Optional[str] = None, 78 | description: Optional[str] = None, 79 | auto_error: bool = True, 80 | ): 81 | self.model: APIKey = APIKey( 82 | **{"in": APIKeyIn.cookie}, # type: ignore[arg-type] 83 | name=name, 84 | description=description, 85 | ) 86 | self.scheme_name = scheme_name or self.__class__.__name__ 87 | self.auto_error = auto_error 88 | 89 | @handle_exc_for_ws 90 | async def __call__(self, request: HTTPConnection) -> Optional[str]: 91 | api_key = request.cookies.get(self.model.name) 92 | if not api_key: 93 | if self.auto_error: 94 | raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Not authenticated") 95 | else: 96 | return None 97 | return api_key 98 | -------------------------------------------------------------------------------- /csp_gateway/server/middleware/hacks/api_key_middleware_websocket_fix/utils.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from typing import Any, Awaitable, Callable, Optional, Tuple, TypeVar 3 | 4 | from fastapi.exceptions import HTTPException, WebSocketException 5 | from starlette.requests import HTTPConnection 6 | from starlette.status import WS_1008_POLICY_VIOLATION 7 | from starlette.websockets import WebSocket 8 | 9 | 10 | def get_authorization_scheme_param( 11 | authorization_header_value: Optional[str], 12 | ) -> Tuple[str, str]: 13 | if not authorization_header_value: 14 | return "", "" 15 | scheme, _, param = authorization_header_value.partition(" ") 16 | return scheme, param 17 | 18 | 19 | _SecurityDepFunc = TypeVar("_SecurityDepFunc", bound=Callable[[Any, HTTPConnection], Awaitable[Any]]) 20 | 21 | 22 | def handle_exc_for_ws(func: _SecurityDepFunc) -> _SecurityDepFunc: 23 | @wraps(func) 24 | async def wrapper(self: Any, request: HTTPConnection) -> Any: 25 | try: 26 | return await func(self, request) 27 | except HTTPException as e: 28 | if not isinstance(request, WebSocket): 29 | raise e 30 | # close before accepted with result a HTTP 403 so the exception argument is ignored 31 | # ref: https://asgi.readthedocs.io/en/latest/specs/www.html#close-send-event 32 | raise WebSocketException(code=WS_1008_POLICY_VIOLATION, reason=e.detail) from None 33 | 34 | return wrapper # type: ignore 35 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/__init__.py: -------------------------------------------------------------------------------- 1 | from .controls import * 2 | from .initializer import * 3 | from .io import * 4 | from .kafka import * 5 | from .logging import * 6 | from .mirror import * 7 | from .web import * 8 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/controls/__init__.py: -------------------------------------------------------------------------------- 1 | from .controls import MountControls 2 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/controls/controls.py: -------------------------------------------------------------------------------- 1 | import getpass 2 | import resource 3 | import socket 4 | import threading 5 | from datetime import datetime 6 | 7 | import csp 8 | import psutil 9 | from csp import ts 10 | from fastapi import FastAPI 11 | 12 | from csp_gateway.server import GatewayChannels, GatewayModule 13 | 14 | # separate to avoid circular 15 | from csp_gateway.server.web import GatewayWebApp 16 | from csp_gateway.utils import Controls 17 | 18 | _HOSTNAME = socket.gethostname() 19 | _USER = getpass.getuser() 20 | 21 | 22 | class MountControls(GatewayModule): 23 | app: GatewayWebApp = None 24 | fastapi: FastAPI = None 25 | 26 | def connect(self, channels: GatewayChannels) -> None: 27 | self.subscribe(channels.get_channel("controls")) 28 | channels.add_send_channel("controls") 29 | 30 | def rest(self, app: GatewayWebApp) -> None: 31 | self.app = app 32 | self.fastapi = app.get_fastapi() 33 | 34 | @csp.node 35 | def manage_controls(self, data: ts[Controls]): 36 | if csp.ticked(data): 37 | # TODO better check if "seen" 38 | if data.name == "heartbeat": 39 | # don't have to do anything 40 | data.status = "ok" 41 | 42 | elif data.name == "stats" and not data.data: 43 | stats = {} 44 | 45 | # Machine information 46 | stats["cpu"] = psutil.cpu_percent() 47 | stats["memory"] = psutil.virtual_memory().percent 48 | stats["memory-total"] = round( 49 | psutil.virtual_memory().available * 100 / psutil.virtual_memory().total, 50 | 2, 51 | ) 52 | 53 | # Process and thread information 54 | current_process = psutil.Process() 55 | stats["pid"] = current_process.pid 56 | stats["active_threads"] = threading.active_count() 57 | 58 | # Get max threads from ulimit 59 | _, hard_limit = resource.getrlimit(resource.RLIMIT_NPROC) 60 | stats["max_threads"] = hard_limit if hard_limit != resource.RLIM_INFINITY else "unlimited" 61 | 62 | # Time information 63 | stats["now"] = datetime.utcnow() 64 | stats["csp-now"] = csp.now() 65 | 66 | stats["host"] = _HOSTNAME 67 | stats["user"] = _USER 68 | 69 | data.data = stats 70 | data.status = "ok" # we mark as ok at the end only after we have all the data 71 | 72 | @csp.graph 73 | def subscribe(self, data: ts[Controls]): 74 | self.manage_controls(data) 75 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/initializer.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from typing import Any, Dict 3 | 4 | import csp 5 | 6 | from csp_gateway.server import ChannelsType, GatewayModule 7 | from csp_gateway.server.gateway.csp.channels import _CSP_ENGINE_CYCLE_TIMESTAMP_FIELD 8 | from csp_gateway.server.shared.engine_replay import JSONConverter 9 | 10 | __all__ = ("Initialize",) 11 | 12 | 13 | class Initialize(GatewayModule): 14 | """A generic initializer, uses pydantic to parse objects into the correct type for pushing into a Gateway at startup.""" 15 | 16 | channel: str 17 | value: Any = None 18 | # This is effectively a python dict/list representing the structure we want to push into the graph 19 | # (This structure can be defined in a yaml file using hydra). 20 | # It should have the structure of the channel (note the caveat where `unroll` is set to True) 21 | # If the channel takes a csp.ts[GatewayStruct], then "value" should have the full structure of the 22 | # GatewayStruct. Refer to the tests for examples. 23 | seconds_offset: int = 0 24 | # How many seconds we want to delay before ticking the "value" into the graph. 25 | unroll: bool = False 26 | # This allows us to send multiple values, that get unrolled. 27 | # Setting this to True, the "value" must be a list of values to push into the graph. 28 | # For example, if a channel is type csp.ts[List[MyGatewayStruct]], then 29 | # "value" will be a list of GatewayStruct, if "unroll" = False. If "unroll" = True, 30 | # "value" should be a list of lists of GatewayStruct (where each entry in the outer list is one tick) 31 | 32 | @csp.node 33 | def tick_engine_cycle(self) -> csp.ts[Dict[str, object]]: 34 | with csp.alarms(): 35 | a_send = csp.alarm(object) 36 | 37 | with csp.start(): 38 | if not self.unroll: 39 | goal_dict = {self.channel: self.value} 40 | csp.schedule_alarm(a_send, timedelta(seconds=self.seconds_offset), goal_dict) 41 | else: 42 | for val in self.value: 43 | goal_dict = { 44 | self.channel: val, 45 | } 46 | csp.schedule_alarm(a_send, timedelta(seconds=self.seconds_offset), goal_dict) 47 | 48 | if csp.ticked(a_send): 49 | goal_dict = a_send 50 | goal_dict[_CSP_ENGINE_CYCLE_TIMESTAMP_FIELD] = csp.now() 51 | return goal_dict 52 | 53 | def connect(self, channels: ChannelsType): 54 | if self.value is not None: 55 | json_channel_converter = JSONConverter( 56 | channels=channels, 57 | decode_channels=[self.channel], 58 | encode_channels=[], 59 | log_lagging_engine_cycles=False, 60 | ) 61 | # The "decode" function decodes the dictionary representing the 62 | # channel tick and ticks it into the graph 63 | json_channel_converter.decode(self.tick_engine_cycle()) 64 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/io/__init__.py: -------------------------------------------------------------------------------- 1 | from .graph_output import * 2 | from .json import * 3 | from .json_pull_adapter import * 4 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/io/graph_output.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import csp 4 | from csp.impl.types.tstype import isTsType 5 | from pydantic import Field 6 | 7 | from csp_gateway.server import ChannelSelection, ChannelsType, GatewayModule 8 | from csp_gateway.utils import is_dict_basket 9 | 10 | __all__ = ("AddChannelsToGraphOutput",) 11 | 12 | 13 | class AddChannelsToGraphOutput(GatewayModule): 14 | selection: ChannelSelection = Field(default_factory=ChannelSelection) 15 | requires: Optional[ChannelSelection] = [] 16 | 17 | def connect(self, channels: ChannelsType): 18 | for field in self.selection.select_from(channels): 19 | outer_type = channels.get_outer_type(field) 20 | # list baskets not supported yet 21 | if is_dict_basket(outer_type): 22 | edge = channels.get_channel(field) 23 | for k, v in edge.items(): 24 | csp.add_graph_output(f"{field}[{k}]", v) 25 | elif isTsType(outer_type): 26 | edge = channels.get_channel(field) 27 | csp.add_graph_output(f"{field}", edge) 28 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/io/json.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from queue import Queue 3 | from threading import Thread 4 | from typing import Any, Dict, Union 5 | 6 | import csp 7 | from csp import ts 8 | 9 | from csp_gateway.server import EncodedEngineCycle 10 | from csp_gateway.server.shared.engine_replay import EngineReplay 11 | 12 | from .json_pull_adapter import JSONPullAdapter 13 | 14 | __all__ = ("ReplayEngineJSON",) 15 | 16 | 17 | class JSONWriterThread(Thread): 18 | def __init__(self, file_path: str, write_queue: Queue, file_mode: str): 19 | super().__init__() 20 | self.file_path = file_path 21 | self.write_queue = write_queue 22 | self.file_mode = file_mode 23 | 24 | def run(self): 25 | p = Path(self.file_path) 26 | p.parent.mkdir(parents=True, exist_ok=True) 27 | with p.open(self.file_mode) as f: 28 | while True: 29 | data = self.write_queue.get() 30 | f.write(data) 31 | f.flush() 32 | self.write_queue.task_done() 33 | 34 | 35 | class ReplayEngineJSON(EngineReplay): 36 | overwrite_if_writing: bool = False 37 | filename: str 38 | 39 | @csp.node 40 | def _dump_to_json(self, to_store: ts[EncodedEngineCycle]): 41 | with csp.state(): 42 | s_json_writer_thread = None 43 | s_queue = None 44 | 45 | with csp.start(): 46 | file_mode = "w" if self.overwrite_if_writing else "a" 47 | s_queue = Queue() 48 | s_json_writer_thread = JSONWriterThread(self.filename, s_queue, file_mode) 49 | s_json_writer_thread.daemon = True 50 | s_json_writer_thread.start() 51 | 52 | with csp.stop(): 53 | s_queue.join() 54 | 55 | if csp.ticked(to_store): 56 | s_queue.put(to_store.encoding) 57 | 58 | def subscribe(self) -> Union[ts[str], ts[Dict[str, Any]]]: 59 | return JSONPullAdapter(self.filename) 60 | 61 | def publish(self, encoded_channels: ts[EncodedEngineCycle]) -> None: 62 | self._dump_to_json(encoded_channels) 63 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/io/json_pull_adapter.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import orjson 4 | from csp import ts 5 | from csp.impl.pulladapter import PullInputAdapter 6 | from csp.impl.wiring import py_pull_adapter_def 7 | 8 | from csp_gateway.server.gateway.csp.channels import _CSP_ENGINE_CYCLE_TIMESTAMP_FIELD 9 | 10 | __all__ = ("JSONPullAdapter",) 11 | 12 | 13 | # The Impl object is created at runtime when the graph is converted into the runtime engine 14 | # it does not exist at graph building time! 15 | class JSONPullAdapterImpl(PullInputAdapter): 16 | def __init__(self, filename: str): 17 | self._filename = filename 18 | self._file = None 19 | self._next_row = None 20 | super().__init__() 21 | 22 | def start(self, start_time, end_time): 23 | self._file = open(self._filename, "r") 24 | for line in self._file: 25 | self._next_row = line 26 | json_dict = orjson.loads(line) 27 | time_str = json_dict[_CSP_ENGINE_CYCLE_TIMESTAMP_FIELD] 28 | # since csp engine times are in utc we can just 29 | # drop the timezone info to compare it to start_time 30 | time = datetime.fromisoformat(time_str).replace(tzinfo=None) 31 | if time >= start_time: 32 | break 33 | 34 | super().start(start_time, end_time) 35 | 36 | def stop(self): 37 | self._file.close() 38 | 39 | def next(self): 40 | if self._next_row is None: 41 | return None 42 | 43 | while True: 44 | json_dict = orjson.loads(self._next_row) 45 | time_str = json_dict[_CSP_ENGINE_CYCLE_TIMESTAMP_FIELD] 46 | time = datetime.fromisoformat(time_str).replace(tzinfo=None) 47 | try: 48 | self._next_row = next(self._file) 49 | except StopIteration: 50 | self._next_row = None 51 | return time, json_dict 52 | 53 | 54 | # MyPullAdapter is the graph-building time construct. This is simply a representation of what the 55 | # input adapter is and how to create it, including the Impl to use and arguments to pass into it upon construction 56 | JSONPullAdapter = py_pull_adapter_def("JSONPullAdapter", JSONPullAdapterImpl, ts[dict], filename=str) 57 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/kafka/__init__.py: -------------------------------------------------------------------------------- 1 | from csp.adapters.kafka import KafkaAdapterManager 2 | 3 | from .kafka import * 4 | from .utils import * 5 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/kafka/utils.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import Any, List, Union, get_args, get_origin 3 | 4 | import csp 5 | from ccflow import BaseModel 6 | from csp import ts 7 | 8 | from csp_gateway.utils import GatewayStruct 9 | 10 | __all__ = ("KafkaChannelProcessor",) 11 | 12 | 13 | class KafkaChannelProcessor(BaseModel, abc.ABC): 14 | """ 15 | Process channel inputs before sending to Kafka, or before propagating into the graph. 16 | """ 17 | 18 | @abc.abstractmethod 19 | def process(self, obj: Union[List[GatewayStruct], GatewayStruct], topic: str, key: str) -> Any: 20 | raise NotImplementedError() 21 | 22 | def apply_process(self, typ: Any, obj: ts[object], topic: str, key: str) -> ts[object]: 23 | """Applies a function to a ticking edge. 24 | 25 | Args: 26 | typ: The class of the processed object, according to csp. 27 | obj: The ticking edge that ticks values of the object 28 | topic: The Kafka topic 29 | key: The Kafka key 30 | 31 | Returns: 32 | A ticking edge with the processed output, of type specified by `typ` 33 | """ 34 | 35 | actual_type = typ 36 | # csp doesnt like typing.List, so we replace it if we see it 37 | if get_origin(typ) is list: 38 | arg = get_args(typ)[0] 39 | actual_type = [arg] 40 | 41 | res = csp.apply(obj, lambda x, topic=topic, key=key: self.process(x, topic, key), object) 42 | flag = csp.apply(res, lambda x: x is not None, bool) 43 | 44 | filtered_res = csp.filter(flag, res) 45 | # safer than static_cast since csp does type checking at runtime for us 46 | return csp.dynamic_cast(filtered_res, actual_type) 47 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/logging/__init__.py: -------------------------------------------------------------------------------- 1 | from .logging import LogChannels 2 | from .printing import PrintChannels 3 | 4 | try: 5 | from .symphony import PublishSymphony 6 | except ImportError: 7 | pass 8 | 9 | try: 10 | from .datadog import PublishDatadog 11 | except ImportError: 12 | pass 13 | 14 | try: 15 | from .opsgenie import PublishOpsGenie 16 | except ImportError: 17 | pass 18 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/logging/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Optional 3 | 4 | import csp 5 | from pydantic import Field 6 | 7 | from csp_gateway.server import ChannelSelection, ChannelsType, GatewayModule 8 | 9 | 10 | class LogChannels(GatewayModule): 11 | selection: ChannelSelection = Field(default_factory=ChannelSelection) 12 | log_states: bool = False 13 | log_level: int = logging.INFO 14 | log_name: str = str(__name__) 15 | requires: Optional[ChannelSelection] = [] 16 | 17 | def connect(self, channels: ChannelsType): 18 | logger_to_use = logging.getLogger(self.log_name) 19 | 20 | for field in self.selection.select_from(channels, state_channels=self.log_states): 21 | data = channels.get_channel(field) 22 | # list baskets not supported yet 23 | if isinstance(data, dict): 24 | for k, v in data.items(): 25 | csp.log(self.log_level, f"{field}[{k}]", v, logger=logger_to_use) 26 | else: 27 | edge = channels.get_channel(field) 28 | csp.log(self.log_level, field, edge, logger=logger_to_use) 29 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/logging/printing.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import csp 4 | from pydantic import Field 5 | 6 | from csp_gateway.server import ChannelSelection, ChannelsType, GatewayModule 7 | 8 | 9 | class PrintChannels(GatewayModule): 10 | """ 11 | Gateway Module for printing channels, which could be useful for debugging. 12 | There exists a designated logging node class `LogChannels` which is preferred 13 | and can specify a logger to use by name. 14 | """ 15 | 16 | selection: ChannelSelection = Field(default_factory=ChannelSelection) 17 | requires: Optional[ChannelSelection] = [] 18 | 19 | def connect(self, channels: ChannelsType): 20 | for field in self.selection.select_from(channels): 21 | csp.print(f"{field}", channels.get_channel(field)) 22 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/logging/symphony.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List, Tuple, TypeVar 3 | 4 | import csp 5 | from csp import ts 6 | from csp_adapter_symphony import SymphonyAdapter, SymphonyMessage 7 | 8 | from csp_gateway.server import GatewayChannels, GatewayModule 9 | 10 | T = TypeVar("T") 11 | 12 | 13 | class PublishSymphony(GatewayModule): 14 | """ 15 | Takes a set of channels (selections) and turns them into symphony messages. 16 | 17 | To get setup with symphony: 18 | 1. Email 'Tech - Symphony Admin' to create a new symphony account for your bot and for them to generate a pfx keyc. 19 | You should include the following in your email: 20 | * Bot Display Name (AAAA Alerts) 21 | * Distribution Email Address (e.g. trading-AAAA@cubistsystematic.com) 22 | 2. Create a key.pem and crt.pem file locally (these will be used to authenticate your bot on your machine of choice): 23 | ( replace pfx_file with your filename) 24 | 25 | ```bash 26 | PFX="mypfxfile.pfx" 27 | openssl pkcs12 -in $PFX -out key.pem -nocerts -nodes 28 | openssl pkcs12 -in $PFX -out crt.pem -clcerts -nokeys 29 | ``` 30 | 31 | 3. On your target machine move your newly created key.pem and crt.pem files to an accessible directory. 32 | 33 | Args 34 | ----- 35 | cert_path: str 36 | path to the crt.pem file (absolute) 37 | key_path: str 38 | path to the key.pem file (absolute) 39 | room_name: str 40 | name of the chat room with your bot 41 | user: str 42 | display name of your bot (from above) 43 | selections: List[str] 44 | list of channels to push to your symphony room 45 | 46 | Configuration 47 | ------------- 48 | To be included in your watchtower config. 49 | 50 | ```yaml 51 | symphony_alerts: 52 | _target_: csp_gateway.PublishSymphony 53 | cert_path: /my/path/to/crt.pem 54 | key_path: /my/path/to/key.pem 55 | room_name: "My Symphony Room" 56 | user: "My Symphony Bot" 57 | ``` 58 | 59 | """ 60 | 61 | cert_path: str 62 | key_path: str 63 | room_name: str 64 | user: str 65 | 66 | selections: List[str] = [] 67 | 68 | def get_cert_and_key(self) -> Tuple[str, str]: 69 | with open(self.cert_path, "r") as f: 70 | cert = f.read() 71 | with open(self.key_path, "r") as f: 72 | key = f.read() 73 | return cert, key 74 | 75 | def connect(self, channels: GatewayChannels): 76 | cert, key = self.get_cert_and_key() 77 | symphony_manager = SymphonyAdapter(cert, key) 78 | for channel in self.selections: 79 | sub = self.subscribe_channel(channels.get_channel(channel)) 80 | symphony_manager.publish(csp.unroll(sub.messages)) 81 | 82 | @csp.node 83 | def subscribe_channel(self, channel: ts["T"]) -> csp.Outputs(messages=ts[List[SymphonyMessage]]): 84 | if csp.ticked(channel): 85 | msgs = [] 86 | # TODO: Not sure how to manage a dict basket here 87 | if isinstance(channel, list): 88 | msgs.extend( 89 | [ 90 | SymphonyMessage( 91 | user=self.user, 92 | room=self.room_name, 93 | msg=json.dumps(c.to_dict(), default=str), 94 | ) 95 | for c in channel 96 | ] 97 | ) 98 | else: 99 | msgs.append( 100 | SymphonyMessage( 101 | user=self.user, 102 | room=self.room_name, 103 | msg=json.dumps(channel.to_dict(), default=str), 104 | ) 105 | ) 106 | csp.output(messages=msgs) 107 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/mirror.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional, Tuple 2 | 3 | from pydantic import Field, field_validator 4 | 5 | from csp_gateway.server import ChannelSelection, ChannelsType, GatewayModule 6 | from csp_gateway.server.shared.engine_replay import EngineReplay 7 | from csp_gateway.utils import ReadWriteMode 8 | 9 | 10 | class Mirror(GatewayModule): 11 | """ 12 | Designed to mirror the ticks of a gateway instance from a given source. 13 | Wires in the state channels to allow state queries. 14 | """ 15 | 16 | requires: ChannelSelection = [] 17 | selection: ChannelSelection = Field(default_factory=ChannelSelection) 18 | encode_selection: Optional[ChannelSelection] = Field( 19 | default=None, 20 | description=("Optional selection that can be specified to override the selection to specify channels only for encoding."), 21 | ) 22 | decode_selection: Optional[ChannelSelection] = Field( 23 | default=None, 24 | description=("Optional selection that can be specified to override the selection to specify channels only for decoding."), 25 | ) 26 | mirror_source: EngineReplay 27 | state_channels: Dict[str, Tuple[str, ...]] = Field( 28 | default_factory=dict, 29 | description="Set which channels should be state channels and what their keyby value is", 30 | ) 31 | 32 | @field_validator("state_channels", mode="before") 33 | def validate_state_channels_for_replay(cls, v): 34 | return {state: (tuple(keyby) if isinstance(keyby, (list, tuple)) else (keyby,)) for state, keyby in v.items()} 35 | 36 | @field_validator("mirror_source", mode="after") 37 | def set_same_channels(cls, v, info): 38 | v.requires = info.data.get("requires", ChannelSelection(include=[])) 39 | v.selection = info.data.get("selection", ChannelSelection()) 40 | v.read_write_mode = ReadWriteMode.READ 41 | v.encode_selection = info.data.get("encode_selection") 42 | v.decode_selection = info.data.get("decode_selection") 43 | return v 44 | 45 | def connect(self, channels: ChannelsType): 46 | for channel, keyby in self.state_channels.items(): 47 | channels.set_state(channel, keyby) 48 | # the requirements should be the same so 49 | # the channels context manager missing these updates 50 | # shouldn't matter 51 | self.mirror_source.connect(channels) 52 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/web/__init__.py: -------------------------------------------------------------------------------- 1 | from .channels_graph import MountChannelsGraph 2 | from .mount import MountRestRoutes 3 | from .mount_fields import MountFieldRestRoutes 4 | from .outputs import MountOutputsFolder 5 | from .perspective import * 6 | from .websocket import MountWebSocketRoutes 7 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/web/channels_graph.py: -------------------------------------------------------------------------------- 1 | from json import dumps 2 | from typing import Dict, List 3 | 4 | from fastapi import Request 5 | from fastapi.responses import HTMLResponse 6 | 7 | from csp_gateway.server import GatewayChannels, GatewayModule 8 | 9 | # separate to avoid circular 10 | from csp_gateway.server.web import GatewayWebApp 11 | 12 | 13 | class MountChannelsGraph(GatewayModule): 14 | route: str = "/channels_graph" 15 | 16 | def connect(self, channels: GatewayChannels) -> None: 17 | # NO-OP 18 | ... 19 | 20 | def rest(self, app: GatewayWebApp) -> None: 21 | api_router = app.get_router("api") 22 | app_router = app.get_router("app") 23 | 24 | # TODO subselect 25 | @api_router.get( 26 | self.route, 27 | response_model=Dict[str, Dict[str, List[str]]], 28 | tags=["Utility"], 29 | ) 30 | def channels_graph_data(request: Request) -> Dict[str, Dict[str, List[str]]]: 31 | """ 32 | This endpoint returns the structure of the GatewayChannels graph as a JSON object. 33 | It is used by the `Browse Channels Graph` endpoint to generate a nice, interactive view of the graph. 34 | 35 | Data is of the form: 36 | 37 | ``` 38 | { 39 | "": { 40 | "getters": [`GatewayModule`s that pull from that channel], 41 | "setters": [`GatewayModule`s that push to that channel] 42 | }, 43 | ... 44 | } 45 | ``` 46 | """ 47 | return request.app.gateway.channels.graph() 48 | 49 | @app_router.get("/channels_graph", response_class=HTMLResponse, tags=["Utility"]) 50 | def browse_channels_graph(request: Request): 51 | """ 52 | This endpoint is a small webpage that shows the dependency relationship of the GatewayChannels graph powering this API. 53 | """ 54 | channels_graph = request.app.gateway.channels.graph() 55 | return app.templates.TemplateResponse( 56 | "channels_graph.html.j2", 57 | {"request": request, "channels_graph": dumps(channels_graph)}, 58 | ) 59 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/web/mount.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pydantic import Field, model_validator 4 | 5 | from csp_gateway.server import ChannelSelection, GatewayChannels, GatewayModule 6 | 7 | # separate to avoid circular 8 | from csp_gateway.server.web import GatewayWebApp 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class MountRestRoutes(GatewayModule): 14 | force_mount_all: bool = Field( 15 | False, 16 | description="For debugging, will mount all rest routes for every channel, including state and send routes, even if not added by any modules", 17 | ) 18 | 19 | mount_last: ChannelSelection = ChannelSelection() 20 | mount_next: ChannelSelection = ChannelSelection() 21 | mount_send: ChannelSelection = ChannelSelection() 22 | mount_state: ChannelSelection = ChannelSelection() 23 | mount_lookup: ChannelSelection = ChannelSelection() 24 | 25 | @model_validator(mode="before") 26 | @classmethod 27 | def _deprecate_mount_all(cls, values): 28 | if (mount_all := values.pop("mount_all", None)) is not None: 29 | log.warning("mount_all is deprecated, please use force_mount_all instead") 30 | if "force_mount_all" not in values: 31 | values["force_mount_all"] = mount_all 32 | return values 33 | 34 | def connect(self, channels: GatewayChannels) -> None: 35 | # NO-OP 36 | ... 37 | 38 | def rest(self, app: GatewayWebApp) -> None: 39 | self._mount_last(app) 40 | self._mount_next(app) 41 | self._mount_send(app) 42 | self._mount_state(app) 43 | self._mount_lookup(app) 44 | 45 | def _mount_last(self, app: GatewayWebApp) -> None: 46 | selection = ChannelSelection() if self.force_mount_all else self.mount_last 47 | channels_set = set(selection.select_from(app.gateway.channels_model)) 48 | 49 | # Bind every wire 50 | for name in channels_set: 51 | # Install on router 52 | app.add_last_api(name) 53 | 54 | app.add_last_api_available_channels(channels_set) 55 | 56 | def _mount_next(self, app: GatewayWebApp) -> None: 57 | selection = ChannelSelection() if self.force_mount_all else self.mount_next 58 | channels_set = set(selection.select_from(app.gateway.channels_model)) 59 | 60 | # Bind every wire 61 | for name in channels_set: 62 | # Install on router 63 | app.add_next_api(name) 64 | 65 | app.add_next_api_available_channels(channels_set) 66 | 67 | def _mount_send(self, app: GatewayWebApp) -> None: 68 | selection = ChannelSelection() if self.force_mount_all else self.mount_send 69 | channels_set = set(selection.select_from(app.gateway.channels_model)) 70 | seen_channels = set() 71 | 72 | # Bind every wire 73 | if self.force_mount_all: 74 | for channel in channels_set: 75 | app.add_send_api(channel) 76 | seen_channels = channels_set 77 | 78 | else: 79 | for channel, _ in app.gateway.channels._send_channels.keys(): 80 | if channel in seen_channels or channel not in channels_set: 81 | continue 82 | seen_channels.add(channel) 83 | # Install on router 84 | app.add_send_api(channel) 85 | 86 | missing_channels = channels_set - seen_channels 87 | if missing_channels: 88 | log.info(f"Requested channels missing send routes are: {list(missing_channels)}") 89 | 90 | app.add_send_api_available_channels(seen_channels) 91 | 92 | def _mount_state(self, app: GatewayWebApp) -> None: 93 | selection = ChannelSelection() if self.force_mount_all else self.mount_state 94 | channels_set = set(selection.select_from(app.gateway.channels_model, state_channels=True)) 95 | seen_channels = set() 96 | 97 | # Bind every wire 98 | if self.force_mount_all: 99 | for state_channel in channels_set: 100 | app.add_state_api(state_channel) 101 | seen_channels = channels_set 102 | 103 | else: 104 | for state_channel, _ in app.gateway.channels._state_requests.keys(): 105 | if state_channel in seen_channels or state_channel not in channels_set: 106 | continue 107 | seen_channels.add(state_channel) 108 | # Install on router 109 | app.add_state_api(state_channel) 110 | 111 | missing_channels = channels_set - seen_channels 112 | if missing_channels: 113 | log.info(f"Requested channels missing state routes: {list(channel[2:] for channel in missing_channels)}") 114 | 115 | app.add_state_api_available_channels(seen_channels) 116 | 117 | def _mount_lookup(self, app: GatewayWebApp) -> None: 118 | selection = ChannelSelection() if self.force_mount_all else self.mount_lookup 119 | # Bind every wire 120 | for name in selection.select_from(app.gateway.channels_model): 121 | # Install on router 122 | app.add_lookup_api(name) 123 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/web/mount_fields.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Type, Union 2 | 3 | from fastapi import APIRouter, HTTPException, Request 4 | from pydantic import BaseModel, Field 5 | 6 | from csp_gateway.server import ChannelSelection, GatewayChannels, GatewayModule 7 | from csp_gateway.server.web import GatewayWebApp, get_default_responses 8 | 9 | 10 | class MountFieldRestRoutes(GatewayModule): 11 | """Mount rest routes for specific non-csp fields of the GatewayChannels. 12 | 13 | This is not done generically across all static fields as they may not always be serializable. 14 | """ 15 | 16 | requires: Optional[ChannelSelection] = [] 17 | fields: List[str] = Field(description="Static fields on the Channels that should be exposed via REST. These must be JSON serializable.") 18 | route: str = "/field" 19 | 20 | def connect(self, channels: GatewayChannels) -> None: 21 | # NO-OP 22 | ... 23 | 24 | def rest(self, app: GatewayWebApp) -> None: 25 | # Get API Router 26 | api_router: APIRouter = app.get_router("api") 27 | 28 | for field in self.fields: 29 | model = app.gateway.channels_model.get_outer_type(field) 30 | add_field_routes(api_router, field, self.route, model) 31 | 32 | @api_router.get( 33 | "{}".format(self.route), 34 | responses=get_default_responses(), 35 | response_model=List[str], 36 | include_in_schema=False, 37 | ) 38 | async def get_field(request: Request) -> List[str]: 39 | """ 40 | This endpoint will return a list of string values of all available channels under the `/field` route. 41 | """ 42 | return self.fields 43 | 44 | 45 | def add_field_routes( 46 | api_router: APIRouter, 47 | field: str, 48 | route: str, 49 | model: Union[BaseModel, Type], 50 | ) -> None: 51 | async def get_field(request: Request) -> model: # type: ignore[misc, valid-type] 52 | """ 53 | Get static field value on a static channel. 54 | """ 55 | # Throw 404 if not a supported channel 56 | if not hasattr(request.app.gateway.channels, field): 57 | raise HTTPException(status_code=404, detail="Channel field not found: {}".format(field)) 58 | 59 | # Grab the request off the edge 60 | try: 61 | res = getattr(request.app.gateway.channels, field) 62 | except AttributeError: 63 | raise HTTPException( 64 | status_code=404, 65 | detail="Channel field not found: {}".format(field), 66 | ) 67 | 68 | return res 69 | 70 | api_router.get( 71 | "{}/{}".format(route, field), 72 | responses=get_default_responses(), 73 | response_model=model, 74 | name="Get Channel field {}".format(field), 75 | )(get_field) 76 | 77 | api_router.get( 78 | "/{}".format(field.replace("_", "-")), 79 | responses=get_default_responses(), 80 | response_model=model, 81 | include_in_schema=False, 82 | )(get_field) 83 | -------------------------------------------------------------------------------- /csp_gateway/server/modules/web/outputs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | from fastapi import HTTPException, Request 5 | from fastapi.responses import HTMLResponse, StreamingResponse 6 | 7 | try: 8 | # conditional on libmagic being installed on the machine 9 | from magic import Magic 10 | except ImportError: 11 | Magic = None 12 | 13 | from csp_gateway.server import GatewayChannels, GatewayModule 14 | 15 | # separate to avoid circular 16 | from csp_gateway.server.web import GatewayWebApp 17 | 18 | 19 | class MountOutputsFolder(GatewayModule): 20 | def connect(self, channels: GatewayChannels) -> None: 21 | # NO-OP 22 | ... 23 | 24 | def rest(self, app: GatewayWebApp) -> None: 25 | app_router = app.get_router("app") 26 | mime = Magic(mime=True) if Magic else None 27 | 28 | # TODO subselect 29 | @app_router.get("/outputs/{full_path:path}", response_class=HTMLResponse, tags=["Utility"]) 30 | def browse_logs(full_path: str, request: Request): 31 | """ 32 | This endpoint is a small webpage for browsing the [hydra](https://github.com/facebookresearch/hydra) 33 | output logs and configuration settings of the running application. 34 | """ 35 | outputs_dir = os.path.join(os.getcwd(), "outputs") 36 | file_or_dir = outputs_dir 37 | if full_path: 38 | file_or_dir = os.path.join(file_or_dir, full_path) 39 | if os.path.abspath(file_or_dir).startswith(outputs_dir) and os.path.exists(file_or_dir): 40 | if os.path.isdir(file_or_dir): 41 | files = os.listdir(file_or_dir) 42 | files_paths = sorted([f"{request.url._url}/{f}".replace("outputs//", "outputs/") for f in files]) 43 | return app.templates.TemplateResponse( 44 | "files.html.j2", {"request": request, "files": files_paths, "pid": os.getpid()}, media_type="text/html" 45 | ) 46 | 47 | def iterfile(): # 48 | with open(file_or_dir, "rb") as fp: 49 | yield from fp 50 | 51 | if file_or_dir.endswith((".log", ".txt")): 52 | # NOTE: so viewable in browser, magic is guessing wrong type 53 | media_type = "text/plain; charset=utf-8" 54 | elif mime: 55 | media_type = mime.from_file(file_or_dir) 56 | else: 57 | media_type = None 58 | 59 | return StreamingResponse(iterfile(), media_type=media_type) 60 | raise HTTPException(status_code=404, detail="Not found: {}".format(request.url._url)) 61 | -------------------------------------------------------------------------------- /csp_gateway/server/settings.py: -------------------------------------------------------------------------------- 1 | from secrets import token_urlsafe 2 | from socket import gethostname 3 | from typing import List 4 | 5 | from pydantic import AnyHttpUrl, Field 6 | 7 | from csp_gateway import __version__ 8 | 9 | try: 10 | from pydantic_settings import BaseSettings 11 | except ImportError: 12 | from pydantic import BaseModel as BaseSettings 13 | 14 | 15 | class Settings(BaseSettings): 16 | """Generic settings for the CSP Gateway.""" 17 | 18 | model_config = dict(case_sensitive=True) 19 | 20 | API_V1_STR: str = "/api/v1" 21 | BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = [] 22 | 23 | TITLE: str = "Gateway" 24 | DESCRIPTION: str = "# Welcome to CSP Gateway API\nContains REST/Websocket interfaces to underlying CSP Gateway engine" 25 | VERSION: str = __version__ 26 | AUTHOR: str = "" 27 | EMAIL: str = "example@domain.com" 28 | 29 | BIND: str = "0.0.0.0" 30 | PORT: int = 8000 31 | 32 | UI: bool = Field(False, description="Enables ui in the web application") 33 | AUTHENTICATE: bool = Field(False, description="Whether to authenticate users for access to the web application") 34 | API_KEY: str = Field( 35 | token_urlsafe(32), 36 | description="The API key for access if `AUTHENTICATE=True`. The default is auto-generated, but a user-provided value can be used.", 37 | ) 38 | AUTHENTICATION_DOMAIN: str = gethostname() 39 | -------------------------------------------------------------------------------- /csp_gateway/server/shared/__init__.py: -------------------------------------------------------------------------------- 1 | from .adapters import * 2 | from .channel_selection import * 3 | from .json_converter import * 4 | -------------------------------------------------------------------------------- /csp_gateway/server/shared/adapters.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | from datetime import timedelta 4 | from threading import Thread 5 | from typing import Any, Callable, Optional 6 | 7 | import pandas as pd 8 | import pyarrow as pa 9 | from csp import ts 10 | from csp.impl.pushadapter import PushInputAdapter 11 | from csp.impl.wiring import py_push_adapter_def 12 | 13 | __all__ = ( 14 | "sql_polling_adapter_def", 15 | "poll_sql_for_arrow_tbl", 16 | "poll_sql_for_pandas_df", 17 | ) 18 | 19 | 20 | def poll_sql_for_arrow_tbl(connection: str, query: str, logger_name: str = __name__) -> pa.Table: 21 | from arrow_odbc import read_arrow_batches_from_odbc 22 | 23 | reader = read_arrow_batches_from_odbc( 24 | query=query, 25 | connection_string=connection, 26 | batch_size=10_000, 27 | ) 28 | return pa.Table.from_batches(batches=reader, schema=reader.schema) 29 | 30 | 31 | def poll_sql_for_pandas_df(connection: str, query: str, logger_name: str = __name__) -> pd.DataFrame: 32 | return poll_sql_for_arrow_tbl(connection, query, logger_name).to_pandas() 33 | 34 | 35 | class PollingSQLAdapterImpl(PushInputAdapter): 36 | def __init__( 37 | self, 38 | interval: timedelta, 39 | connection: str, 40 | query: str, 41 | poll: Callable[[str, str, logging.Logger], Any], 42 | callback: Optional[Callable[[Any], Any]] = None, 43 | logger_name: str = __name__, 44 | failed_poll_msg: str = "Failed to poll sql database", 45 | connection_timeout_seconds: int = 0, 46 | query_timeout_seconds: int = 0, 47 | ): 48 | self._interval = interval 49 | self._connection = connection 50 | if connection_timeout_seconds: 51 | self._connection += f";timeout={connection_timeout_seconds}" 52 | 53 | if query_timeout_seconds: 54 | self._connection += f";commandTimeout={query_timeout_seconds}" 55 | 56 | self._query = query 57 | self._callback = callback 58 | self._last_update = 0 59 | self._poll = poll 60 | self._logger_name = logger_name 61 | self._failed_poll_msg = failed_poll_msg 62 | 63 | self._thread = None 64 | self._running = False 65 | self._paused = False 66 | 67 | def start(self, starttime, endtime): 68 | """start will get called at the start of the engine, at which point the push 69 | input adapter should start its thread that will push the data onto the adapter. Note 70 | that push adapters will ALWAYS have a separate thread driving ticks into the csp engine thread 71 | """ 72 | self._running = True 73 | self._thread = Thread(target=self._run, daemon=True) 74 | self._thread.start() 75 | 76 | def stop(self): 77 | """stop will get called at the end of the run, at which point resources should 78 | be cleaned up 79 | """ 80 | if self._running: 81 | self._running = False 82 | # if it is paused, we just kill it 83 | if self._paused: 84 | self._thread.join(0.01) 85 | else: 86 | # we give it the interval time to finish 87 | self._thread.join(self._interval.total_seconds()) 88 | 89 | def _run(self): 90 | log = logging.getLogger(self._logger_name) 91 | while self._running: 92 | self._paused = False 93 | try: 94 | now = time.time() 95 | res = self._poll(self._connection, self._query, self._logger_name) 96 | if self._callback is not None: 97 | res = self._callback(res) 98 | self._last_update = now 99 | self.push_tick(res) 100 | 101 | except Exception: 102 | # TODO: What do we want to do here? 103 | log.warning(self._failed_poll_msg, exc_info=True) 104 | self.push_tick(None) # None is a bad value 105 | 106 | # TODO: Handle what happens if it hangs, by having a separate thread listen for data in a queue 107 | # and publish pd.DataFrame if it misses a heartbeat 108 | # sleep interval 109 | self._paused = True 110 | time.sleep(self._interval.total_seconds()) 111 | 112 | 113 | sql_polling_adapter_def = py_push_adapter_def( 114 | "sql_polling_adapter_def", 115 | PollingSQLAdapterImpl, 116 | ts[object], 117 | interval=timedelta, 118 | connection=str, 119 | query=str, 120 | poll=object, 121 | callback=object, 122 | logger_name=str, 123 | failed_poll_msg=str, 124 | connection_timeout_seconds=int, 125 | query_timeout_seconds=int, 126 | ) 127 | -------------------------------------------------------------------------------- /csp_gateway/server/shared/channel_selection.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Set, Union 2 | 3 | from ccflow import BaseModel 4 | from csp.impl.types.tstype import isTsType 5 | from pydantic import Field, model_validator 6 | 7 | from csp_gateway.server.gateway.csp import Channels, ChannelsType 8 | from csp_gateway.utils import is_dict_basket 9 | 10 | __all__ = ("ChannelSelection",) 11 | 12 | 13 | class ChannelSelection(BaseModel): 14 | """ 15 | A class to represent channel selection options for filtering channels based on inclusion and exclusion criteria. 16 | 17 | Attributes: 18 | `include` (Optional[List[str]]): A list of channel names to include in the selection. 19 | The order here matters 20 | Defaults to None (include everything). 21 | `exclude` (Set[str]): A list of channel names to exclude from the selection. 22 | This overrides anything in `include` 23 | Defaults to an empty set. 24 | 25 | Methods: 26 | select_from(channels, static_fields=False, state_channels=False): Returns a list of selected channel names 27 | based on the inclusion and exclusion criteria, and optional static_fields and 28 | state_channels flags. The order of channels is based on the order of the channels 29 | in `include`. State channels always follow their corresponding channels. 30 | If `include` is None, the order is based on the order of the fields in the channels object. 31 | validate(v): Validates and coerces the input value to a ChannelSelection instance. 32 | """ 33 | 34 | include: Optional[List[str]] = None 35 | exclude: Set[str] = Field(default_factory=set) 36 | 37 | @model_validator(mode="before") 38 | def validate_requires(cls, v): 39 | if v is None: 40 | return {} 41 | if isinstance(v, list): 42 | return dict(include=list(v)) 43 | return v 44 | 45 | def select_from( 46 | self, 47 | channels: Union[Channels, ChannelsType], 48 | *, 49 | static_fields: bool = False, # Select only static fields 50 | state_channels: bool = False, # Select only state channels 51 | all_fields: bool = False, # Select all fields in include and not in exclude 52 | ) -> List[str]: 53 | """ 54 | Select fields from the given channels based on the specified criteria. 55 | 56 | Args: 57 | channels (Union[Channels, ChannelsType]): The channels to select fields from. 58 | static_fields (bool, optional): If True, select only static fields. Defaults to False. 59 | state_channels (bool, optional): If True, select only state channels. Defaults to False. 60 | all_fields (bool, optional): If True, select all fields in include and not in exclude. Defaults to False. 61 | 62 | Returns: 63 | List[str]: A list of selected field names. 64 | """ 65 | names = {} 66 | 67 | if all_fields: 68 | fields = channels.model_fields if self.include is None else self.include 69 | return list(dict.fromkeys([field for field in fields if field not in self.exclude])) 70 | 71 | for idx, field in enumerate(channels.model_fields): 72 | # avoid duplicates 73 | if field in names: 74 | continue 75 | 76 | # TODO not this `s_` business... 77 | # Return state channels or regular channels 78 | if state_field := field.startswith("s_"): 79 | if not state_channels: 80 | continue 81 | field = field[2:] 82 | else: 83 | if state_channels: 84 | continue 85 | 86 | # Check whether static 87 | outer_type = channels.get_outer_type(field) 88 | if is_dict_basket(outer_type) or isTsType(outer_type): 89 | if static_fields: 90 | continue 91 | else: 92 | if not static_fields: 93 | continue 94 | 95 | # Check whether included 96 | if self.include is not None: 97 | try: 98 | idx = self.include.index(field) 99 | except ValueError: 100 | continue 101 | 102 | # Check whether excluded 103 | if field in self.exclude: 104 | continue 105 | 106 | if state_field: 107 | field = f"s_{field}" 108 | 109 | names[field] = idx 110 | 111 | return [k for k, _ in sorted(names.items(), key=lambda x: x[1])] 112 | -------------------------------------------------------------------------------- /csp_gateway/server/web/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import GatewayWebApp 2 | from .routes import prepare_response 3 | from .utils import get_default_responses 4 | -------------------------------------------------------------------------------- /csp_gateway/server/web/routes/__init__.py: -------------------------------------------------------------------------------- 1 | from .controls import add_controls_routes 2 | from .last import add_last_api_available_channels, add_last_routes 3 | from .lookup import add_lookup_routes 4 | from .next import add_next_api_available_channels, add_next_routes 5 | from .send import add_send_api_available_channels, add_send_routes 6 | from .shared import prepare_response 7 | from .state import add_state_api_available_channels, add_state_routes 8 | -------------------------------------------------------------------------------- /csp_gateway/server/web/routes/controls.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | 4 | from fastapi import APIRouter, BackgroundTasks, HTTPException, Request 5 | 6 | from csp_gateway.utils import Controls 7 | 8 | from ..utils import get_default_responses 9 | from .shared import prepare_response 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | _WAIT_THRESHOLD = 0.1 15 | 16 | 17 | def add_controls_routes(api_router: APIRouter) -> None: 18 | # Add heartbeat channel 19 | @api_router.get( 20 | "/heartbeat", 21 | responses=get_default_responses(), 22 | response_model=Controls, 23 | name="Get Heartbeat", 24 | ) 25 | async def heartbeat(request: Request) -> Controls: 26 | """ 27 | This endpoint is a lightweight `ping`/`pong` endpoint that can be used to determine the status of the underlying webserver. 28 | """ 29 | data = Controls(name="heartbeat") 30 | 31 | # Throw 404 if not a supported channel 32 | if not hasattr(request.app.gateway.channels, "controls"): 33 | raise HTTPException(status_code=404, detail="Channel not found: controls") 34 | 35 | # send data to csp 36 | request.app.gateway.channels.send("controls", data) 37 | 38 | # don't care about the result 39 | while data.status != "ok": 40 | await asyncio.sleep(_WAIT_THRESHOLD) 41 | 42 | return prepare_response(data, is_list_model=False) 43 | 44 | @api_router.get( 45 | "/stats", 46 | responses=get_default_responses(), 47 | response_model=Controls, 48 | name="Get CSP Stats", 49 | ) 50 | async def stats(request: Request) -> Controls: 51 | """This endpoint will collect and return various engine and system stats, including: 52 | 53 | - CPU utilization (`cpu`) 54 | - Virtual memory utilization (`memory`) 55 | - Total memory available (`memory-total`) 56 | - Current system time (`now`) 57 | - CSP engine time (`csp-now`) 58 | - Hostname (`host`) 59 | - Username (`user`) 60 | """ 61 | data = Controls(name="stats") 62 | 63 | # Throw 404 if not a supported channel 64 | if not hasattr(request.app.gateway.channels, "controls"): 65 | raise HTTPException(status_code=404, detail="Channel not found: controls") 66 | 67 | # send data to csp 68 | request.app.gateway.channels.send("controls", data) 69 | 70 | while not data.data: 71 | await asyncio.sleep(_WAIT_THRESHOLD) 72 | data.update_str() 73 | 74 | return prepare_response(data, is_list_model=False) 75 | 76 | @api_router.post( 77 | "/shutdown", 78 | responses=get_default_responses(), 79 | response_model=Controls, 80 | name="Shutdown Server", 81 | ) 82 | async def shutdown(request: Request, background_tasks: BackgroundTasks) -> Controls: 83 | """ 84 | **WARNING:** Use this endpoint with caution. 85 | 86 | This endpoint will cleanly shutdown the engine and webserver. It is used for the kill switch in UIs. 87 | """ 88 | # FIXME ugly 89 | background_tasks.add_task(request.app.gateway.stop, user_initiated=True) 90 | 91 | data = Controls(name="shutdown", status="ok") 92 | return prepare_response(data, is_list_model=False) 93 | -------------------------------------------------------------------------------- /csp_gateway/server/web/routes/lookup.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union, get_args, get_origin 2 | 3 | from fastapi import APIRouter, HTTPException, Request 4 | from pydantic import BaseModel 5 | 6 | from ..utils import get_default_responses 7 | from .shared import prepare_response 8 | 9 | 10 | def add_lookup_routes( 11 | api_router: APIRouter, 12 | field: str, 13 | model: Union[BaseModel, List[BaseModel]], 14 | ) -> None: 15 | if model and get_origin(model) is list: 16 | model = get_args(model)[0] 17 | 18 | async def lookup(id: str, request: Request) -> List[model]: # type: ignore[misc, valid-type] 19 | """ 20 | This endpoint lets you lookup any GatewayStruct by its uniquely generated `id`. 21 | """ 22 | # Throw 404 if not a supported channel 23 | if not hasattr(request.app.gateway.channels, field): 24 | raise HTTPException(status_code=404, detail="Channel not found: {}".format(field)) 25 | 26 | # lookup by id 27 | res = model.lookup(id) 28 | 29 | return prepare_response(res, is_list_model=False) 30 | 31 | api_router.get( 32 | "/{}/{{id:path}}".format(field), 33 | responses=get_default_responses(), 34 | response_model=List[model], 35 | name="Lookup {}".format(field), 36 | )(lookup) 37 | 38 | api_router.get( 39 | "/{}/{{id:path}}".format(field.replace("_", "-")), 40 | responses=get_default_responses(), 41 | response_model=List[model], 42 | include_in_schema=False, 43 | )(lookup) 44 | -------------------------------------------------------------------------------- /csp_gateway/server/web/routes/shared.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Any, Dict, List 3 | 4 | from fastapi.responses import Response 5 | 6 | 7 | def prepare_response( 8 | res: Any, 9 | is_list_model: bool = False, 10 | is_dict_basket: bool = False, 11 | wrap_in_response: bool = True, 12 | ) -> List[Dict[Any, Any]]: 13 | # If we've ticked 14 | if res: 15 | # Convert the dict basket to just the list of values 16 | if is_dict_basket: 17 | res = res.values() 18 | 19 | # If its not a list model, it means we got back 1 thing, so wrap it 20 | elif not is_list_model and not isinstance(res, list): 21 | res = [res] 22 | 23 | else: 24 | # Else return an empty json 25 | res = [] 26 | 27 | json_res_bytes = b"[" + b",".join(r.type_adapter().dump_json(r) for r in res) + b"]" 28 | json_res = json_res_bytes.decode() 29 | # Prepare and return response 30 | if wrap_in_response: 31 | return Response( 32 | content=json_res, 33 | media_type="application/json", 34 | ) 35 | # useful when you want the data, but outside a fastapi response object 36 | return json_res 37 | 38 | 39 | async def get_next_tick(gateway, field, key=""): 40 | """Need to do some fanciness so that a `next` call doesnt block the webserver""" 41 | return await asyncio.get_event_loop().run_in_executor(None, gateway.channels.next, field, key) 42 | -------------------------------------------------------------------------------- /csp_gateway/server/web/templates/channels_graph.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | Channels Graph 4 | 13 | 14 | 15 | 16 | 17 | 18 | Home 19 |
20 | 21 | 74 | 140 | -------------------------------------------------------------------------------- /csp_gateway/server/web/templates/css/common.css: -------------------------------------------------------------------------------- 1 | 2 | body { 3 | font-family: "Roboto"; 4 | background-color: #eee; 5 | } 6 | 7 | div.centered { 8 | position: absolute; 9 | left: 0; 10 | right: 0; 11 | top: 0; 12 | bottom: 0; 13 | 14 | margin: auto; 15 | padding: 50px; 16 | 17 | display: flex; 18 | flex-direction: column; 19 | justify-content: center; 20 | align-items: center; 21 | } 22 | 23 | div.row { 24 | display: flex; 25 | flex-direction: row; 26 | align-items: center; 27 | justify-content: space-between; 28 | } 29 | 30 | div.centered > h1 { 31 | text-align: "center"; 32 | } 33 | 34 | form.centered-form { 35 | display: flex; 36 | flex-direction: column; 37 | } 38 | form.centered-form > * { 39 | margin: 5px; 40 | } 41 | 42 | div.error { 43 | display: flex; 44 | flex-direction: column; 45 | background-color: red; 46 | /* width: 50%; */ 47 | text-align: center; 48 | justify-content: space-around; 49 | padding: 10px; 50 | } 51 | 52 | span.error-code { 53 | font-weight: bolder; 54 | } -------------------------------------------------------------------------------- /csp_gateway/server/web/templates/files.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | Logs 4 | 7 | 8 | 9 |
10 |

Logs

11 |

pid[{{pid}}]

12 |
13 | 18 | 19 | -------------------------------------------------------------------------------- /csp_gateway/server/web/templates/js/common.js: -------------------------------------------------------------------------------- 1 | 2 | const get_openapi = async () => { 3 | const openapi = await fetch(`${window.location.protocol}//${window.location.host}/openapi.json`); 4 | return openapi.json(); 5 | } 6 | 7 | 8 | (async () => { 9 | const openapi = await get_openapi(); 10 | document.title = openapi.info.title; 11 | })() 12 | -------------------------------------------------------------------------------- /csp_gateway/server/web/templates/login.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | Login 4 | 7 | 10 | 11 | 12 |
13 | {% if status_code is defined %} 14 |
15 | Error {{ status_code }} 16 | {{ detail }} 17 |
18 | {% endif %} 19 |

Login

20 |
21 | 22 | 23 |
24 |
25 | 26 | -------------------------------------------------------------------------------- /csp_gateway/server/web/templates/logout.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | Logout 4 | 7 | 10 | 11 | 12 |
13 |

Logout

14 |
15 | 16 |
17 |
18 | 19 | -------------------------------------------------------------------------------- /csp_gateway/server/web/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Coroutine, Dict, Union 2 | 3 | from fastapi.exceptions import RequestErrorModel 4 | 5 | NoArgsNoReturnFuncT = Callable[[], None] 6 | NoArgsNoReturnAsyncFuncT = Callable[[], Coroutine[Any, Any, None]] 7 | NoArgsNoReturnDecorator = Callable[[Union[NoArgsNoReturnFuncT, NoArgsNoReturnAsyncFuncT]], NoArgsNoReturnAsyncFuncT] 8 | 9 | 10 | class Error404(RequestErrorModel): # type: ignore[misc, valid-type] 11 | detail: str = "" 12 | 13 | 14 | def get_default_responses() -> Dict[Union[int, str], Dict[str, Any]]: 15 | return {404: {"model": Error404}} 16 | -------------------------------------------------------------------------------- /csp_gateway/testing/__init__.py: -------------------------------------------------------------------------------- 1 | from .harness import * 2 | from .shared_helpful_classes import * 3 | from .web import * 4 | -------------------------------------------------------------------------------- /csp_gateway/testing/web.py: -------------------------------------------------------------------------------- 1 | import time 2 | from threading import Thread 3 | 4 | import csp 5 | from csp import ts 6 | 7 | from csp_gateway import GatewayModule 8 | 9 | __all__ = ( 10 | "NeverDieModule", 11 | "CspDieModule", 12 | "LongStartModule", 13 | ) 14 | 15 | 16 | def _never_die(): 17 | while True: 18 | try: 19 | print("here") 20 | time.sleep(1) 21 | except: # noqa: E722 22 | ... 23 | 24 | 25 | class NeverDieModule(GatewayModule): 26 | def connect(self, *args, **kwargs) -> None: 27 | self._thread = Thread(target=_never_die, daemon=True) 28 | self._thread.start() 29 | 30 | 31 | class CspDieModule(GatewayModule): 32 | count: int = 5 33 | channel_to_access: str = "example" 34 | attribute_to_access: str = "blerg" 35 | 36 | def connect(self, channels) -> None: 37 | self._tick(channels.get_channel(self.channel_to_access)) 38 | 39 | @csp.node 40 | def _tick(self, thing: ts[object]): 41 | with csp.state(): 42 | s_counter = 0 43 | if csp.ticked(thing): 44 | s_counter += 1 45 | if s_counter >= self.count: 46 | print("running getter to trigger problem") 47 | getattr(thing, self.attribute_to_access) 48 | 49 | 50 | class LongStartModule(GatewayModule): 51 | sleep: int = 5 52 | channel_to_access: str = "example" 53 | 54 | def connect(self, channels) -> None: 55 | time.sleep(self.sleep) 56 | self._tick(channels.get_channel(self.channel_to_access)) 57 | 58 | @csp.node 59 | def _tick(self, thing: ts[object]): 60 | with csp.start(): 61 | time.sleep(self.sleep) 62 | if csp.ticked(thing): 63 | print("do nothing") 64 | -------------------------------------------------------------------------------- /csp_gateway/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/config/test_load.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import hydra 4 | 5 | import csp_gateway.server.config 6 | from csp_gateway import Gateway 7 | 8 | 9 | def test_config_file_load(): 10 | """This case tests a config where the gateway is specified in the user config file""" 11 | user_config_dir = os.path.dirname(__file__) 12 | r = csp_gateway.server.config.load_config( 13 | overwrite=True, 14 | config_dir=user_config_dir, 15 | overrides=["+user_config=sample_config"], 16 | ) 17 | try: 18 | assert isinstance(r["gateway"], Gateway) 19 | finally: 20 | r.clear() 21 | 22 | 23 | def test_config_file_load_gateway(): 24 | user_config_dir = os.path.dirname(__file__) 25 | g = csp_gateway.server.config.load_gateway( 26 | overwrite=True, 27 | config_dir=user_config_dir, 28 | overrides=["+user_config=sample_config"], 29 | ) 30 | assert isinstance(g, Gateway) 31 | 32 | 33 | def test_start_load(): 34 | config_dir = os.path.join(os.path.dirname(__file__), "../../server/config") 35 | with hydra.initialize_config_dir(version_base=None, config_dir=config_dir): 36 | cfg = hydra.compose(config_name="base.yaml") 37 | assert dict(cfg["start"]) == { 38 | "realtime": True, 39 | "block": False, 40 | "show": False, 41 | "rest": True, 42 | "ui": True, 43 | } 44 | -------------------------------------------------------------------------------- /csp_gateway/tests/config/user_config/sample_config.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | defaults: 3 | - _self_ 4 | 5 | gateway: 6 | _target_: csp_gateway.Gateway 7 | modules: 8 | - /modules/example_module 9 | - /modules/mount_rest_routes 10 | - /modules/mount_controls 11 | -------------------------------------------------------------------------------- /csp_gateway/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(scope="class") 7 | def free_port(): 8 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 9 | s.bind(("", 0)) 10 | s.listen(1) 11 | return s.getsockname()[1] 12 | -------------------------------------------------------------------------------- /csp_gateway/tests/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/config/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/config/test_load_config.py: -------------------------------------------------------------------------------- 1 | from csp_gateway import Gateway, load_config 2 | 3 | 4 | class TestLoadConfig: 5 | def test_load_config(self): 6 | gw = load_config(overrides=["+gateway=demo", "+port=1234"], overwrite=True)["/gateway"] 7 | assert isinstance(gw, Gateway) 8 | assert gw.settings.PORT == 1234 9 | -------------------------------------------------------------------------------- /csp_gateway/tests/server/gateway/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/gateway/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/gateway/csp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/gateway/csp/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/modules/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/controls/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/modules/controls/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/modules/io/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/kafka/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/modules/kafka/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/modules/logging/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/test_initializer.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | import csp 4 | import pytest 5 | 6 | from csp_gateway.server import Initialize 7 | from csp_gateway.testing import ( 8 | GatewayTestHarness, 9 | MyEnum, 10 | MyGateway, 11 | MyGatewayChannels, 12 | MyStruct, 13 | ) 14 | 15 | 16 | @pytest.mark.parametrize("unroll", [True, False]) 17 | @pytest.mark.parametrize("offset", [0, 30]) 18 | def test_initializer(offset, unroll): 19 | offset_ts = timedelta(seconds=offset) 20 | start_dt = datetime(2023, 1, 1) 21 | target_dt = start_dt + offset_ts 22 | 23 | goal_struct = MyStruct(foo=1.0, my_flag=True, time=timedelta(seconds=10), id="test", timestamp=target_dt) 24 | val = {"foo": 1.0, "time": 10, "id": "test", "timestamp": target_dt} 25 | if unroll: 26 | val = [val.copy(), val.copy()] 27 | 28 | init = Initialize( 29 | channel=MyGatewayChannels.my_channel, 30 | seconds_offset=offset, 31 | value=val, 32 | unroll=unroll, 33 | ) 34 | h = GatewayTestHarness(test_channels=[MyGatewayChannels.my_channel]) 35 | if offset > 0: 36 | h.advance(delay=offset_ts) 37 | elif unroll: 38 | h.delay(delay=timedelta(microseconds=1)) # we have to wait for the unrolling 39 | 40 | def assert_func(vals): 41 | goal_len = 1 if not unroll else 2 42 | assert len(vals) == goal_len 43 | for dt, val in vals: 44 | assert dt == target_dt 45 | assert val.to_dict() == goal_struct.to_dict() 46 | 47 | h.assert_ticked_values(MyGatewayChannels.my_channel, assert_func) 48 | gw = MyGateway(modules=[h, init], channels=MyGatewayChannels()) 49 | 50 | csp.run(gw.graph, starttime=start_dt, endtime=timedelta(1)) 51 | 52 | 53 | @pytest.mark.parametrize("unroll", [True, False]) 54 | @pytest.mark.parametrize("offset", [0, 30]) 55 | def test_initializer_list(offset, unroll): 56 | offset_ts = timedelta(seconds=offset) 57 | start_dt = datetime(2023, 1, 1) 58 | target_dt = start_dt + offset_ts 59 | val = [{"foo": 1.0, "time": 10, "id": "test", "timestamp": target_dt}] 60 | if unroll: 61 | val = [val.copy(), val.copy()] # we unroll 2 lists 62 | goal_struct = MyStruct(foo=1.0, my_flag=True, time=timedelta(seconds=10), id="test", timestamp=target_dt) 63 | init = Initialize( 64 | channel=MyGatewayChannels.my_list_channel, 65 | unroll=unroll, 66 | seconds_offset=offset, 67 | value=val, 68 | ) 69 | h = GatewayTestHarness(test_channels=[MyGatewayChannels.my_list_channel]) 70 | if offset > 0: 71 | h.advance(delay=offset_ts) 72 | elif unroll: 73 | h.delay(delay=timedelta(microseconds=1)) # we have to wait for the unrolling 74 | 75 | def assert_func(vals): 76 | goal_len = 1 if not unroll else 2 77 | assert len(vals) == goal_len 78 | for dt, val in vals: 79 | assert dt == target_dt 80 | assert len(val) == 1 81 | assert val[0].to_dict() == goal_struct.to_dict() 82 | 83 | h.assert_ticked_values(MyGatewayChannels.my_list_channel, assert_func) 84 | gw = MyGateway(modules=[h, init], channels=MyGatewayChannels()) 85 | 86 | csp.run(gw.graph, starttime=start_dt, endtime=timedelta(1)) 87 | 88 | 89 | @pytest.mark.parametrize("offset", [0, 30]) 90 | def test_initializer_dict_basket(offset): 91 | offset_ts = timedelta(seconds=offset) 92 | start_dt = datetime(2023, 1, 1) 93 | target_dt = start_dt + offset_ts 94 | 95 | goal_struct = MyStruct(foo=1.0, my_flag=True, time=timedelta(seconds=10), id="test", timestamp=target_dt) 96 | init = Initialize( 97 | channel=MyGatewayChannels.my_enum_basket, 98 | seconds_offset=offset, 99 | value={ 100 | "ONE": {"foo": 1.0, "time": 10, "id": "test", "timestamp": target_dt}, 101 | "TWO": {"foo": 1.0, "time": 0, "id": "test", "timestamp": target_dt}, 102 | }, 103 | ) 104 | h = GatewayTestHarness(test_channels=[MyGatewayChannels.my_enum_basket]) 105 | h.delay(delay=offset_ts) 106 | 107 | def assert_value_one(val): 108 | assert val.to_dict() == goal_struct.to_dict() 109 | 110 | def assert_value_two(val): 111 | goal_dict = goal_struct.to_dict() 112 | goal_dict["time"] = timedelta(seconds=0) 113 | assert val.to_dict() == goal_dict 114 | 115 | h.assert_value( 116 | (MyGatewayChannels.my_enum_basket, MyEnum.ONE), 117 | assert_value_one, 118 | ) 119 | h.assert_value( 120 | (MyGatewayChannels.my_enum_basket, MyEnum.TWO), 121 | assert_value_two, 122 | ) 123 | gw = MyGateway(modules=[h, init], channels=MyGatewayChannels()) 124 | 125 | csp.run(gw.graph, starttime=start_dt, endtime=timedelta(1)) 126 | -------------------------------------------------------------------------------- /csp_gateway/tests/server/modules/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/modules/web/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/shared/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/shared/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/shared/test_channel_selection.py: -------------------------------------------------------------------------------- 1 | from pydantic import TypeAdapter 2 | 3 | from csp_gateway.server import ChannelSelection 4 | from csp_gateway.testing.shared_helpful_classes import MyGatewayChannels 5 | 6 | 7 | def test_selection(): 8 | selection = ChannelSelection() 9 | channels = [ 10 | "my_channel", 11 | "my_list_channel", 12 | "my_enum_basket", 13 | "my_str_basket", 14 | "my_enum_basket_list", 15 | "my_array_channel", 16 | ] 17 | static = ["my_static", "my_static_dict", "my_static_list"] 18 | 19 | assert selection.select_from(MyGatewayChannels) == channels 20 | assert selection.select_from(MyGatewayChannels, state_channels=True) == [ 21 | "s_my_channel", 22 | "s_my_list_channel", 23 | ] 24 | assert selection.select_from(MyGatewayChannels, static_fields=True) == static 25 | 26 | 27 | def test_selection_duplicates(): 28 | channels = ["my_channel", "my_enum_basket"] 29 | static = ["my_static", "my_static_dict"] 30 | selection = ChannelSelection(include=channels + channels + static) 31 | 32 | assert selection.select_from(MyGatewayChannels) == channels 33 | assert selection.select_from(MyGatewayChannels, state_channels=True) == ["s_my_channel"] 34 | assert selection.select_from(MyGatewayChannels, static_fields=True) == static 35 | 36 | 37 | def test_selection_all_fields(): 38 | channels = ["my_enum_basket", "my_channel", "s_my_channel"] 39 | static = ["my_static", "my_static_dict"] 40 | selection = ChannelSelection(include=channels + static) 41 | 42 | assert selection.select_from(MyGatewayChannels, all_fields=True) == [ 43 | "my_enum_basket", 44 | "my_channel", 45 | "s_my_channel", 46 | "my_static", 47 | "my_static_dict", 48 | ] 49 | 50 | 51 | def test_selection_include(): 52 | channels = ["my_channel", "my_enum_basket"] 53 | static = ["my_static", "my_static_dict"] 54 | selection = ChannelSelection(include=channels + static) 55 | 56 | assert selection.select_from(MyGatewayChannels) == channels 57 | assert selection.select_from(MyGatewayChannels, state_channels=True) == ["s_my_channel"] 58 | assert selection.select_from(MyGatewayChannels, static_fields=True) == static 59 | assert selection.select_from(MyGatewayChannels, all_fields=True) == [ 60 | "my_channel", 61 | "my_enum_basket", 62 | "my_static", 63 | "my_static_dict", 64 | ] 65 | 66 | 67 | def test_selection_exclude(): 68 | selection = ChannelSelection(exclude=["my_channel", "my_enum_basket", "my_static"]) 69 | channels = [ 70 | "my_list_channel", 71 | "my_str_basket", 72 | "my_enum_basket_list", 73 | "my_array_channel", 74 | ] 75 | static = ["my_static_dict", "my_static_list"] 76 | 77 | assert selection.select_from(MyGatewayChannels) == channels 78 | assert selection.select_from(MyGatewayChannels, state_channels=True) == ["s_my_list_channel"] 79 | assert selection.select_from(MyGatewayChannels, static_fields=True) == static 80 | 81 | 82 | def test_validate(): 83 | selection = TypeAdapter(ChannelSelection).validate_python(["my_channel", "my_enum_basket"]) 84 | assert selection.include == ["my_channel", "my_enum_basket"] 85 | 86 | selection = TypeAdapter(ChannelSelection).validate_python(None) 87 | assert selection.include is None 88 | -------------------------------------------------------------------------------- /csp_gateway/tests/server/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/server/web/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/server/web/test_shared.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | import starlette.responses 5 | 6 | from csp_gateway.server.demo import ExampleData 7 | from csp_gateway.server.web import prepare_response 8 | 9 | 10 | def test_prepare_response(): 11 | data = ExampleData() 12 | response = prepare_response(res=data, is_list_model=False, is_dict_basket=False, wrap_in_response=True) 13 | assert isinstance(response, starlette.responses.Response) 14 | response = prepare_response(res=data, is_list_model=False, is_dict_basket=False, wrap_in_response=False) 15 | assert json.loads(response) == [json.loads(data.type_adapter().dump_json(data))] 16 | 17 | 18 | def test_prepare_response_list(): 19 | data = ExampleData() 20 | response = prepare_response(res=[data], is_list_model=False, is_dict_basket=False, wrap_in_response=False) 21 | assert json.loads(response) == [json.loads(data.type_adapter().dump_json(data))] 22 | response = prepare_response(res=[data], is_list_model=True, is_dict_basket=False, wrap_in_response=False) 23 | assert json.loads(response) == [json.loads(data.type_adapter().dump_json(data))] 24 | response = prepare_response(res=(data,), is_list_model=True, is_dict_basket=False, wrap_in_response=False) 25 | assert json.loads(response) == [json.loads(data.type_adapter().dump_json(data))] 26 | 27 | 28 | def test_prepare_response_dict(): 29 | data = ExampleData() 30 | with pytest.raises(AttributeError): 31 | prepare_response(res={"foo": data}, is_list_model=False, is_dict_basket=False, wrap_in_response=False) 32 | 33 | response = prepare_response(res={"foo": data}, is_list_model=False, is_dict_basket=True, wrap_in_response=False) 34 | assert json.loads(response) == [json.loads(data.type_adapter().dump_json(data))] 35 | -------------------------------------------------------------------------------- /csp_gateway/tests/test_harness.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | import csp 4 | import pytest 5 | 6 | from csp_gateway.server.modules import AddChannelsToGraphOutput 7 | from csp_gateway.testing import GatewayTestHarness 8 | from csp_gateway.testing.shared_helpful_classes import MyGateway, MyGatewayChannels, MyStruct 9 | 10 | 11 | @pytest.mark.parametrize("make_invalid", (True, False)) 12 | def test_delay(make_invalid): 13 | channels = [ 14 | MyGatewayChannels.my_channel, 15 | ] 16 | h = GatewayTestHarness(test_channels=channels) 17 | 18 | # Engine start 19 | h.send(MyGatewayChannels.my_channel, MyStruct()) 20 | h.assert_ticked(MyGatewayChannels.my_channel, 1) 21 | 22 | # timedelta delay 23 | h.advance(delay=timedelta(seconds=1)) 24 | h.send(MyGatewayChannels.my_channel, MyStruct()) 25 | 26 | # timedelta delay on timedelta delay 27 | h.advance(delay=timedelta(seconds=2)) 28 | h.send(MyGatewayChannels.my_channel, MyStruct()) 29 | 30 | # datetime delay 31 | h.advance(delay=datetime(2020, 1, 2)) 32 | h.send(MyGatewayChannels.my_channel, MyStruct()) 33 | 34 | # timedelta delay on datetime delay 35 | h.advance(delay=timedelta(seconds=5)) 36 | h.send(MyGatewayChannels.my_channel, MyStruct()) 37 | 38 | if make_invalid: 39 | # Jumping back in time. 40 | h.advance(delay=datetime(2020, 1, 2, 0, 0, 1)) 41 | 42 | gateway = MyGateway( 43 | modules=[h, AddChannelsToGraphOutput()], 44 | channels=MyGatewayChannels(), 45 | ) 46 | 47 | if make_invalid: 48 | with pytest.raises(ValueError): 49 | csp.run(gateway.graph, starttime=datetime(2020, 1, 1), endtime=timedelta(3)) 50 | else: 51 | res = csp.run(gateway.graph, starttime=datetime(2020, 1, 1), endtime=timedelta(3)) 52 | 53 | assert "my_channel" in res 54 | assert len(res["my_channel"]) == 5 55 | expected_times = [ 56 | datetime(2020, 1, 1), 57 | datetime(2020, 1, 1, 0, 0, 1), 58 | datetime(2020, 1, 1, 0, 0, 3), 59 | datetime(2020, 1, 2), 60 | datetime(2020, 1, 2, 0, 0, 5), 61 | ] 62 | for (actual_time, _), expected_time in zip(res["my_channel"], expected_times): 63 | assert actual_time == expected_time 64 | 65 | 66 | def test_delay_jump_straight_away(): 67 | channels = [ 68 | MyGatewayChannels.my_channel, 69 | ] 70 | h = GatewayTestHarness(test_channels=channels) 71 | 72 | # datetime delay 73 | h.advance(delay=datetime(2020, 1, 2)) 74 | h.send(MyGatewayChannels.my_channel, MyStruct()) 75 | 76 | # timedelta delay on datetime delay 77 | h.advance(delay=timedelta(seconds=5)) 78 | h.send(MyGatewayChannels.my_channel, MyStruct()) 79 | 80 | gateway = MyGateway( 81 | modules=[h, AddChannelsToGraphOutput()], 82 | channels=MyGatewayChannels(), 83 | ) 84 | res = csp.run(gateway.graph, starttime=datetime(2020, 1, 1), endtime=timedelta(3)) 85 | 86 | assert "my_channel" in res 87 | assert len(res["my_channel"]) == 2 88 | expected_times = [ 89 | datetime(2020, 1, 2), 90 | datetime(2020, 1, 2, 0, 0, 5), 91 | ] 92 | for (actual_time, _), expected_time in zip(res["my_channel"], expected_times): 93 | assert actual_time == expected_time 94 | -------------------------------------------------------------------------------- /csp_gateway/tests/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/utils/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/utils/struct/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/utils/struct/__init__.py -------------------------------------------------------------------------------- /csp_gateway/tests/utils/struct/test_lookup.py: -------------------------------------------------------------------------------- 1 | from csp_gateway import GatewayStruct as Base 2 | 3 | 4 | class LookupModel(Base): 5 | foo: int = 9 6 | 7 | 8 | class NoLookupModel(Base): 9 | foo: int = 10 10 | 11 | 12 | NoLookupModel.omit_from_lookup(True) 13 | 14 | 15 | def test_automatic_id_generation(): 16 | for Model in [LookupModel, NoLookupModel]: 17 | o1 = Model() 18 | value1 = str(Model.id_generator.current()) 19 | assert o1.id == value1 20 | 21 | o2 = Model() 22 | value2 = str(Model.id_generator.current()) 23 | assert o2.id == value2 24 | assert o2.id == str(int(o1.id) + 1) 25 | 26 | if Model == LookupModel: 27 | assert Model.lookup(value1) == o1 28 | assert Model.lookup(value2) == o2 29 | 30 | 31 | def test_lookup_fails(): 32 | o1 = LookupModel() 33 | value1 = str(LookupModel.id_generator.current()) 34 | assert o1.id == value1 35 | 36 | o2 = LookupModel() 37 | value2 = str(LookupModel.id_generator.current()) 38 | assert o2.id == value2 39 | assert o2.id == str(int(o1.id) + 1) 40 | 41 | assert LookupModel.lookup(value1) == o1 42 | assert LookupModel.lookup(value2) == o2 43 | 44 | o1 = NoLookupModel() 45 | value1 = str(NoLookupModel.id_generator.current()) 46 | assert o1.id == value1 47 | 48 | o2 = NoLookupModel() 49 | value2 = str(NoLookupModel.id_generator.current()) 50 | assert o2.id == value2 51 | assert o2.id == str(int(o1.id) + 1) 52 | 53 | assert NoLookupModel.lookup(value1) is None 54 | assert NoLookupModel.lookup(value2) is None 55 | -------------------------------------------------------------------------------- /csp_gateway/tests/utils/test_csp.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from typing import Dict, List 3 | 4 | import csp 5 | import pytest 6 | 7 | from csp_gateway.utils import ( 8 | get_dict_basket_key_type, 9 | get_dict_basket_value_tstype, 10 | get_dict_basket_value_type, 11 | is_dict_basket, 12 | is_list_basket, 13 | set_alarm_and_fetch_alarm_time, 14 | to_list, 15 | ) 16 | 17 | 18 | def test_dict_basket_type_checks(): 19 | test_dict_typ = Dict[int, str] 20 | test_dict_basket_typ = Dict[int, csp.ts[int]] 21 | assert not is_dict_basket(test_dict_typ) 22 | assert is_dict_basket(test_dict_basket_typ) 23 | 24 | with pytest.raises(TypeError): 25 | get_dict_basket_key_type(test_dict_typ) 26 | 27 | with pytest.raises(TypeError): 28 | get_dict_basket_value_tstype(test_dict_typ) 29 | 30 | assert get_dict_basket_key_type(test_dict_basket_typ) is int 31 | assert get_dict_basket_value_tstype(test_dict_basket_typ) is csp.ts[int] 32 | assert get_dict_basket_value_type(test_dict_basket_typ) is int 33 | 34 | 35 | def test_is_list_basket(): 36 | test_list_typ = List[int] 37 | test_list_basket_typ = List[csp.ts[int]] 38 | assert not is_list_basket(test_list_typ) 39 | assert is_list_basket(test_list_basket_typ) 40 | 41 | 42 | def test_to_list(): 43 | out = csp.run( 44 | to_list, 45 | csp.const(9), 46 | starttime=datetime(2020, 1, 1), 47 | endtime=timedelta(1), 48 | ) 49 | assert out[0] == [(datetime(2020, 1, 1), [9])] 50 | 51 | 52 | def test_set_alarm_and_fetch_alarm_time(): 53 | out = csp.run( 54 | set_alarm_and_fetch_alarm_time, 55 | timedelta(), 56 | starttime=datetime(2020, 1, 1), 57 | endtime=timedelta(1), 58 | ) 59 | assert out["alarm_time"] == [(datetime(2020, 1, 1), datetime(2020, 1, 1))] 60 | assert out["alarm_ticked"] == [(datetime(2020, 1, 1), True)] 61 | 62 | out = csp.run( 63 | set_alarm_and_fetch_alarm_time, 64 | timedelta(minutes=1), 65 | starttime=datetime(2020, 1, 1), 66 | endtime=timedelta(1), 67 | ) 68 | assert out["alarm_time"] == [(datetime(2020, 1, 1), datetime(2020, 1, 1, 0, 1))] 69 | assert out["alarm_ticked"] == [(datetime(2020, 1, 1, 0, 1), True)] 70 | 71 | out = csp.run( 72 | set_alarm_and_fetch_alarm_time, 73 | datetime(2020, 1, 7), 74 | starttime=datetime(2020, 1, 1), 75 | endtime=timedelta(days=14), 76 | ) 77 | assert out["alarm_time"] == [(datetime(2020, 1, 1), datetime(2020, 1, 7))] 78 | assert out["alarm_ticked"] == [(datetime(2020, 1, 7), True)] 79 | 80 | out = csp.run( 81 | set_alarm_and_fetch_alarm_time, 82 | datetime(2021, 1, 1), 83 | starttime=datetime(2020, 1, 1), 84 | endtime=timedelta(days=14), 85 | ) 86 | assert out["alarm_time"] == [(datetime(2020, 1, 1), datetime(2021, 1, 1))] 87 | assert out["alarm_ticked"] == [] 88 | -------------------------------------------------------------------------------- /csp_gateway/tests/utils/test_picklable_queue.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | from csp_gateway.utils.picklable_queue import PickleableQueue 4 | 5 | 6 | def test_queue_pickling(): 7 | queue = PickleableQueue() 8 | unpickled_queue = pickle.loads(pickle.dumps(queue)) 9 | 10 | assert queue.queue == unpickled_queue.queue 11 | assert queue.maxsize == unpickled_queue.maxsize 12 | -------------------------------------------------------------------------------- /csp_gateway/tests/utils/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/csp_gateway/tests/utils/web/__init__.py -------------------------------------------------------------------------------- /csp_gateway/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .csp import * 2 | from .enums import * 3 | from .exceptions import * 4 | from .fastapi import query_json 5 | from .id_generator import get_counter 6 | from .picklable_queue import PickleableQueue 7 | from .struct import GatewayStruct, IdType 8 | from .web import * 9 | -------------------------------------------------------------------------------- /csp_gateway/utils/csp.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from typing import List, TypeVar, Union, get_args, get_origin # noqa: TYP001 3 | 4 | import csp 5 | from csp import Outputs, ts 6 | from csp.impl.types.tstype import TsType, isTsType 7 | from deprecation import deprecated 8 | 9 | T = TypeVar("T") 10 | K = TypeVar("K") 11 | V = TypeVar("V") 12 | 13 | __all__ = ( 14 | "_get_dict_basket_key_type", 15 | "_get_dict_basket_value_tstype", 16 | "_get_dict_basket_value_type", 17 | "_is_dict_basket", 18 | "_is_list_basket", 19 | "get_dict_basket_key_type", 20 | "get_dict_basket_value_tstype", 21 | "get_dict_basket_value_type", 22 | "is_dict_basket", 23 | "is_list_basket", 24 | "get_args", 25 | "get_origin", 26 | "set_alarm_and_fetch_alarm_time", 27 | "to_list", 28 | ) 29 | 30 | 31 | def is_dict_basket(val: type) -> bool: 32 | return get_origin(val) is dict and isTsType(get_args(val)[1]) 33 | 34 | 35 | def is_list_basket(val: type) -> bool: 36 | return get_origin(val) is list and isTsType(get_args(val)[0]) 37 | 38 | 39 | def get_dict_basket_key_type(val: type) -> type: 40 | if not is_dict_basket(val): 41 | raise TypeError(f"object is not a Dict Basket but is of type: {type(val)}") 42 | return get_args(val)[0] 43 | 44 | 45 | def get_dict_basket_value_tstype(val: type) -> TsType: 46 | if not is_dict_basket(val): 47 | raise TypeError(f"object is not a Dict Basket but is of type: {type(val)}") 48 | return get_args(val)[1] 49 | 50 | 51 | def get_dict_basket_value_type(val: type) -> type: 52 | return get_dict_basket_value_tstype(val).typ 53 | 54 | 55 | @deprecated(details="Use is_dict_basket instead.") 56 | def _is_dict_basket(val: type) -> bool: 57 | return is_dict_basket(val) 58 | 59 | 60 | @deprecated(details="Use is_list_basket instead.") 61 | def _is_list_basket(val: type) -> bool: 62 | return is_list_basket(val) 63 | 64 | 65 | @deprecated(details="Use get_dict_basket_key_type instead.") 66 | def _get_dict_basket_key_type(val: type) -> type: 67 | return get_dict_basket_key_type(val) 68 | 69 | 70 | @deprecated(details="Use get_dict_basket_value_tstype instead.") 71 | def _get_dict_basket_value_tstype(val: type) -> TsType: 72 | return get_dict_basket_value_tstype(val) 73 | 74 | 75 | @deprecated(details="Use get_dict_basket_value_type instead.") 76 | def _get_dict_basket_value_type(val: type) -> type: 77 | return get_dict_basket_value_type(val) 78 | 79 | 80 | @csp.node 81 | def to_list(x: ts["T"]) -> ts[List["T"]]: 82 | if csp.ticked(x): 83 | return [x] 84 | 85 | 86 | @csp.node 87 | def set_alarm_and_fetch_alarm_time(time: Union[datetime, timedelta]) -> Outputs(alarm_time=ts[datetime], alarm_ticked=ts[bool]): 88 | with csp.alarms(): 89 | engine_start: ts[bool] = csp.alarm(bool) 90 | alarm: ts[bool] = csp.alarm(bool) 91 | with csp.state(): 92 | s_time = None 93 | 94 | with csp.start(): 95 | csp.schedule_alarm(engine_start, timedelta(), True) 96 | csp.schedule_alarm(alarm, time, True) 97 | if isinstance(time, datetime): 98 | s_time = time 99 | else: 100 | s_time = csp.now() + time 101 | 102 | if csp.ticked(engine_start): 103 | csp.output(alarm_time=s_time) 104 | if csp.ticked(alarm): 105 | csp.output(alarm_ticked=True) 106 | -------------------------------------------------------------------------------- /csp_gateway/utils/enums.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class ReadWriteMode(str, Enum): 5 | """Enum representing whether a component is set to read, write, or both.""" 6 | 7 | READ = "READ" 8 | WRITE = "WRITE" 9 | READ_AND_WRITE = "READ_AND_WRITE" 10 | -------------------------------------------------------------------------------- /csp_gateway/utils/exceptions.py: -------------------------------------------------------------------------------- 1 | class GatewayException(Exception): ... 2 | 3 | 4 | class NoProviderException(GatewayException): ... 5 | 6 | 7 | class ServerException(GatewayException): ... 8 | 9 | 10 | class ServerRouteNotMountedException(ServerException): ... 11 | 12 | 13 | class ServerRouteNotFoundException(ServerException): ... 14 | 15 | 16 | class ServerUnprocessableException(ServerException): ... 17 | 18 | 19 | class ServerUnknownException(ServerException): ... 20 | 21 | 22 | class _Controls(GatewayException): 23 | def __init__(self, control: str): 24 | super().__init__("Control: {}".format(control)) 25 | -------------------------------------------------------------------------------- /csp_gateway/utils/fastapi.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional 2 | 3 | from fastapi import Depends, HTTPException, Query 4 | from pydantic import Json, ValidationError 5 | 6 | from .web import Query as QueryParamType 7 | 8 | 9 | def json_param(param_name: str, model: Any, **query_kwargs): 10 | """Parse JSON-encoded query parameters as pydantic models. 11 | The function returns a `Depends()` instance that takes the JSON-encoded value from 12 | the query parameter `param_name` and converts it to a Pydantic model, defined 13 | by the `model` attribute. 14 | """ 15 | 16 | def get_parsed_object(value: Optional[Json] = Query(default=None, alias=param_name, **query_kwargs)): 17 | try: 18 | if value is None: 19 | return None 20 | 21 | return model.model_validate(value) 22 | 23 | except ValidationError as err: 24 | raise HTTPException(400, detail=err.errors()) 25 | 26 | return Depends(get_parsed_object) 27 | 28 | 29 | def query_json(): 30 | # NOTE: if we switch querys back to be csp structs, do this: 31 | # return json_param("query", QueryParamType.__pydantic_model__, description="Query Parameters") 32 | return json_param("query", QueryParamType, description="Query Parameters") 33 | -------------------------------------------------------------------------------- /csp_gateway/utils/id_generator.py: -------------------------------------------------------------------------------- 1 | import typing 2 | from datetime import datetime 3 | 4 | from atomic_counter import Counter 5 | 6 | if typing.TYPE_CHECKING: 7 | from csp_gateway.server import GatewayModule 8 | 9 | 10 | def get_counter(kind: "GatewayModule"): 11 | if not hasattr(get_counter, "id_map"): 12 | get_counter.map = {} 13 | if kind not in get_counter.map: 14 | nowish = datetime.utcnow() 15 | base = datetime(nowish.year, nowish.month, nowish.day) 16 | get_counter.map[kind] = Counter(int(base.timestamp()) * 1_000_000_000) 17 | return get_counter.map[kind] 18 | -------------------------------------------------------------------------------- /csp_gateway/utils/picklable_queue.py: -------------------------------------------------------------------------------- 1 | from queue import Queue 2 | 3 | 4 | class PickleableQueue(Queue): 5 | """ 6 | An extension of the base Queue to allow it to be pickled 7 | 8 | NOTE: Pickled queues will not retain the contents of their queues 9 | """ 10 | 11 | def __init__(self, *args, **kwargs): 12 | super().__init__(*args, **kwargs) 13 | 14 | def __getstate__(self): 15 | return {} 16 | 17 | def __setstate__(self, state): 18 | self.__dict__.update(Queue().__dict__) 19 | -------------------------------------------------------------------------------- /csp_gateway/utils/struct/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import * 2 | from .psp import PerspectiveUtilityMixin 3 | -------------------------------------------------------------------------------- /csp_gateway/utils/web/__init__.py: -------------------------------------------------------------------------------- 1 | from .controls import Controls 2 | from .filter import Filter, FilterCondition, FilterWhere, FilterWhereLambdaMap 3 | from .query import Query 4 | -------------------------------------------------------------------------------- /csp_gateway/utils/web/controls.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | import orjson 4 | from typing_extensions import override 5 | 6 | from ..struct import GatewayStruct 7 | 8 | 9 | class Controls(GatewayStruct): 10 | name: str = "none" 11 | status: str = "none" 12 | data: dict = {} 13 | data_str: str = "" 14 | 15 | # TODO: all `GatewayStructs`` cross thread boundaries and should 16 | # have locks, but just doing `Controls`` for now 17 | _lock: object 18 | 19 | def lock(self, blocking=True, timeout=-1): 20 | # FIXME: first call is...not threadsafe 21 | if not hasattr(self, "_lock"): 22 | self._lock = threading.Lock() 23 | self._lock.acquire(blocking=blocking, timeout=timeout) 24 | 25 | def unlock(self): 26 | if not hasattr(self, "_lock"): 27 | self._lock = threading.Lock() 28 | if self._lock.locked(): 29 | self._lock.release() 30 | 31 | def update_str(self): 32 | # NOTE: This is needed to resolve a race condition between 33 | # Web API and Perspective. We need to create a data_str 34 | # because perspective cannot read python dict types and 35 | # csp.Structs.to_json creates dictionaries. This will 36 | # be removed when better to_json support is added. 37 | self.lock() 38 | if self.data_str == "": 39 | self.data_str = orjson.dumps(self.data).decode() 40 | self.unlock() 41 | 42 | @override 43 | def psp_flatten(self, custom_jsonifier=None): 44 | self.update_str() 45 | res = super().psp_flatten(custom_jsonifier) 46 | return res 47 | -------------------------------------------------------------------------------- /csp_gateway/utils/web/filter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from enum import Enum as PyEnum 4 | from typing import Literal, Optional, Union 5 | 6 | from csp.impl.enum import Enum as CspEnum, EnumMeta as CspEnumMeta 7 | from pydantic import BaseModel, Field 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | FilterWhere = Literal["==", "!=", "<", "<=", ">", ">="] 12 | FilterWhereLambdaMap = { 13 | "==": lambda a, b: a == b, 14 | "!=": lambda a, b: a != b, 15 | "<": lambda a, b: a < b, 16 | "<=": lambda a, b: a <= b, 17 | ">": lambda a, b: a > b, 18 | ">=": lambda a, b: a >= b, 19 | } 20 | 21 | 22 | class FilterCondition(BaseModel): 23 | # in priority order 24 | value: Optional[Union[float, int, str]] = Field(None) 25 | # have to handle separately otherwise 26 | # all ints would be datetimes.. 27 | when: Optional[datetime] = Field(None) 28 | attr: str = "" 29 | where: FilterWhere = "==" 30 | 31 | 32 | class Filter(BaseModel): 33 | attr: str = "" 34 | by: FilterCondition 35 | 36 | def calculate(self, obj) -> bool: 37 | """ 38 | calculate the filter condition on the object 39 | 40 | returns `True` if SHOULD NOT BE FILTERED, else `False` 41 | """ 42 | try: 43 | if self.by.value is not None: 44 | lhs = getattr(obj, self.attr) 45 | # Convert enums attrs to strings during filtering 46 | if isinstance(lhs, (CspEnum, PyEnum, CspEnumMeta)): 47 | lhs = lhs.name 48 | log.info(f"Filtering: {lhs} {self.by.where} {self.by.value}") 49 | return FilterWhereLambdaMap[self.by.where](lhs, self.by.value) 50 | if self.by.when is not None: 51 | log.info(f"Filtering: {getattr(obj, self.attr)} {self.by.where} {self.by.when}") 52 | return FilterWhereLambdaMap[self.by.where](getattr(obj, self.attr), self.by.when) 53 | if self.by.attr: 54 | log.info(f"Filtering: {getattr(obj, self.attr)} {self.by.where} {getattr(obj, self.by.attr)}") 55 | return FilterWhereLambdaMap[self.by.where](getattr(obj, self.attr), getattr(obj, self.by.attr)) 56 | except (ValueError, AttributeError) as e: 57 | # TODO probably surface to webserver 58 | log.warning(f"Error during filtering {type(obj)} / {self}: {e}") 59 | 60 | # default case, if there was an issue assume its not included 61 | return False 62 | -------------------------------------------------------------------------------- /csp_gateway/utils/web/query.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | from pydantic import BaseModel 5 | 6 | from .filter import Filter 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | class Query(BaseModel): 12 | filters: List[Filter] = [] 13 | 14 | def calculate(self, objs): 15 | """calculate and filter down objs by filters""" 16 | log.info(f"Querying {len(objs)} with query: {self}") 17 | return [o for o in objs if all(filter.calculate(o) for filter in self.filters)] 18 | -------------------------------------------------------------------------------- /docs/img/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/docs/img/demo.gif -------------------------------------------------------------------------------- /docs/img/logo-name-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/docs/img/logo-name-dark.png -------------------------------------------------------------------------------- /docs/img/logo-name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/docs/img/logo-name.png -------------------------------------------------------------------------------- /docs/wiki/API.md: -------------------------------------------------------------------------------- 1 | `csp-gateway` provides a [FastAPI](https://fastapi.tiangolo.com/) based REST API optionally for every instance. 2 | This provides a few nice features out of the box: 3 | 4 | - [OpenAPI](https://github.com/OAI/OpenAPI-Specification) based endpoints automatically derived from the underlying `csp` types into [JSON Schema](https://json-schema.org/) (`/openapi.json`) 5 | - [Swagger](https://github.com/swagger-api/swagger-ui) / [Redoc](https://github.com/Redocly/redoc) API documentation based on the `GatewayModule` and `GatewayChannels` in the application (`/docs` / `/redoc`) 6 | 7 | The `csp-gateway` REST API is designed to be simple to consume from outside `csp`, with a few fundamental methods. 8 | 9 | > [!NOTE] 10 | > 11 | > The REST API is launched when starting the `Gateway` instance with `rest=True` 12 | 13 | ## API 14 | 15 | As described in [Overview#Channels](Overview#Channels), the `csp-gateway` REST API has several methods for interacting with ticking / stateful data living on the `GatewayChannels` instance. 16 | 17 | - **last** (`GET`, `/api/v1/last/`): Get the last tick of data on a channel 18 | - **next** (`GET`, `/api/v1/next/`): Wait for the next tick of data on a channel: **WARNING**: blocks, and can often be misused into race conditions 19 | - **state** (`GET`, `/api/v1/state/`): Get the accumulated state for any channel 20 | - **send** (`POST`, `/api/v1/send/`): Send a new datum as a tick into the running csp graph 21 | - **lookup** (`POST`, `/api/v1/lookup//`): Lookup an individual GatewayStruct by its required `id` field 22 | 23 | > [!NOTE] 24 | > 25 | > Channels are included in the REST API by using a `GatewayModule`. 26 | > Most commonly, this is [`MountRestRoutes`](MountRestRoutes). 27 | 28 | > [!IMPORTANT] 29 | > 30 | > `lookup` has substantial memory overhead, as we cache a copy of every instance of every datum. 31 | > `GatewayModule` subclasses can disable it via the `classmethod` `omit_from_lookup`. 32 | 33 | ## State 34 | 35 | A `GatewayModule` can call `set_state` in its `connect` method to allow for this API to be available. 36 | State is collected by one or more instance attributes into an in-memory [DuckDB](https://duckdb.org/) instance. 37 | 38 | For example, suppose I have the following type: 39 | 40 | ```python 41 | class ExampleData(GatewayStruct): 42 | x: str 43 | y: str 44 | z: str 45 | ``` 46 | 47 | If my `GatewayModule` called `set_state("example", ("x",))`, state would be collected as the last tick of `ExampleData` per each unique value of `x`. If called with `set_state("example", ("x", "y"))`, it would be collected as the last tick per each unique pair `x,y`, etc. 48 | 49 | > [!IMPORTANT] 50 | > 51 | > This code and API will likely change a bit as we allow for more granular collection of records, 52 | > and expose more DuckDB functionality. 53 | 54 | ## Query 55 | 56 | [State](#State) accepts an additional query parameter `query`. 57 | This allows REST API users to query state and only return satisfying records. 58 | Here are some examples from the autodocumentation illustrating the use of filters: 59 | 60 | ```raw 61 | # Filter only records where `record.x` == 5 62 | api/v1/state/example?query={"filters":[{"attr":"x","by":{"value":5,"where":"=="}}]} 63 | 64 | # Filter only records where `record.x` < 10 65 | /api/v1/state/example?query={"filters":[{"attr":"x","by":{"value":10,"where":"<"}}]} 66 | 67 | # Filter only records where `record.timestamp` < "2023-03-30T14:45:26.394000" 68 | /api/v1/state/example?query={"filters":[{"attr":"timestamp","by":{"when":"2023-03-30T14:45:26.394000","where":"<"}}]} 69 | 70 | # Filter only records where `record.id` < `record.y` 71 | /api/v1/state/example?query={"filters":[{"attr":"id","by":{"attr":"y","where":"<"}}]} 72 | 73 | # Filter only records where `record.x` < 50 and `record.x` >= 30 74 | /api/v1/state/example?query={"filters":[{"attr":"x","by":{"value":50,"where":"<"}},{"attr":"x","by":{"value":30,"where":">="}}]} 75 | ``` 76 | 77 | > [!IMPORTANT] 78 | > 79 | > This code and API will likely change a bit as we expose more DuckDB functionality. 80 | 81 | ## Websockets 82 | 83 | In addition to a REST API, a more `csp`-like streaming API is also available via Websockets when the `MountWebSocketRoutes` module is included in a `Gateway` instance. 84 | 85 | This API is bidirectional, providing the ability to receive data as it ticks or to tick in new data. 86 | Any data available via `last`/`send` above is available via websocket. 87 | 88 | To subscribe to channels, send a JSON message of the form: 89 | 90 | ``` 91 | { 92 | "action": "subscribe", 93 | "channel": "" 94 | } 95 | ``` 96 | 97 | To unsubscribe to channels, send a JSON message of the form: 98 | 99 | ``` 100 | { 101 | "action": "unsubscribe", 102 | "channel": "" 103 | } 104 | ``` 105 | 106 | Data will be sent across the websocket for all subscribed channels. It has the form: 107 | 108 | ``` 109 | { 110 | "channel": "", 111 | "data": 112 | } 113 | ``` 114 | 115 | ## Python Client 116 | 117 | See [Client](Client) for details on our integrated python client. 118 | -------------------------------------------------------------------------------- /docs/wiki/CSP-Notes.md: -------------------------------------------------------------------------------- 1 | ## Why `csp-gateway`? 2 | 3 | Many `csp` users often question the necessity of `csp-gateway`. 4 | `csp-gateway` provides 3 key functionalities on top of the `csp`: 5 | 6 | ## Dependency Injection 7 | 8 | `csp` graphs are composed of individual nodes which are called when inputs update. 9 | Every `node` and `graph` can itself take a collection of ticking and non-ticking arguments. 10 | Let's say I want to create 2 version of the same graph: 11 | 12 | 1. Read from kafka topic `abc`, perform calculation `1`, write to kafka topic `ghi` 13 | 2. Read from kafka topic `abc`, perform calculation `2`, write to kafka topic `ghi` 14 | 15 | ```mermaid 16 | graph TB 17 | subgraph 1 18 | i1[Kafka topic='abc'] --> c1[Calculation 1] 19 | c1 --> o1[Kafka topic='ghi'] 20 | end 21 | subgraph 2 22 | i2[Kafka topic='def'] --> c2[Calculation 2] 23 | c2 --> o2[Kafka topic='ghi'] 24 | end 25 | ``` 26 | 27 | Python code for this might look like: 28 | 29 | ```python 30 | @csp.graph 31 | def my_graph_1(): 32 | kafka_in = KafkaInput(topic="abc") 33 | calculated = CalculationOne(kafka_in) 34 | KafkaOutput(topic="ghi") 35 | 36 | @csp.graph 37 | def my_graph_2(): 38 | kafka_in = KafkaInput(topic="def") 39 | calculated = CalculationTwo(kafka_in) 40 | KafkaOutput(topic="ghi") 41 | 42 | ``` 43 | 44 | Here, our `KafkaInput` node takes a static argument `topic`, while our `Calculation*` nodes take a ticking argument. 45 | 46 | `csp` code connects together nodes inside a `graph` in a point-to-point fashion. 47 | As we build more and more `csp` graphs, if we want to avoid code duplication, we end up writing some form of _graph builder_ logic. 48 | 49 | ```python 50 | def my_graph_builder(input_topic: str, output_topic: str, calculation: Node) 51 | kafka_in = KafkaInput(topic=input_topic) 52 | calculated = calculation(kafka_in) 53 | KafkaOutput(topic=output_topic) 54 | 55 | my_graph_1 = my_graph_builder("abc", "ghi", CalculationOne) 56 | my_graph_2 = my_graph_builder("def", "ghi", CalculationTwo) 57 | ``` 58 | 59 | As this starts to become increasingly complex, it is more and more difficult to configure graphs and nodes nested inside other graphs. 60 | 61 | `csp-gateway` solves this by combining `csp` with [`ccflow`](https://github.com/Point72/ccflow). 62 | 63 | `GatewayModule` instances are `ccflow` [BaseModel](https://github.com/Point72/ccflow/wiki/Key-Features#base-model) instances. 64 | This provides type validation, coercion, and dynamic initialization via [Pydantic](https://docs.pydantic.dev/latest/). 65 | Additionally, with `ccflow` you can overlay a configuration graph onto the initialization of your `csp-gateway` instances via [Hydra](https://hydra.cc/) / [OmegaConf](https://omegaconf.readthedocs.io/en/2.3_branch/) - See this [worked example](https://github.com/Point72/ccflow/wiki/First-Steps) for more information. 66 | 67 | For more documentation, see [Configuration](Configuration). 68 | 69 | ## Deferred Instantiation 70 | 71 | In a normal `csp` graph, nodes and graphs are wired together "point-to-point". 72 | In other words, upstream nodes and graphs must be instantiated first, with subsequent instantions moving from source to sink in a downstream fashion. 73 | Furthermore, graphs must end up acyclic by default. 74 | 75 | - If you want to instantiate in a different order, as you may very well want to do if building something like the above section, you need to use [csp.DelayedEdge](https://github.com/Point72/csp/wiki/Feedback-and-Delayed-Edge). 76 | - If you want a cyclic graph, you must manually insert [csp.Feedback](https://github.com/Point72/csp/wiki/Feedback-and-Delayed-Edge). 77 | 78 | Both of these two are a bit cumbersome, and do not gel well with the `csp-gateway` "Data Bus" oriented approach. 79 | Instead, `csp-gateway` automatically instantiates all channels in a `GatewayChannels` as `DelayedEdge`. 80 | When a `GatewayModule` sets them, they are bound with the real edge. 81 | If no `GatewayModule` sets them and they're not required, they are replaced with a `null_ts`. 82 | `Feedback` instances are automatically inserted where necessary. 83 | 84 | ## REST API/UI 85 | 86 | `csp` graphs can be difficult to interrogate, and the builtin mechanisms for doing so are `csp.print`/`csp.log` and `csp.show_graph` (generate a static `graphviz`-based graph of every node/edge). 87 | 88 | With an automatic REST API and UI, `csp-gateway` makes it easy to see every tick of data across every relevant data stream in your `csp` graph. 89 | Additionally, the [MountChannelsGraph](MountChannelsGraph) module provides a "30,000 foot view" of the graph, which is useful when the granular `csp.show_graph` becomes too complex. 90 | -------------------------------------------------------------------------------- /docs/wiki/Installation.md: -------------------------------------------------------------------------------- 1 | ## Pre-requisites 2 | 3 | You need Python >=3.9 on your machine to install `csp-gateway`. 4 | 5 | ## Install with `pip` 6 | 7 | ```bash 8 | pip install csp-gateway 9 | ``` 10 | 11 | ## Install with `conda` 12 | 13 | ```bash 14 | conda install csp-gateway --channel conda-forge 15 | ``` 16 | 17 | ## Source installation 18 | 19 | For other platforms and for development installations, [build `csp-gateway` from source](Build-from-Source). 20 | -------------------------------------------------------------------------------- /docs/wiki/UI.md: -------------------------------------------------------------------------------- 1 | `csp-gateway` provides an automatically generated UI based on React and [Perspective](https://perspective.finos.org). 2 | 3 | > [!NOTE] 4 | > To enable the UI, ensure you run your [`Gateway`](Overview#Gateway) with `ui=True` and include the `MountPerspectiveTables` module. 5 | 6 | ## Perspective 7 | 8 | Perspective is an interactive analytics and data visualization component, which is especially well-suited for large and/or streaming datasets. 9 | See the [Perspective Documentation](https://perspective.finos.org/guide/) (and the media section in particular) for more information on how to use Perspective. 10 | 11 | ## Top Bar 12 | 13 | The top bar has several buttons on the righthand side for selecting/saving/downloading layouts, toggling light/dark mode, and opening the settings drawer. 14 | 15 | ### Layouts 16 | 17 | Perspective layouts are driven via JSON. 18 | You can drag/drop to build your own layout, and click the save button to store it locally in your browser. 19 | Layouts can also be downloaded as a JSON, and integrated into the server-side configuration for sharing across multiple users. 20 | 21 | ```yaml 22 | modules: 23 | mount_perspective_tables: 24 | _target_: csp_gateway.MountPerspectiveTables 25 | layouts: 26 | A Layout Name: "" 27 | ``` 28 | 29 | ## Settings 30 | 31 | The rightmost top bar button opens the settings drawer. Depending on your server configuration, this has one or more [Controls](MountControls). 32 | 33 | - _"Big Red Button"_: Shut down the backend `Gateway` server 34 | - Email: if your server settings have an email contact, this will generate a `mailto:` link 35 | - Logs: if your server includes the [`MountOutputsFolder`](MountOutputsFolder) module, this will link to an integrated log and configuration viewer 36 | - Graph View: if your server includes the [`MountChannelsGraph`](MountChannelsGraph) module, this will link to an integrated graph viewer 37 | -------------------------------------------------------------------------------- /docs/wiki/_Footer.md: -------------------------------------------------------------------------------- 1 | _This wiki is autogenerated. To made updates, open a PR against the original source file in [`docs/wiki`](https://github.com/Point72/csp-gateway/tree/main/docs/wiki)._ 2 | -------------------------------------------------------------------------------- /docs/wiki/_Sidebar.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | **[Home](Home)** 10 | 11 | **Get Started** 12 | 13 | - [Installation](Installation) 14 | - [Overview](Overview) 15 | - [Engine](Overview#Engine) 16 | - [Channels](Overview#Channels) 17 | - [Modules](Overview#Modules) 18 | - [Gateway](Overview#Gateway) 19 | 20 | **Key Components** 21 | 22 | - [Configuration](Configuration) 23 | - [API](API) 24 | - [UI](UI) 25 | - [Client](Client) 26 | - [CSP Notes](CSP-Notes) 27 | 28 | **For Developers** 29 | 30 | - [Writing Modules](Develop#Writing-Modules) 31 | - [Extending the API](Develop#Extending-the-API) 32 | - [Extending the UI](Develop#Extending-the-UI) 33 | - [Advanced Usage](Develop#Advanced-Usage) 34 | 35 | **Modules** 36 | 37 | - API/UI Modules 38 | - [MountRestRoutes](Modules#MountRestRoutes) 39 | - [MountFieldRestRoutes](Modules#MountFieldRestRoutes) 40 | - [MountWebSocketRoutes](Modules#MountWebSocketRoutes) 41 | - [MountPerspectiveTables](Modules#MountPerspectiveTables) 42 | - [MountOutputsFolder](Modules#MountOutputsFolder) 43 | - [MountControls](Modules#MountControls) 44 | - [MountChannelsGraph](Modules#MountChannelsGraph) 45 | - [MountAPIKeyMiddleware](Modules#MountAPIKeyMiddleware) 46 | - Logging Modules 47 | - [PrintChannels](Modules#PrintChannels) 48 | - [LogChannels](Modules#LogChannels) 49 | - [PublishSQLA](Modules#PublishSQLA) 50 | - [PublishDatadog](Modules#PublishDatadog) 51 | - [PublishOpsGenie](Modules#PublishOpsGenie) 52 | - [PublishSymphony](Modules#PublishSymphony) 53 | - Replay Engine Modules 54 | - [ReplayEngineJSON](Modules#ReplayEngineJSON) 55 | - [ReplayEngineKafka](Modules#ReplayEngineKafka) 56 | - Utility Modules 57 | - [AddChannelsToGraphOutput](Modules#AddChannelsToGraphOutput) 58 | - [Initialize](Modules#Initialize) 59 | - [Mirror](Modules#Mirror) 60 | 61 | **For Contributors** 62 | 63 | - [Contributing](Contribute) 64 | - [Development Setup](Local-Development-Setup) 65 | - [Build from Source](Build-from-Source) 66 | -------------------------------------------------------------------------------- /docs/wiki/contribute/Build-from-Source.md: -------------------------------------------------------------------------------- 1 | `csp-gateway` is written in Python and Javascript. While prebuilt wheels are provided for end users, it is also straightforward to build `csp-gateway` from either the Python [source distribution](https://packaging.python.org/en/latest/specifications/source-distribution-format/) or the GitHub repository. 2 | 3 | ## Table of Contents 4 | 5 | - [Table of Contents](#table-of-contents) 6 | - [Make commands](#make-commands) 7 | - [Prerequisites](#prerequisites) 8 | - [Clone](#clone) 9 | - [Install NodeJS](#install-nodejs) 10 | - [Install Python dependencies](#install-python-dependencies) 11 | - [Build](#build) 12 | - [Lint and Autoformat](#lint-and-autoformat) 13 | - [Testing](#testing) 14 | 15 | ## Make commands 16 | 17 | As a convenience, `csp-gateway` uses a `Makefile` for commonly used commands. You can print the main available commands by running `make` with no arguments 18 | 19 | ```bash 20 | > make 21 | 22 | build build the library 23 | clean clean the repository 24 | fix run autofixers 25 | install install library 26 | lint run lints 27 | test run the tests 28 | ``` 29 | 30 | ## Prerequisites 31 | 32 | `csp-gateway` has a few system-level dependencies which you can install from your machine package manager. Other package managers like `conda`, `nix`, etc, should also work fine. 33 | 34 | ## Clone 35 | 36 | Clone the repo with: 37 | 38 | ```bash 39 | git clone https://github.com/Point72/csp-gateway.git 40 | cd csp-gateway 41 | ``` 42 | 43 | ## Install NodeJS 44 | 45 | Follow the instructions for [installing NodeJS](https://nodejs.org/en/download/package-manager/all) for your system. Once installed, you can [install `pnpm`](https://pnpm.io/installation) with: 46 | 47 | ```bash 48 | npm instal --global pnpm 49 | ``` 50 | 51 | ## Install Python dependencies 52 | 53 | Python build and develop dependencies are specified in the `pyproject.toml`, but you can manually install them: 54 | 55 | ```bash 56 | make requirements 57 | ``` 58 | 59 | Note that these dependencies would otherwise be installed normally as part of [PEP517](https://peps.python.org/pep-0517/) / [PEP518](https://peps.python.org/pep-0518/). 60 | 61 | ## Build 62 | 63 | Build the python project in the usual manner: 64 | 65 | ```bash 66 | make build 67 | ``` 68 | 69 | ## Lint and Autoformat 70 | 71 | `csp-gateway` has linting and auto formatting. 72 | 73 | | Language | Linter | Autoformatter | Description | 74 | | :--------- | :--------- | :------------ | :---------- | 75 | | Python | `ruff` | `ruff` | Style | 76 | | Python | `ruff` | `ruff` | Imports | 77 | | JavaScript | `prettier` | `prettier` | Style | 78 | | Markdown | `prettier` | `prettier` | Style | 79 | 80 | **Python Linting** 81 | 82 | ```bash 83 | make lint-py 84 | ``` 85 | 86 | **Python Autoformatting** 87 | 88 | ```bash 89 | make fix-py 90 | ``` 91 | 92 | **JavaScript Linting** 93 | 94 | ```bash 95 | make lint-js 96 | ``` 97 | 98 | **JavaScript Autoformatting** 99 | 100 | ```bash 101 | make fix-js 102 | ``` 103 | 104 | **Documentation Linting** 105 | 106 | We use `prettier` for our Markdown linting, so follow the above docs. 107 | 108 | ## Testing 109 | 110 | `csp-gateway` has both Python and JavaScript tests. The bulk of the functionality is tested in Python, which can be run via `pytest`. First, install the Python development dependencies with 111 | 112 | ```bash 113 | make develop 114 | ``` 115 | 116 | **Python** 117 | 118 | ```bash 119 | make test-py 120 | ``` 121 | 122 | **JavaScript** 123 | 124 | ```bash 125 | make test-js 126 | ``` 127 | -------------------------------------------------------------------------------- /docs/wiki/contribute/Contribute.md: -------------------------------------------------------------------------------- 1 | Contributions are welcome on this project. We distribute under the terms of the [Apache 2.0 license](https://github.com/Point72/csp-gateway/blob/main/LICENSE). 2 | 3 | > [!NOTE] 4 | > 5 | > `csp-gateway` requires [Developer Certificate of Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) for all contributions. 6 | > This is enforced by a [Probot GitHub App](https://probot.github.io/apps/dco/), which checks that commits are "signed". 7 | > Read [instructions to configure commit signing](Local-Development-Setup#configure-commit-signing). 8 | 9 | For **bug reports** or **small feature requests**, please open an issue on our [issues page](https://github.com/Point72/csp-gateway/issues). 10 | 11 | For **questions** or to discuss **larger changes or features**, please use our [discussions page](https://github.com/Point72/csp-gateway/discussions). 12 | 13 | For **contributions**, please see our [developer documentation](Local-Development-Setup). We have `help wanted` and `good first issue` tags on our issues page, so these are a great place to start. 14 | 15 | For **documentation updates**, make PRs that update the pages in `/docs/wiki`. The documentation is pushed to the GitHub wiki automatically through a GitHub workflow. Note that direct updates to this wiki will be overwritten. 16 | -------------------------------------------------------------------------------- /docs/wiki/contribute/Local-Development-Setup.md: -------------------------------------------------------------------------------- 1 | ## Table of Contents 2 | 3 | - [Table of Contents](#table-of-contents) 4 | - [Step 1: Build from Source](#step-1-build-from-source) 5 | - [Step 2: Configuring Git and GitHub for Development](#step-2-configuring-git-and-github-for-development) 6 | - [Create your fork](#create-your-fork) 7 | - [Configure remotes](#configure-remotes) 8 | - [Authenticating with GitHub](#authenticating-with-github) 9 | - [Guidelines](#guidelines) 10 | 11 | ## Step 1: Build from Source 12 | 13 | To work on `csp-gateway`, you are going to need to build it from source. See 14 | [Build from Source](Build-from-Source) for 15 | detailed build instructions. 16 | 17 | Once you've built `csp-gateway` from a `git` clone, you will also need to 18 | configure `git` and your GitHub account for `csp-gateway` development. 19 | 20 | ## Step 2: Configuring Git and GitHub for Development 21 | 22 | ### Create your fork 23 | 24 | The first step is to create a personal fork of `csp-gateway`. To do so, click 25 | the "fork" button at https://github.com/Point72/csp-gateway, or just navigate 26 | [here](https://github.com/Point72/csp-gateway/fork) in your browser. Set the 27 | owner of the repository to your personal GitHub account if it is not 28 | already set that way and click "Create fork". 29 | 30 | ### Configure remotes 31 | 32 | Next, you should set some names for the `git` remotes corresponding to 33 | main Point72 repository and your fork. See the [GitHub Docs](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/configuring-a-remote-repository-for-a-fork) for more information. 34 | 35 | ### Authenticating with GitHub 36 | 37 | If you have not already configured `ssh` access to GitHub, you can find 38 | instructions to do so 39 | [here](https://docs.github.com/en/authentication/connecting-to-github-with-ssh), 40 | including instructions to create an SSH key if you have not done 41 | so. Authenticating with SSH is usually the easiest route. If you are working in 42 | an environment that does not allow SSH connections to GitHub, you can look into 43 | [configuring a hardware 44 | passkey](https://docs.github.com/en/authentication/authenticating-with-a-passkey/about-passkeys) 45 | or adding a [personal access 46 | token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) 47 | to avoid the need to type in your password every time you push to your fork. 48 | 49 | ## Guidelines 50 | 51 | After developing a change locally, ensure that both [lints](Build-from-Source#lint-and-autoformat) and [tests](Build-from-Source#testing) pass. Commits should be squashed into logical units, and all commits must be signed (e.g. with the `-s` git flag). We require [Developer Certificate of Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) for all contributions. 52 | 53 | If your work is still in-progress, open a [draft pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests). Otherwise, open a normal pull request. It might take a few days for a maintainer to review and provide feedback, so please be patient. If a maintainer asks for changes, please make said changes and squash your commits if necessary. If everything looks good to go, a maintainer will approve and merge your changes for inclusion in the next release. 54 | 55 | Please note that non substantive changes, large changes without prior discussion, etc, are not accepted and pull requests may be closed. 56 | -------------------------------------------------------------------------------- /examples/kafka_example.py: -------------------------------------------------------------------------------- 1 | import csp 2 | import os 3 | 4 | from datetime import datetime, timedelta 5 | 6 | from csp_gateway import ( 7 | GatewayChannels, 8 | GatewayStruct, 9 | ChannelSelection, 10 | GatewayModule, 11 | Gateway, 12 | ReadWriteMode, 13 | ReplayEngineKafka, 14 | KafkaConfiguration, 15 | ReadWriteKafka, 16 | AddChannelsToGraphOutput, 17 | ) 18 | 19 | 20 | def create_kafka_config(): 21 | group_id = "foo" 22 | broker = "kafka-broker:9093" 23 | auth = True 24 | 25 | user_lower = os.environ.get("USER").lower() 26 | user_principal = os.environ.get("USER_PRINCIPAL_NAME") 27 | 28 | # NOTE: This file is used for verifying the Kafka broker 29 | ssl_ca_location = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" 30 | 31 | # NOTE: This will have to be set up by the user. 32 | kerberos_keytab = ( 33 | f'/home/{user_lower}/.keytab/{user_principal.split("@")[0]}.keytab.rc4-hmac' 34 | ) 35 | return KafkaConfiguration( 36 | group_id=group_id, 37 | broker=broker, 38 | sasl_kerberos_keytab=kerberos_keytab, 39 | sasl_kerberos_principal=user_principal, 40 | ssl_ca_location=ssl_ca_location, 41 | auth=auth, 42 | ) 43 | 44 | 45 | class MyStruct(GatewayStruct): 46 | foo: str 47 | 48 | 49 | class GWC(GatewayChannels): 50 | my_proclamation: csp.ts[MyStruct] = None 51 | 52 | 53 | class MyCallToTheWorld(GatewayModule): 54 | my_data: csp.ts[MyStruct] 55 | 56 | def connect(self, channels: GWC): 57 | channels.set_channel(GWC.my_proclamation, self.my_data) 58 | 59 | 60 | if __name__ == "__main__": 61 | # In this example, the read and write functionalities for the 62 | # ReadWriteKafka GatewayModule are shown. This allows users 63 | # to use Kafka to send and receive Gateway channel ticks on 64 | # Gateway Strcuts. 65 | 66 | # When ran, this file will print a struct called MyStruct, 67 | # with foo="HELLO WORLD!!" many times. One ReadWriteKafka instance 68 | # writes the struct to Kafka from the channel, and the other reads it 69 | # from Kafka and populates the channel. 70 | encoding_with_engine_timestamps = True 71 | 72 | set_module = MyCallToTheWorld(my_data=csp.const(MyStruct(foo="HELLO WORLD!!"))) 73 | channels = [GWC.my_proclamation] 74 | kafka_write_module = ReadWriteKafka( 75 | config=create_kafka_config(), 76 | publish_channel_to_topic_and_key={ 77 | GWC.my_proclamation: {"kafka_test": "kafka_read_write_example"} 78 | }, 79 | encoding_with_engine_timestamps=encoding_with_engine_timestamps, 80 | ) 81 | kafka_read_module = ReadWriteKafka( 82 | config=create_kafka_config(), 83 | subscribe_channel_to_topic_and_key={ 84 | GWC.my_proclamation: {"kafka_test": "kafka_read_write_example"} 85 | }, 86 | encoding_with_engine_timestamps=encoding_with_engine_timestamps, 87 | ) 88 | 89 | gateway = Gateway( 90 | modules=[ 91 | set_module, 92 | kafka_read_module, 93 | kafka_write_module, 94 | AddChannelsToGraphOutput(), 95 | ], 96 | channels=GWC(), 97 | channels_model=GWC, 98 | ) 99 | 100 | out = csp.run( 101 | gateway.graph, 102 | starttime=datetime.utcnow(), 103 | endtime=timedelta(seconds=2), 104 | realtime=True, 105 | ) 106 | print(f"{out[GWC.my_proclamation] = }") 107 | -------------------------------------------------------------------------------- /examples/multiprocessing_example.py: -------------------------------------------------------------------------------- 1 | """This example demonstrates a module that takes data from an input channel and applies some expensive calculation to it 2 | in a subprocess, and puts the results back onto an output channel. This may help throughput if the operation is a lot 3 | more expensive than the cost of pickling/unpickling the data and all the IPC stuff from multiprocessing. 4 | 5 | TODO: Extend this for simulation mode - possibly by enforcing that each input must have an output. 6 | """ 7 | 8 | import csp 9 | import logging 10 | import numpy as np 11 | from csp import ts 12 | from ccflow import Frequency 13 | from datetime import datetime, timedelta 14 | from multiprocessing import Process, Queue 15 | from pydantic import Field 16 | 17 | from csp_gateway import Gateway, GatewayChannels, GatewayModule, GatewayStruct 18 | 19 | 20 | class MyInputStruct(GatewayStruct): 21 | x: int 22 | 23 | 24 | class MyComputedStruct(GatewayStruct): 25 | x: int 26 | 27 | 28 | class MyGatewayChannel(GatewayChannels): 29 | input_data: ts[MyInputStruct] = None 30 | computed_data: ts[MyComputedStruct] = None 31 | 32 | 33 | def _some_really_expensive_operation(x: MyInputStruct): 34 | # "Expensive" operation. 35 | return MyComputedStruct(x=x.x + 1) 36 | 37 | 38 | def _run_worker(input_queue, output_queue): 39 | while True: 40 | x: MyInputStruct = input_queue.get(block=True) 41 | output = _some_really_expensive_operation(x) 42 | output_queue.put_nowait(output) 43 | 44 | 45 | class MyInputModule(GatewayModule): 46 | """Module for generating random inputs.""" 47 | 48 | def connect(self, channels: MyGatewayChannel): 49 | trigger = csp.timer(timedelta(seconds=1)) 50 | input_data = self._generate_random_inputs(trigger) 51 | channels.set_channel("input_data", input_data) 52 | 53 | @staticmethod 54 | @csp.node 55 | def _generate_random_inputs(trigger: ts[bool]) -> csp.Outputs(ts[MyInputStruct]): 56 | if csp.ticked(trigger): 57 | return MyInputStruct(x=int(np.random.uniform(-100, 100))) 58 | 59 | 60 | class MyMultiProcessingModule(GatewayModule): 61 | """Module for applying some very expensive function in a sub-process.""" 62 | 63 | drain_queue_interval: Frequency = Field(Frequency("1s")) 64 | 65 | def connect(self, channels: MyGatewayChannel): 66 | input_data = channels.get_channel("input_data") 67 | # Use queues for communication between this module and the worker process. 68 | input_queue = Queue() # Us -> worker 69 | output_queue = Queue() # Worker -> us 70 | 71 | # Start worker. 72 | p = Process(target=_run_worker, args=(input_queue, output_queue)) 73 | p.start() 74 | 75 | # Queue inputs + drain queue 76 | drain_queue = csp.timer(self.drain_queue_interval.timedelta) 77 | computed_data = csp.unroll(self._process_monitor(input_data, drain_queue, input_queue, output_queue, p)) 78 | channels.set_channel("computed_data", computed_data) 79 | 80 | # Log so we can see outputs whilst running. 81 | csp.log(logging.INFO, "input_data", input_data) 82 | csp.log(logging.INFO, "computed_data", computed_data) 83 | 84 | @staticmethod 85 | @csp.node 86 | def _process_monitor(x: ts[MyInputStruct], drain_queue: ts[bool], input_queue: object, output_queue: object, process: object) -> csp.Outputs( 87 | ts[[MyComputedStruct]] 88 | ): 89 | with csp.stop(): 90 | process.join(0) 91 | if process.is_alive(): 92 | process.terminate() 93 | 94 | if csp.ticked(x): 95 | input_queue.put_nowait(x) 96 | 97 | if csp.ticked(drain_queue): 98 | output = [] 99 | while not output_queue.empty(): 100 | output.append(output_queue.get_nowait()) 101 | 102 | if output: 103 | return output 104 | 105 | 106 | if __name__ == "__main__": 107 | logging.basicConfig(level=logging.INFO) 108 | input_module = MyInputModule() 109 | multi_processing_module = MyMultiProcessingModule() 110 | gateway = Gateway( 111 | modules=[ 112 | input_module, 113 | multi_processing_module, 114 | ], 115 | channels=MyGatewayChannel(), 116 | channels_model=MyGatewayChannel, 117 | ) 118 | 119 | out = csp.run( 120 | gateway.graph, 121 | starttime=datetime.utcnow(), 122 | endtime=timedelta(seconds=10), 123 | realtime=True, 124 | ) 125 | -------------------------------------------------------------------------------- /examples/websocket_client.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | from csp_gateway import GatewayClient, AsyncGatewayClient, GatewayClientConfig 4 | 5 | 6 | # Put your configuration here 7 | config = GatewayClientConfig(host="HOSTNAME", port=8000, authenticate=True, api_key="12345") 8 | 9 | 10 | if __name__ == "__main__": 11 | parser = argparse.ArgumentParser(prog='WebSocket demo') 12 | parser.add_argument('choice', choices=['sync', 'async']) 13 | args = parser.parse_args() 14 | 15 | if args.choice == "sync": 16 | print("Running synchronous example...") 17 | sync_client = GatewayClient(config) 18 | 19 | # Sync Example 20 | sync_client.stream(channels=["example", "example_list"], callback=print) 21 | 22 | else: 23 | print("Running asynchronous example...") 24 | async_client = AsyncGatewayClient(config) 25 | 26 | # Async example 27 | async def print_all(): 28 | async for datum in async_client.stream(channels=[]): 29 | print(datum) 30 | 31 | async def subscribe(): 32 | await async_client.subscribe("example") 33 | await async_client.subscribe("example_list") 34 | await async_client.publish("example", {"x": 12345, "y": "54321"}) 35 | 36 | all_routines = asyncio.gather(print_all(), subscribe()) 37 | 38 | asyncio.get_event_loop().run_until_complete(all_routines) 39 | -------------------------------------------------------------------------------- /js/README.md: -------------------------------------------------------------------------------- 1 | # @point72/csp-gateway 2 | 3 | Frontend assets for `csp-gateway`. 4 | 5 | ## Installation 6 | 7 | ```bash 8 | npm install @point72/csp-gateway 9 | ``` 10 | 11 | ## Usage 12 | 13 | See [our wiki](https://github.com/Point72/csp-gateway/wiki/Develop#Extending-the-UI). 14 | -------------------------------------------------------------------------------- /js/build.mjs: -------------------------------------------------------------------------------- 1 | import { NodeModulesExternal } from "@finos/perspective-esbuild-plugin/external.js"; 2 | import { build } from "@finos/perspective-esbuild-plugin/build.js"; 3 | import { BuildCss } from "@prospective.co/procss/target/cjs/procss.js"; 4 | import cpy from "cpy"; 5 | import fs from "fs"; 6 | import { createRequire } from "node:module"; 7 | 8 | const BUILD = [ 9 | { 10 | define: { 11 | global: "window", 12 | }, 13 | entryPoints: ["src/index.jsx"], 14 | plugins: [NodeModulesExternal()], 15 | format: "esm", 16 | loader: { 17 | ".css": "text", 18 | ".html": "text", 19 | ".jsx": "jsx", 20 | ".png": "file", 21 | ".ttf": "file", 22 | ".wasm": "file", 23 | }, 24 | outfile: "./lib/index.js", 25 | }, 26 | { 27 | define: { 28 | global: "window", 29 | }, 30 | entryPoints: ["src/index.jsx"], 31 | bundle: true, 32 | plugins: [], 33 | format: "esm", 34 | loader: { 35 | ".css": "text", 36 | ".html": "text", 37 | ".jsx": "jsx", 38 | ".png": "file", 39 | ".ttf": "file", 40 | ".wasm": "file", 41 | }, 42 | outfile: "../csp_gateway/server/build/main.js", 43 | publicPath: "/static/", 44 | }, 45 | ]; 46 | 47 | const require = createRequire(import.meta.url); 48 | function add(builder, path, path2) { 49 | builder.add(path, fs.readFileSync(require.resolve(path2 || path)).toString()); 50 | } 51 | 52 | async function compile_css() { 53 | const builder1 = new BuildCss(""); 54 | add(builder1, "./src/style/index.css"); 55 | add(builder1, "./src/style/base.css"); 56 | add(builder1, "./src/style/nord.css"); 57 | add(builder1, "./src/style/header_footer.css"); 58 | add(builder1, "./src/style/perspective.css"); 59 | add(builder1, "./src/style/settings.css"); 60 | add( 61 | builder1, 62 | "perspective-viewer-pro.css", 63 | "@finos/perspective-viewer/dist/css/pro.css", 64 | ); 65 | add( 66 | builder1, 67 | "perspective-viewer-pro-dark.css", 68 | "@finos/perspective-viewer/dist/css/pro-dark.css", 69 | ); 70 | add( 71 | builder1, 72 | "perspective-viewer-monokai.css", 73 | "@finos/perspective-viewer/dist/css/monokai.css", 74 | ); 75 | add( 76 | builder1, 77 | "perspective-viewer-vaporwave.css", 78 | "@finos/perspective-viewer/dist/css/vaporwave.css", 79 | ); 80 | add( 81 | builder1, 82 | "perspective-viewer-dracula.css", 83 | "@finos/perspective-viewer/dist/css/dracula.css", 84 | ); 85 | add( 86 | builder1, 87 | "perspective-viewer-gruvbox.css", 88 | "@finos/perspective-viewer/dist/css/gruvbox.css", 89 | ); 90 | add( 91 | builder1, 92 | "perspective-viewer-gruvbox-dark.css", 93 | "@finos/perspective-viewer/dist/css/gruvbox-dark.css", 94 | ); 95 | add( 96 | builder1, 97 | "perspective-viewer-solarized.css", 98 | "@finos/perspective-viewer/dist/css/solarized.css", 99 | ); 100 | add( 101 | builder1, 102 | "perspective-viewer-solarized-dark.css", 103 | "@finos/perspective-viewer/dist/css/solarized-dark.css", 104 | ); 105 | add( 106 | builder1, 107 | "react-modern-drawer.css", 108 | "react-modern-drawer/dist/index.css", 109 | ); 110 | 111 | const css = builder1.compile().get("index.css"); 112 | 113 | // write to extension 114 | fs.writeFileSync("../csp_gateway/server/build/index.css", css); 115 | } 116 | 117 | async function cp_to_paths(path) { 118 | await cpy(path, "../csp_gateway/server/build/", { flat: true }); 119 | } 120 | 121 | async function build_all() { 122 | /* make directories */ 123 | fs.mkdirSync("../csp_gateway/server/build/", { recursive: true }); 124 | 125 | /* Compile and copy JS */ 126 | await Promise.all(BUILD.map(build)).catch(() => process.exit(1)); 127 | // await cp_to_paths("./src/style/*.css"); 128 | await cp_to_paths("./src/html/*.html"); 129 | await cp_to_paths( 130 | "node_modules/@finos/perspective/dist/wasm/perspective-server.wasm", 131 | ); 132 | await cp_to_paths( 133 | "node_modules/@finos/perspective-viewer/dist/wasm/perspective-viewer.wasm", 134 | ); 135 | 136 | /* Compile and copy css */ 137 | await compile_css(); 138 | } 139 | 140 | build_all(); 141 | -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@point72/csp-gateway", 3 | "version": "2.1.0", 4 | "description": "CSP Gateway with Perspective Workspace", 5 | "author": "CSPOpenSource@point72.com", 6 | "license": "Apache-2.0", 7 | "main": "./lib/index.js", 8 | "scripts": { 9 | "build": "node build.mjs", 10 | "clean": "rimraf dist", 11 | "lint": "prettier --check \"src/**/*.js\" \"src/**/*.jsx\" \"src/style/*.css\" \"src/html/*.html\" \"*.mjs\" \"*.json\" \"*.md\" \"../*.md\" \"../docs/wiki/**/*.md\"", 12 | "fix": "prettier --write \"src/**/*.js\" \"src/**/*.jsx\" \"src/style/*.css\" \"src/html/*.html\" \"*.mjs\" \"*.json\" \"*.md\" \"../*.md\" \"../docs/wiki/**/*.md\"", 13 | "test": "echo \"todo\"", 14 | "preinstall": "npx only-allow pnpm", 15 | "prepack": "pnpm run build" 16 | }, 17 | "dependencies": { 18 | "@finos/perspective": "3.6.1", 19 | "@finos/perspective-viewer": "3.6.1", 20 | "@finos/perspective-viewer-d3fc": "3.6.1", 21 | "@finos/perspective-viewer-datagrid": "3.6.1", 22 | "@finos/perspective-workspace": "3.6.1", 23 | "react": "^19.1.0", 24 | "react-dom": "^19.1.0", 25 | "react-icons": "^4.10.1", 26 | "react-modern-drawer": "^1.2.2" 27 | }, 28 | "devDependencies": { 29 | "@finos/perspective-esbuild-plugin": "3.2.1", 30 | "@prospective.co/procss": "^0.1.16", 31 | "cpy": "^11.1.0", 32 | "esbuild": "^0.25.1", 33 | "npm-cli-login": "1.0.0", 34 | "npm-run-all": "^4.1.5", 35 | "prettier": "^3.5.3" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /js/src/common.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line import/no-mutable-exports, prefer-const 2 | export let CUSTOM_LAYOUT_CONFIG_NAME = "csp_gateway_demo_config"; 3 | 4 | export const changeLayoutConfigName = (newName) => { 5 | CUSTOM_LAYOUT_CONFIG_NAME = newName; 6 | }; 7 | 8 | export const hideLoader = () => { 9 | setTimeout(() => { 10 | const progress = document.getElementById("progress"); 11 | progress.setAttribute("style", "display:none;"); 12 | }, 3000); 13 | }; 14 | 15 | export const getOpenApi = async () => { 16 | const openapi = await fetch( 17 | `${window.location.protocol}//${window.location.host}/openapi.json`, 18 | ); 19 | const json = await openapi.json(); 20 | return json; 21 | }; 22 | 23 | export const shutdownDefault = async () => { 24 | // TODO check if can shutdown by checking openapi 25 | await fetch( 26 | `${window.location.protocol}//${window.location.host}/api/v1/controls/shutdown`, 27 | { method: "POST" }, 28 | ); 29 | }; 30 | -------------------------------------------------------------------------------- /js/src/components/footer.jsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { FaGithub } from "react-icons/fa"; 3 | 4 | export function Footer(props) { 5 | // overrideable data 6 | let { footerLogo } = props; 7 | 8 | return ( 9 |
10 |
11 | {footerLogo !== undefined && footerLogo} 12 |
13 |
14 | 19 | 20 | 21 |

22 | Built with{" "} 23 | 24 | Perspective 25 | 26 |

27 |
28 |
29 | ); 30 | } 31 | 32 | export default Footer; 33 | -------------------------------------------------------------------------------- /js/src/components/header.jsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { FaBars, FaDownload, FaMoon, FaSave, FaSun } from "react-icons/fa"; 3 | import { CspGatewayLogo } from "./logo"; 4 | import { saveCustomLayout } from "./perspective/layout"; 5 | import { getCurrentTheme } from "./perspective/theme"; 6 | 7 | const ICON_SIZE = 20; 8 | 9 | export function Header(props) { 10 | // theme and layout data 11 | const { layouts, theme } = props; 12 | 13 | // toggles and state changers 14 | const { changeLayouts, toggleTheme } = props; 15 | 16 | // settings toggle 17 | const { toggleSettings } = props; 18 | 19 | // openapi data 20 | const { openapi } = props; 21 | 22 | // overrideable data 23 | let { headerLogo } = props; 24 | 25 | if (headerLogo === undefined) { 26 | headerLogo = ; 27 | } 28 | 29 | if (openapi?.info.title) { 30 | document.title = openapi.info.title; 31 | } 32 | 33 | return ( 34 |
35 | {/* Left Aligned header */} 36 |
37 | {headerLogo} 38 |

{openapi?.info.title}

39 |

{openapi?.info.version}

40 |
41 | 42 | {/* Right Aligned header */} 43 |
44 | {/* Layout dropdown */} 45 | 62 | 63 | {/* Save current layout */} 64 | 82 | 83 | {/* Download current layout */} 84 | 101 | 102 | {/* Light / Dark theme switch */} 103 | 118 | 119 | {/* Settings drawer */} 120 | 128 |
129 |
130 | ); 131 | } 132 | 133 | export default Header; 134 | -------------------------------------------------------------------------------- /js/src/components/index.js: -------------------------------------------------------------------------------- 1 | export * from "./perspective"; 2 | export * from "./footer"; 3 | export * from "./header"; 4 | export * from "./settings"; 5 | -------------------------------------------------------------------------------- /js/src/components/logo.jsx: -------------------------------------------------------------------------------- 1 | import React, { Component } from "react"; 2 | import Logo from "../style/favicon.png"; 3 | 4 | export const CspGatewayLogo = ({ size = 40 }) => ( 5 | 6 | ); 7 | -------------------------------------------------------------------------------- /js/src/components/perspective/gateway.js: -------------------------------------------------------------------------------- 1 | import { version } from "@finos/perspective/package.json"; 2 | 3 | export const getDefaultViewerConfig = (tableName, schema, theme = "light") => { 4 | const viewer_config = { 5 | title: tableName, 6 | table: tableName, 7 | sort: [["timestamp", "desc"]], 8 | theme: theme === "dark" ? "Pro Dark" : "Pro Light", 9 | version, 10 | }; 11 | 12 | // setup groupbys and pivoting if available 13 | if (Object.keys(schema).includes("id")) { 14 | // include all columns except id 15 | viewer_config.columns = Object.keys(schema).filter((col) => col !== "id"); 16 | } 17 | 18 | return viewer_config; 19 | }; 20 | -------------------------------------------------------------------------------- /js/src/components/perspective/index.js: -------------------------------------------------------------------------------- 1 | export * from "./gateway"; 2 | export * from "./layout"; 3 | export * from "./tables"; 4 | export * from "./theme"; 5 | export * from "./workspace"; 6 | -------------------------------------------------------------------------------- /js/src/components/perspective/layout.js: -------------------------------------------------------------------------------- 1 | import { CUSTOM_LAYOUT_CONFIG_NAME } from "../../common"; 2 | 3 | export const getCustomLayout = () => { 4 | const possibleLayout = window.localStorage.getItem(CUSTOM_LAYOUT_CONFIG_NAME); 5 | return possibleLayout ? { "Custom Layout": JSON.parse(possibleLayout) } : {}; 6 | }; 7 | 8 | export const saveCustomLayout = (layout) => { 9 | window.localStorage.setItem( 10 | CUSTOM_LAYOUT_CONFIG_NAME, 11 | JSON.stringify(layout), 12 | ); 13 | }; 14 | 15 | export const getServerDefinedLayouts = async () => { 16 | const data = await fetch( 17 | `${window.location.protocol}//${window.location.host}/api/v1/perspective/layouts`, 18 | ); 19 | if (data.status === 403) { 20 | window.location.replace( 21 | `${window.location.protocol}//${window.location.host}/login${window.location.search}`, 22 | ); 23 | } 24 | const json = await data.json(); 25 | Object.keys(json).forEach((key) => { 26 | json[key] = JSON.parse(json[key]); 27 | }); 28 | return json; 29 | }; 30 | 31 | export const getDefaultWorkspaceLayout = () => ({ 32 | sizes: [1], 33 | detail: { 34 | main: { 35 | type: "tab-area", 36 | widgets: [], 37 | currentIndex: 0, 38 | }, 39 | }, 40 | master: { 41 | sizes: [], 42 | widgets: [], 43 | }, 44 | mode: "globalFilters", 45 | viewers: {}, 46 | }); 47 | -------------------------------------------------------------------------------- /js/src/components/perspective/tables.js: -------------------------------------------------------------------------------- 1 | import perspective from "@finos/perspective"; 2 | import perspective_viewer from "@finos/perspective-viewer"; 3 | import SERVER_WASM from "@finos/perspective/dist/wasm/perspective-server.wasm"; 4 | import CLIENT_WASM from "@finos/perspective-viewer/dist/wasm/perspective-viewer.wasm"; 5 | 6 | import "@finos/perspective-workspace"; 7 | import "@finos/perspective-viewer-datagrid"; 8 | import "@finos/perspective-viewer-d3fc"; 9 | 10 | const perspective_init_promise = Promise.all([ 11 | perspective.init_server(fetch(SERVER_WASM)), 12 | perspective_viewer.init_client(fetch(CLIENT_WASM)), 13 | ]); 14 | 15 | export const fetchTables = async () => { 16 | await perspective_init_promise; 17 | const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; 18 | const websocket = await perspective.websocket( 19 | `${protocol}//${window.location.host}/api/v1/perspective`, 20 | ); 21 | 22 | const response = await fetch("/api/v1/perspective/tables"); 23 | const schemas = await response.json(); 24 | const table_names = [...Object.keys(schemas)]; 25 | const tables = await Promise.all( 26 | table_names.map((table_name) => websocket.open_table(table_name)), 27 | ); 28 | const new_tables = {}; 29 | table_names.forEach((table_name, index) => { 30 | new_tables[table_name] = { 31 | table: tables[index], 32 | schema: schemas[table_name], 33 | }; 34 | }); 35 | return new_tables; 36 | }; 37 | -------------------------------------------------------------------------------- /js/src/components/perspective/theme.js: -------------------------------------------------------------------------------- 1 | export const getCurrentTheme = () => 2 | // handle theme 3 | document.documentElement.getAttribute("data-theme") || "light"; 4 | 5 | export const setupStoredOrDefaultTheme = () => { 6 | const storedOrDefaultTheme = 7 | localStorage.getItem("theme") || 8 | (window.matchMedia("(prefers-color-scheme: dark)").matches 9 | ? "dark" 10 | : "light"); 11 | document.documentElement.setAttribute("data-theme", storedOrDefaultTheme); 12 | return storedOrDefaultTheme; 13 | }; 14 | 15 | export const toggleTheme = (updateState) => (targetTheme) => { 16 | document.documentElement.setAttribute("data-theme", targetTheme); 17 | localStorage.setItem("theme", targetTheme); 18 | 19 | if (targetTheme === "dark") { 20 | document.querySelectorAll("perspective-viewer").forEach((viewer) => { 21 | viewer.restore({ theme: "Pro Dark" }); 22 | }); 23 | } else { 24 | document.querySelectorAll("perspective-viewer").forEach((viewer) => { 25 | viewer.restore({ theme: "Pro Light" }); 26 | }); 27 | } 28 | 29 | updateState(targetTheme); 30 | }; 31 | -------------------------------------------------------------------------------- /js/src/components/perspective/workspace.jsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useRef, useState } from "react"; 2 | import { getDefaultViewerConfig } from "./gateway"; 3 | import { fetchTables } from "./tables"; 4 | import { getDefaultWorkspaceLayout } from "./layout"; 5 | import { getCurrentTheme } from "./theme"; 6 | import { hideLoader } from "../../common"; 7 | 8 | export function Workspace(props) { 9 | // standard attributes 10 | const { layouts } = props; 11 | 12 | // standard modifiers 13 | const { changeLayouts } = props; 14 | 15 | // any overrides to process tables with custom logic 16 | const { processTables } = props; 17 | 18 | const workspace = useRef(null); 19 | let prevLayouts = useRef({ ...layouts, active: "Default" }); 20 | let layoutUpdate = useRef(false); 21 | 22 | // restore layout when it changes 23 | useEffect(() => { 24 | (async () => { 25 | if ( 26 | workspace.current && 27 | layouts && 28 | Object.keys(layouts[layouts.active] || {}).length > 0 29 | ) { 30 | if ( 31 | prevLayouts.current.active !== layouts.active && 32 | JSON.stringify(prevLayouts.current[prevLayouts.current.active]) !== 33 | JSON.stringify(layouts[layouts.active]) 34 | ) { 35 | layoutUpdate.current = true; 36 | 37 | // handle theme if not set 38 | const theme = getCurrentTheme(); 39 | 40 | const layout = structuredClone(layouts[layouts.active]); 41 | if (layout !== undefined && Object.keys(layout.viewers).length > 0) { 42 | Object.keys(layout.viewers).forEach((viewer_id) => { 43 | const viewer = layout.viewers[viewer_id]; 44 | if (!viewer.theme) { 45 | viewer.theme = theme === "dark" ? "Pro Dark" : "Pro Light"; 46 | } 47 | }); 48 | await workspace.current.restore(layout); 49 | } 50 | 51 | // update previous ref 52 | prevLayouts.current = { ...layouts, active: layouts.active }; 53 | layoutUpdate.current = false; 54 | } 55 | } 56 | })(); 57 | }, [layouts]); 58 | 59 | // setup tables 60 | useEffect(() => { 61 | if (workspace.current) { 62 | fetchTables().then((tables) => { 63 | // load tables into perspective workspace 64 | const to_restore = getDefaultWorkspaceLayout(); 65 | 66 | // handle theme 67 | const theme = getCurrentTheme(); 68 | 69 | // handle tables 70 | if (processTables) { 71 | processTables(to_restore, tables, workspace.current, theme); 72 | } else { 73 | const sortedTables = Object.keys(tables); 74 | sortedTables.sort(); 75 | sortedTables.forEach((tableName, index) => { 76 | const { table, schema } = tables[tableName]; 77 | workspace.current.addTable(tableName, table); 78 | const generated_id = `${tableName.toUpperCase()}_GENERATED_${index + 1}`; 79 | to_restore.detail.main.widgets.push(generated_id); 80 | to_restore.viewers[generated_id] = getDefaultViewerConfig( 81 | tableName, 82 | schema, 83 | theme, 84 | ); 85 | }); 86 | } 87 | 88 | // hide the progress bar 89 | hideLoader(); 90 | 91 | // restore 92 | workspace.current.restore(to_restore).then(() => { 93 | // setup new default layout 94 | prevLayouts = { ...layouts, Default: to_restore, active: "Default" }; 95 | changeLayouts(prevLayouts); 96 | 97 | // handle light/dark theme 98 | workspace.current.addEventListener( 99 | "workspace-new-view", 100 | async (event) => { 101 | const { widget } = event.detail; 102 | event.preventDefault(); 103 | event.stopPropagation(); 104 | const theme = getCurrentTheme(); 105 | if (!layoutUpdate.current) { 106 | if (theme === "dark") { 107 | // console.log("calling restore dark from workspace-new-view"); 108 | await widget.restore({ 109 | theme: "Pro Dark", 110 | sort: [["timestamp", "desc"]], 111 | }); 112 | } else { 113 | // console.log("calling restore light from workspace-new-view"); 114 | await widget.restore({ 115 | theme: "Pro Light", 116 | sort: [["timestamp", "desc"]], 117 | }); 118 | } 119 | } 120 | }, 121 | ); 122 | }); 123 | }); 124 | } 125 | }, []); 126 | 127 | return ; 128 | } 129 | 130 | export default Workspace; 131 | -------------------------------------------------------------------------------- /js/src/html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | CSP Gateway Demo 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 | 22 | 23 | -------------------------------------------------------------------------------- /js/src/index.jsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from "react"; 2 | import { createRoot } from "react-dom/client"; 3 | 4 | import { getOpenApi } from "./common"; 5 | import { Header, Footer, Settings, Workspace } from "./components"; 6 | import { 7 | toggleTheme, 8 | setupStoredOrDefaultTheme, 9 | getCustomLayout, 10 | getServerDefinedLayouts, 11 | } from "./components/perspective"; 12 | 13 | /* exports */ 14 | export * from "./common"; 15 | export * from "./components"; 16 | 17 | export default function App(props) { 18 | const { 19 | headerLogo, 20 | footerLogo, 21 | processTables, 22 | overrideSettingsButtons, 23 | extraSettingsButtons, 24 | shutdown, 25 | } = props; 26 | 27 | /** 28 | * OpenAPI 29 | */ 30 | const [openapi, setOpenApi] = useState(null); 31 | useEffect(async () => setOpenApi(await getOpenApi()), []); 32 | 33 | /** 34 | * Layout 35 | */ 36 | const [layouts, changeLayouts] = useState({ Default: {} }); 37 | useEffect(() => { 38 | getServerDefinedLayouts().then((serverLayouts) => { 39 | const newLayouts = { 40 | active: "Default", 41 | ...layouts, 42 | ...serverLayouts, 43 | ...getCustomLayout(), 44 | }; 45 | 46 | if (JSON.stringify(newLayouts) !== JSON.stringify(layouts)) { 47 | changeLayouts(newLayouts); 48 | } 49 | }); 50 | }, [layouts]); 51 | 52 | /** 53 | * Theme 54 | */ 55 | const storedOrDefaultTheme = setupStoredOrDefaultTheme(); 56 | const [theme, changeTheme] = useState(storedOrDefaultTheme); 57 | const toggleThemeAndChangeState = toggleTheme(changeTheme); 58 | useEffect(() => toggleThemeAndChangeState(storedOrDefaultTheme), []); 59 | 60 | /** 61 | * Settings 62 | */ 63 | const [settingsOpen, setLeftDrawerOpen] = useState(false); 64 | const toggleSettings = () => { 65 | setLeftDrawerOpen((prevState) => !prevState); 66 | }; 67 | 68 | /** 69 | * Return nodes 70 | */ 71 | return ( 72 |
73 |
82 | 89 | 97 |
98 |
99 | ); 100 | } 101 | 102 | window.addEventListener("load", () => { 103 | const container = document.getElementById("gateway-root"); 104 | 105 | // handle regular-table in light dom 106 | customElements.whenDefined("perspective-viewer-datagrid").then((datagrid) => { 107 | datagrid.renderTarget = "light"; 108 | }); 109 | 110 | if (container) { 111 | const root = createRoot(container); 112 | root.render(); 113 | } 114 | }); 115 | -------------------------------------------------------------------------------- /js/src/style/base.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --highlight: var(--nord0); 3 | --contrast1: var(--nord10); 4 | --contrast2: var(--nord11); 5 | 6 | --background: var(--nord4); 7 | --background2: var(--nord5); 8 | --background-light: var(--nord6); 9 | 10 | --border: var(--nord7); 11 | --line: var(--nord7); 12 | --subline: var(--nord8); 13 | 14 | --color: var(--nord0); 15 | --font-family: "Roboto"; 16 | } 17 | 18 | [data-theme="dark"] { 19 | --highlight: var(--nord4); 20 | --contrast1: var(--nord8); 21 | --contrast2: var(--nord12); 22 | 23 | --background: var(--nord0); 24 | --background2: var(--nord1); 25 | --background-light: var(--nord2); 26 | 27 | --border: var(--nord2); 28 | --line: var(--nord2); 29 | --subline: var(--nord3); 30 | 31 | --color: var(--nord4); 32 | } 33 | 34 | #main { 35 | position: absolute; 36 | top: 0; 37 | left: 0; 38 | width: 100%; 39 | height: 100%; 40 | min-width: 775px; 41 | } 42 | 43 | .container { 44 | display: flex; 45 | flex-direction: column; 46 | } 47 | 48 | .column { 49 | display: flex; 50 | flex-direction: column; 51 | } 52 | 53 | .row { 54 | display: flex; 55 | flex-direction: row; 56 | align-items: center; 57 | } 58 | 59 | .around { 60 | justify-content: space-around; 61 | } 62 | 63 | .between { 64 | justify-content: space-between; 65 | } 66 | 67 | .full-width { 68 | width: 100%; 69 | margin: auto; 70 | } 71 | 72 | .full-height { 73 | height: 100%; 74 | margin: auto; 75 | } 76 | 77 | .divider { 78 | margin-top: 10px; 79 | border-bottom: 1px solid var(--border); 80 | margin-bottom: 10px; 81 | } 82 | 83 | .icon-button { 84 | background: var(--background2); 85 | border: 1px solid var(--border); 86 | color: var(--highlight); 87 | font-family: "Roboto" !important; 88 | font-size: 11px; 89 | padding: 5px; 90 | cursor: pointer; 91 | margin-left: 5px; 92 | } 93 | 94 | .icon-button:hover, 95 | .text-button:hover { 96 | background-color: var(--highlight); 97 | color: var(--background); 98 | } 99 | 100 | .text-input, 101 | .text-button, 102 | select.layout-config { 103 | background: var(--background2); 104 | border: 1px solid var(--border); 105 | color: var(--highlight); 106 | min-width: 150px; 107 | /* min-height: 150px; */ 108 | font-family: "Roboto" !important; 109 | font-size: 11px; 110 | padding: 10px; 111 | cursor: pointer; 112 | } 113 | 114 | option.layout-config { 115 | background: var(--background); 116 | color: var(--highlight); 117 | } 118 | 119 | .text-button { 120 | margin-left: 5px; 121 | min-width: 100px; 122 | } 123 | 124 | .text-input::placeholder { 125 | color: var(--highlight); 126 | font-family: "Roboto" !important; 127 | font-size: 11px; 128 | } 129 | 130 | select.layout-config:hover { 131 | /* background: #eaeaea; */ 132 | border-color: var(--highlight); 133 | color: var(--highlight); 134 | } 135 | 136 | a.data-permalink { 137 | color: var(--highlight2); 138 | } 139 | 140 | a.data-permalink:visited { 141 | color: var(--highlight2); 142 | } 143 | 144 | #progress { 145 | background: var(--background); 146 | position: absolute; 147 | top: 0; 148 | left: 0; 149 | width: 100%; 150 | height: 100%; 151 | z-index: 100000; 152 | } 153 | 154 | .slider { 155 | position: absolute; 156 | width: 250px; 157 | height: 10px; 158 | overflow-x: hidden; 159 | position: absolute; 160 | top: 50%; 161 | left: 50%; 162 | margin-left: -125px; 163 | margin-top: -5px; 164 | z-index: 1000000; 165 | } 166 | 167 | .line { 168 | position: absolute; 169 | opacity: 0.4; 170 | background: var(--line); 171 | width: 150%; 172 | height: 5px; 173 | } 174 | 175 | .subline { 176 | position: absolute; 177 | background: var(--subline); 178 | height: 5px; 179 | } 180 | 181 | .inc { 182 | animation: increase 2s infinite; 183 | } 184 | 185 | .dec { 186 | animation: decrease 2s 0.5s infinite; 187 | } 188 | 189 | @keyframes increase { 190 | from { 191 | left: -5%; 192 | width: 5%; 193 | } 194 | 195 | to { 196 | left: 130%; 197 | width: 100%; 198 | } 199 | } 200 | 201 | @keyframes decrease { 202 | from { 203 | left: -80%; 204 | width: 80%; 205 | } 206 | 207 | to { 208 | left: 110%; 209 | width: 10%; 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /js/src/style/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Point72/csp-gateway/ddd7ea4cf5ac0a8c8610e27f097df6c758c72e6d/js/src/style/favicon.png -------------------------------------------------------------------------------- /js/src/style/header_footer.css: -------------------------------------------------------------------------------- 1 | .header, 2 | .footer { 3 | background: var(--background) !important; 4 | color: var(--color); 5 | display: flex; 6 | flex-direction: row; 7 | align-content: space-between; 8 | align-items: center; 9 | justify-content: space-between; 10 | height: 30px; 11 | font-family: var(--font-family); 12 | padding: 10px; 13 | } 14 | 15 | .header { 16 | border-bottom: 1px solid var(--border); 17 | } 18 | 19 | .header-title { 20 | margin-left: 10px; 21 | } 22 | 23 | .header-version { 24 | color: var(--contrast1); 25 | margin-left: 5px; 26 | padding-top: 5px; 27 | } 28 | 29 | .footer { 30 | border-top: 1px solid var(--border); 31 | } 32 | 33 | .footer path { 34 | stroke: var(--highlight); 35 | fill: var(--highlight); 36 | transition: 37 | stroke 0.5s, 38 | fill 0.5s; 39 | } 40 | 41 | .footer-meta { 42 | display: flex; 43 | font-size: 12px; 44 | align-items: center; 45 | } 46 | 47 | .footer-meta a:hover, 48 | .footer-meta a:hover:visited { 49 | color: var(--color); 50 | } 51 | 52 | .footer-meta a, 53 | .footer-meta a:visited { 54 | color: var(--highlight); 55 | transition: color 0.5s; 56 | } 57 | 58 | .footer-link { 59 | /* margin-left: 10px; */ 60 | margin-right: 10px; 61 | } 62 | 63 | .footer .footer-link:hover path { 64 | stroke: var(--color); 65 | fill: var(--color); 66 | } 67 | 68 | .footer-meta p { 69 | color: var(--color); 70 | margin-left: 10px; 71 | margin-right: 10px; 72 | } 73 | -------------------------------------------------------------------------------- /js/src/style/index.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css2?family=Roboto&display=swap"); 2 | @import url("https://fonts.googleapis.com/css2?family=Roboto+Mono&display=swap"); 3 | @import url("https://fonts.googleapis.com/css2?family=Material%20Icons&display=swap"); 4 | 5 | @import "./src/style/base.css"; 6 | @import "./src/style/nord.css"; 7 | @import "./src/style/header_footer.css"; 8 | @import "./src/style/perspective.css"; 9 | @import "./src/style/settings.css"; 10 | 11 | @import "perspective-viewer-pro.css"; 12 | @import "perspective-viewer-pro-dark.css"; 13 | @import "perspective-viewer-monokai.css"; 14 | @import "perspective-viewer-vaporwave.css"; 15 | @import "perspective-viewer-solarized.css"; 16 | @import "perspective-viewer-solarized-dark.css"; 17 | @import "perspective-viewer-gruvbox.css"; 18 | @import "perspective-viewer-gruvbox-dark.css"; 19 | @import "react-modern-drawer.css"; 20 | -------------------------------------------------------------------------------- /js/src/style/settings.css: -------------------------------------------------------------------------------- 1 | nav.settings { 2 | background-color: var(--background) !important; 3 | color: var(--color) !important; 4 | } 5 | 6 | div.settings-content { 7 | padding: 5px; 8 | } 9 | 10 | button.big-red-button { 11 | background: red; 12 | color: white; 13 | margin-bottom: 20px; 14 | } 15 | 16 | button.big-red-button:hover { 17 | background: #f88; 18 | color: white; 19 | } 20 | 21 | div.confirm-shutdown, 22 | div.confirm-shutdown-last { 23 | position: absolute; 24 | top: 0; 25 | bottom: 0; 26 | left: 0; 27 | right: 0; 28 | margin: auto; 29 | width: 400px; 30 | height: 200px; 31 | z-index: 102; 32 | display: flex; 33 | background-color: var(--background); 34 | color: var(--color); 35 | } 36 | 37 | div.confirm-shutdown button, 38 | div.confirm-shutdown-last button { 39 | background-color: var(--background-light); 40 | color: var(--color); 41 | } 42 | 43 | div.confirm-shutdown > h1, 44 | div.confirm-shutdown-last > h1 { 45 | text-align: center; 46 | } 47 | 48 | div.confirm-shutdown-last { 49 | background-color: red; 50 | color: white; 51 | } 52 | --------------------------------------------------------------------------------