├── .github └── donation-button.png ├── .gitignore ├── DEVELOPMENT.md ├── LICENSE ├── README.md ├── build_scripts ├── README.md ├── build_linux.sh └── build_macos.sh ├── croquis_fe ├── README ├── __init__.py ├── croquis_fe.json └── static │ ├── croquis_fe.css │ └── croquis_loader.js ├── doc ├── croquis2_1.png ├── ex1.png ├── ex2.png ├── ex3.png ├── ex4.png ├── ex5.csv ├── ex5.png ├── ex6.csv ├── ex6.png ├── reference.md ├── tutorial.md └── vscode.md ├── setup.py └── src ├── CMakeLists.txt ├── croquis ├── __init__.py ├── axis_util.py ├── buf_util.py ├── color_util.py ├── comm.py ├── data_util.py ├── datatype_util.py ├── display.py ├── env_helper.py ├── fig_data.py ├── lib │ ├── .gitignore │ └── README ├── log_util.py ├── misc_util.py ├── plot.py ├── png_util.py ├── tests │ ├── axis_util_test.py │ └── data_util_test.py └── thr_manager.py ├── csrc └── croquis │ ├── bitmap_buffer.cc │ ├── bitmap_buffer.h │ ├── buffer.cc │ ├── buffer.h │ ├── canvas.h │ ├── constants.h │ ├── figure_data.h │ ├── freeform_line_data.cc │ ├── grayscale_buffer.cc │ ├── grayscale_buffer.h │ ├── intersection_finder.cc │ ├── intersection_finder.h │ ├── line_algorithm.h │ ├── message.cc │ ├── message.h │ ├── plotter.cc │ ├── plotter.h │ ├── pybind11_shim.cc │ ├── rectangular_line_data.cc │ ├── rgb_buffer.cc │ ├── rgb_buffer.h │ ├── task.h │ ├── tests │ ├── bitmap_buffer_test.cc │ ├── grayscale_buffer_test.cc │ ├── line_algorithm_test.cc │ └── show_bitmap.py │ ├── thr_manager.cc │ ├── thr_manager.h │ └── util │ ├── avx_util.h │ ├── clock.h │ ├── color_util.h │ ├── error_helper.cc │ ├── error_helper.h │ ├── logging.cc │ ├── logging.h │ ├── macros.h │ ├── math.h │ ├── myhash.h │ ├── optional.h │ ├── stl_container_util.h │ ├── string_printf.cc │ └── string_printf.h ├── doc └── messages.txt ├── js ├── .eslintrc.js ├── .gitignore ├── README.md ├── axis_handler.js ├── canvas_mouse_handler.js ├── croquis_fe.css ├── croquis_loader.js ├── css_helper.js ├── ctxt.js ├── event_replayer.js ├── label.js ├── main.js ├── tile.js ├── tile_handler.js ├── tile_set.js ├── util.js └── webpack.config.js └── ui_tests ├── .gitignore ├── run_all_tests.py ├── test_util ├── jupyter_launcher.py └── test_helper.py └── tests ├── basic_test.py ├── dimension_check_test.py ├── resize_test.py └── save_test.py /.github/donation-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/.github/donation-button.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | BAK 2 | *.BAK 3 | /build/ 4 | /build.*/ 5 | core 6 | core.* 7 | /croquis.egg-info 8 | /dist/ 9 | dbg.log 10 | diff.txt 11 | /private 12 | __pycache__ 13 | *.o 14 | Session.vim 15 | *.so 16 | stack.dump 17 | tags 18 | .vscode 19 | x.txt 20 | xxx 21 | test*.ipynb 22 | .ipynb_checkpoints 23 | -------------------------------------------------------------------------------- /DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Building from the source 2 | 3 | The following should work on Linux and Mac OS. (Other OS's are not supported yet.) 4 | 5 | ## Prerequisites 6 | 7 | In order to build, you need [CMake](https://cmake.org/install/), 8 | [webpack](https://webpack.js.org/), [terser](https://github.com/terser/terser), 9 | and [pybind11](https://pybind11.readthedocs.io/en/stable/index.html). You can 10 | install webpack and terser by: 11 | 12 | ``` 13 | npm install -g webpack webpack-cli terser 14 | ``` 15 | 16 | (If don't like it, you don't *need* `-g`, but the commands `webpack` and 17 | `terser` must be in `$PATH` when you run the build.) 18 | 19 | You can install pybind11 by: 20 | 21 | ``` 22 | # If you're using conda: 23 | conda install pybind11 24 | 25 | # Otherwise: 26 | pip3 install pybind11 27 | ``` 28 | 29 | ## Building the wheel package 30 | 31 | ``` 32 | cd (croquis top level directory) 33 | # Any directory name will do, except for "build", because setup.py uses it. 34 | mkdir build.make 35 | cd build.make 36 | cmake -G'Unix Makefiles' -DCMAKE_BUILD_TYPE=Release ../src 37 | make -j8 VERBOSE=1 wheel 38 | ``` 39 | 40 | If you want to use a different compiler, add the compiler path to CMake argument 41 | (e.g., `-DCMAKE_CXX_COMPILER=/usr/local/bin/clang++`). 42 | 43 | If you want to use [Ninja](https://ninja-build.org/): 44 | 45 | ``` 46 | cd (croquis top level directory) 47 | mkdir build.ninja 48 | cd build.ninja 49 | cmake -GNinja -DCMAKE_BUILD_TYPE=Release ../src 50 | ninja -v wheel 51 | ``` 52 | 53 | After that, wheel file is available in the `dist` directory: 54 | 55 | ``` 56 | cd (top level directory)/dist 57 | pip3 install croquis-0.1.0-cp39-cp39-linux_x86_64.whl # Or something similar. 58 | ``` 59 | 60 | ## Testing at the source tree 61 | 62 | If you want to fiddle around, you can just build the C++ shared library by 63 | omitting `wheel` from the build commands. I.e., 64 | 65 | ``` 66 | cd (croquis top level directory) 67 | mkdir build.make 68 | cd build.make 69 | cmake -G'Unix Makefiles' -DCMAKE_BUILD_TYPE=Release ../src 70 | make -j8 VERBOSE=1 71 | ls ../src/croquis/lib 72 | # Will see something like: _csrc.cpython-39-x86_64-linux-gnu.so 73 | ``` 74 | 75 | Use `-DCMAKE_BUILD_TYPE=Debug` to build in Debug mode. 76 | 77 | Use `make check` to run tests (there aren't many): for running tests, you also 78 | need to install [pytest](https://docs.pytest.org/). 79 | 80 | In addition, there are some integration tests under `src/ui_tests`: you can run 81 | them by `run_all_tests.py`. To run it, you first need to install 82 | [playwright](https://playwright.dev/python/). 83 | 84 | Now you can simply add `src` directory to your Python import path and do: 85 | 86 | ``` 87 | import croquis 88 | ``` 89 | 90 | In this way, croquis is working in the "dev environment", which will slightly 91 | tweak operations to (hopefully) make life easier: 92 | 93 | * We reload js/css file every time you restart the kernel. 94 | * Python/C++ code will leave logging on `dbg.log` (under the current directory). 95 | 96 | In the "dev environment", croquis also runs webpack at runtime(!) to create the 97 | js bundle that is then loaded by the browser, which means that webpack must be 98 | in your `PATH`. 99 | 100 | ## Packaging 101 | 102 | Python packaging is the dark underside of an otherwise great language that 103 | nobody speaks of in broad daylight. There's no need for **you** to know 104 | anything about it, as long as you want to just use croquis (or even modify and 105 | develop it!) - honestly I'm not sure if *I* understand it correctly. 106 | 107 | However, since I had trouble figuring it out, I wrote some 108 | [helper scripts and notes](build_scripts/README.md), in the hopes that it may be 109 | useful to future myself, or any other aspiring Pythonista who's thinking of 110 | building and distributing their own wonderful Python package. 111 | 112 | ## IDE integration 113 | 114 | Here are some [personal notes](doc/vscode.md) for using VS code (work in progress). 115 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Yongjik Kim 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Croquis: plot graphs 100x faster on Jupyter Notebook 2 | 3 | Croquis is a lightweight Python library for drawing interactive graphs *really 4 | fast* on Jupyter Notebook. It lets you effortlessly browse and examine much 5 | larger data than other similar libraries. 6 | 7 | ![banner](doc/croquis2_1.png) 8 | 9 | **Install croquis by running:** 10 | 11 | ``` 12 | pip install croquis 13 | ``` 14 | 15 | 👉 Please look at the [tutorial](doc/tutorial.md) for more example. 16 | 17 | --- 18 | 19 | You might be wondering: there are already many [mature](https://matplotlib.org/) 20 | and [feature-rich](https://plotly.com/python/) plotting 21 | [libraries](https://bokeh.org/) — what's the appeal of a new, experimental 22 | library? Well, croquis is built with the following goals in mind: 23 | 24 | - **Fast:** Croquis contains a multi-threaded C++ plotting engine which 25 | automatically parallelizes graph generation, easily handling gigabytes 26 | of data. 27 | - **Simple:** Croquis provides a unified, simple API, regardless of data size. 28 | Whether your data contains ten data points or fifty million, you can make the 29 | same three function calls (or as many calls as you wish in a loop, if you 30 | prefer). The library will handle the rest. 31 | - **Interactive:** Since the C++ engine is alive as long as your figure is 32 | visible, at any moment you can zoom in, move around, or select a different 33 | subset of your data, and the figure will update accordingly. 34 | 35 | As an example, 36 | [here](https://github.com/yongjik/croquis-extra/tree/master/noaa_temperature_data)'s 37 | hourly ground temperature data of 2020 from the world's 38 | weather stations, downloaded from [NOAA website](https://www.ncdc.noaa.gov/isd/data-access). 39 | The data set contains 127 million points. 40 | 41 | https://user-images.githubusercontent.com/31876421/123535161-0b402d00-d6d7-11eb-9486-6218279eda9d.mp4 42 | 43 | The word "croquis" means [a quick, sketchy drawing](https://en.wikipedia.org/wiki/Croquis) - 44 | it's from French *croquis* which simply means "sketch." (The final -s is 45 | silent: it's French, after all.) 46 | 47 | ## Requirements 48 | 49 | - 64-bit Linux/Mac OS running on x86 with 50 | [AVX2](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#CPUs_with_AVX2) 51 | instruction set support. (Intel: Haswell (2013) or later; AMD: Excavator 52 | (2015) or later.) 53 | - Windows support is under work. 54 | - Sorry, other architectures aren't supported yet. 55 | - Python 3.6 or later. 56 | - Jupyter Notebook. 57 | - A modern browser (if it can run Jupyter Notebook, it's probably fine). 58 | 59 | ## How to install 60 | 61 | ``` 62 | pip install croquis 63 | ``` 64 | 65 | For building from the source, see [DEVELOPMENT.md](DEVELOPMENT.md). 66 | 67 | To test if it's working correctly, try this inside Jupyter Notebook: 68 | 69 | ``` 70 | # Paste into a Jupyter cell. 71 | 72 | import croquis 73 | import numpy as np 74 | 75 | N = 1000000 76 | X = np.random.normal(size=(N, 1)) 77 | Y = np.random.normal(size=(N, 1)) 78 | labels=['pt %d' % i for i in range(N)] 79 | 80 | fig = croquis.plot() 81 | fig.add(X, Y, marker_size=3, labels=labels) 82 | fig.show() 83 | ``` 84 | 85 | It should generate a plot like this: 86 | 87 | ![Gaussian distribution example](./doc/ex1.png) 88 | 89 | For documentation, see the [tutorial](doc/tutorial.md) and the (very short) 90 | [reference](doc/reference.md). 91 | 92 | By the way, this library is of course open source (MIT License) and totally free 93 | to use, but just in case you really liked it for some reason, the author could 94 | use a cup of coffee or two... :) 95 | 96 | [![buy me a coffee](.github/donation-button.png)](https://www.buymeacoffee.com/yongjikkim) 97 | 98 | ## Limitations 99 | 100 | Croquis is still experimental: as of version 0.1, we only support the **absolute 101 | bare minimum** functionality. In particular: 102 | 103 | - Only line plots are supported, nothing else: no bars, pie charts, heatmaps, etc. 104 | - All lines are solid: no dotted/dashed lines. 105 | - All markers are solid circles: no other shapes are currently supported. 106 | - No subplots: each Jupyter cell can contain only one graph. 107 | - Very few options to customize the plot. No titles, axis labels, or secondary axes. 108 | - No support for mobile browsers. 109 | - No dark mode. 110 | - As you can see, the UI is rather primitive. 111 | 112 | If croquis seems useful to you, but some features are missing for your use case, 113 | then please feel free to file an issue. (Of course I can't guarantee anything, 114 | but it will be nice to know that someone's interested.) 115 | 116 | ## FAQ 117 | 118 | ### Is it really 100 times faster? 119 | 120 | With large data, croquis can be *several hundred times* faster than other 121 | popular libraries. With very small data, there's less difference, as fixed-size 122 | overheads start to dominate. 123 | 124 | ### Can we use it outside of Jupyter Notebook? 125 | 126 | No, croquis is currently tied to Jupyter's message passing architecture, and all 127 | computation is done in the backend, so it needs an active Jupyter Python 128 | kernel. 129 | 130 | ### How does it work? 131 | 132 | Unlike most other similar libraries, croquis works by running a C++ "tile 133 | server," which computes fixed-sized "tiles" which is then sent back to the 134 | browser. If you have used Google Maps, the idea should be familiar. This has 135 | an important advantage: 136 | 137 | - The browser only has to know about tiles. Hence, the size of the data the 138 | browser needs to know is independent of the data set size. 139 | 140 | As a result, the browser stays lean and "snappy" even with massive data. 141 | (As explained in the [reference](doc/reference.md), we support `copy_data=False` 142 | option that even eliminates data copies altogether.) 143 | 144 | Moreover, unlike the browser's single-threaded javascript code, the C++-based 145 | tile server can draw multiple tiles in parallel, which allows even more speedup. 146 | 147 | (On the other hand, there are drawbacks - we have to basically re-implement every 148 | graph drawing algorithm inside this tile server, not being able to use any 149 | javascript API, except for very trivial things like coordinate grids.) 150 | -------------------------------------------------------------------------------- /build_scripts/README.md: -------------------------------------------------------------------------------- 1 | This directory contains scripts for building and testing Python packages. 2 | (You probably don't need to look at this unless you're the package maintainer. 3 | Yes, you - you know who I'm talking about.) 4 | 5 | Currently there's no automatic release testing, CI/CD, or anything fancy: 6 | everything is manual. 7 | 8 | # Linux 9 | 10 | Build the wheel by running this [build_linux.sh](build_linux.sh), for example: 11 | 12 | ``` 13 | ./build_linux.sh 3.8 0.1.0 master 14 | ``` 15 | 16 | It will download the [manylinux](https://github.com/pypa/manylinux) docker 17 | image, run a build script inside it, and create a Python wheel (`.whl`) package 18 | file. The filename looks like: `croquis-0.1.0-cp38-cp38-linux_x86_64.whl`. 19 | 20 | You can create a test environment using conda. 21 | 22 | ``` 23 | conda create -n env38 python=3.8 24 | conda activate env38 25 | conda install notebook jinja2 numpy 26 | pip install croquis-0.1.0-cp38-cp38-linux_x86_64.manylinux_2_12_x86_64.whl 27 | ``` 28 | 29 | # Mac OS 30 | 31 | It seems like Anaconda's Python versions are built against Mac OS 10.9, which is 32 | old enough for building a distributable package. For example, Numpy (1.20.x) 33 | uses the same target. 34 | 35 | ``` 36 | $ python -c 'from distutils.util import get_platform; print(get_platform())' 37 | macosx-10.9-x86_64 38 | ``` 39 | 40 | So, you can run [build_macos.sh](build_macos.sh) inside conda environment - 41 | it will create a temporary conda environment (`PKG38` below), download necessary 42 | packages, and run the build inside it. (It's not as "hermetic" as Linux since 43 | it still uses the system C++ compiler, but I think this should be good enough 44 | now.) 45 | 46 | ``` 47 | ./build_macos.sh 3.8 PKG38 0.1.0 master 48 | ``` 49 | 50 | # Uploading package 51 | 52 | Use [twine](https://twine.readthedocs.io/en/latest/) to upload packages to PyPI. 53 | For testing in the "sandbox" (test.pypi.org), see 54 | [these instructions at python.org](https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives). 55 | That is: 56 | 57 | `twine upload -r testpypi *.whl` 58 | 59 | You will probably need to run it on least two machines, once to upload Linux 60 | packages and again for Mac packages. PyPI will automatically register them into 61 | the correct place. (That is, if you upload "version 0.1.0" three times, PyPI 62 | will only show one "version 0.1.0", not three different versions.) To check 63 | whether all the packages were correctly registered, see 64 | [https://test.pypi.org/simple/croquis/](https://test.pypi.org/simple/croquis/). 65 | 66 | I don't know what happens if you upload the "same" file twice - maybe it will 67 | overwrite the previous version? 68 | 69 | Obviously, remove `-r testpypi` (and be extra careful) when you're ready to 70 | upload the real thing. 71 | -------------------------------------------------------------------------------- /build_scripts/build_linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Build wheel files for Linux, using the "manylinux" docker image. 4 | # 5 | # See: https://packaging.python.org/guides/packaging-binary-extensions/ 6 | # https://github.com/pypa/manylinux 7 | # 8 | # How to use: build_linux.sh (python version) (package version) (git branch/tag) 9 | # e.g., build_linux.sh 3.8 0.1.0 master 10 | 11 | set -o errexit 12 | set -o pipefail 13 | set -o xtrace 14 | 15 | if [[ "$INSIDE_DOCKER" != "Y" ]]; then 16 | root_dir=$(dirname "$0")/.. 17 | py_version="$1" 18 | pkg_version="$2" 19 | git_tag="$3" 20 | docker_container_name="crbuild-$pkg_version-$py_version" 21 | 22 | if [[ "$py_version" == "" || "$pkg_version" == "" || "$git_tag" == "" ]]; then 23 | set +o xtrace 24 | echo "How to use: build_linux.sh (python version) (package version) (git branch/tag)" 25 | echo "e.g., build_linux.sh 3.8 0.1.0 master" 26 | exit 1 27 | fi 28 | 29 | rm -rf .pkg_build || true 30 | mkdir .pkg_build 31 | 32 | # Create .zip file out of the whole source code. 33 | # 34 | # However, we copy a potentially "dirty" version of this script 35 | # (build_linux.sh) to run inside Docker, because otherwise it's just too 36 | # painful to debug this script itself. 37 | ( 38 | cd "$root_dir" 39 | git archive --format=zip "$git_tag" 40 | ) > .pkg_build/croquis_src.zip 41 | cp "$0" .pkg_build 42 | 43 | # We now need node.js, which doesn't work on manylinux2010, so let's use 44 | # manylinux2014. 45 | # 46 | # (Well, we could try running node.js *outside* of Docker, and just copy the 47 | # js bundle file into Docker, but that's a lot of work, and I'm not sure if 48 | # anyone actually needs it ...) 49 | docker rm "$docker_container_name" || true 50 | docker pull quay.io/pypa/manylinux2014_x86_64 51 | docker run --name "$docker_container_name" \ 52 | --mount type=bind,src="$PWD/.pkg_build",dst=/mnt/bind \ 53 | quay.io/pypa/manylinux2014_x86_64:latest \ 54 | bash -c "mkdir /build ; cd /build ; unzip /mnt/bind/croquis_src.zip ; 55 | INSIDE_DOCKER=Y /mnt/bind/build_linux.sh \ 56 | '$py_version' '$pkg_version'" 57 | 58 | else 59 | # We're inside docker now. 60 | py_version="$1" # E.g., "3.8" 61 | py_short_version=$(echo "$py_version" | sed -e 's/\.//') # E.g., 38 62 | pkg_version="$2" 63 | 64 | # Directory names looks like: /opt/python/cp36-cp36m. /opt/python/cp310-cp310, 65 | # etc. 66 | # 67 | # Hmm not sure why globbing doesn't work, but let's use `find` ... 68 | pypath=$(find /opt/python -name "cp${py_short_version}-*")/bin 69 | export PATH="$pypath":"$PATH" 70 | 71 | # Install necessary components. 72 | pip3 install pybind11 73 | 74 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash 75 | export NVM_DIR="$HOME/.nvm" 76 | set +o xtrace 77 | . $NVM_DIR/nvm.sh 78 | nvm install node 79 | set -o xtrace 80 | npm install -g webpack webpack-cli terser 81 | 82 | # Now build! 83 | mkdir -p /build/build.make 84 | cd /build/build.make 85 | cmake -G'Unix Makefiles' -DCMAKE_BUILD_TYPE=Release ../src 86 | CR_PKG_VERSION="$pkg_version" make -j8 VERBOSE=1 wheel 87 | 88 | # cp ../dist/croquis-$pkg_version-*.whl /mnt/bind/ 89 | 90 | # Add manylinux tags to the package. 91 | auditwheel repair ../dist/croquis-$pkg_version-*.whl 92 | cp wheelhouse/croquis-$pkg_version-*.whl /mnt/bind/ 93 | 94 | exit 0 95 | fi 96 | 97 | # We're back outside. 98 | docker container rm "$docker_container_name" 99 | pkg_file=$(find .pkg_build -name "croquis-$pkg_version-*.whl") 100 | cp "$pkg_file" . 101 | rm -rf .pkg_build 102 | 103 | set +o xtrace 104 | echo "Created package file: "$(basename "$pkg_file") 105 | -------------------------------------------------------------------------------- /build_scripts/build_macos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Build wheel files for Mac OS, using Anaconda. 4 | # 5 | # See: https://packaging.python.org/guides/packaging-binary-extensions/ 6 | # https://github.com/MacPython/wiki/wiki/Spinning-wheels 7 | # 8 | # The `conda` command should be in PATH. 9 | # 10 | # How to use: build_linux.sh (python version) (conda env name to create) (package version) (git branch/tag) 11 | # e.g., build_linux.sh 3.8 PKG38 0.1.0 master 12 | 13 | if [[ $(uname -s) != "Darwin" ]]; then 14 | echo "This script is for Mac OS." 15 | exit 1 16 | fi 17 | 18 | set -o errexit 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | if [[ "$INSIDE_BUILD_ENV" != "Y" ]]; then 23 | root_dir=$(dirname "$0")/.. 24 | py_version="$1" 25 | conda_env="$2" 26 | pkg_version="$3" 27 | git_tag="$4" 28 | 29 | if [[ "$py_version" == "" || "$conda_env" == "" || "$pkg_version" == "" || "$git_tag" == "" ]]; then 30 | set +o xtrace 31 | echo "How to use: build_linux.sh (python version) (conda env name to create) (package version) (git branch/tag)" 32 | echo "e.g., build_linux.sh 3.8 PKG38 0.1.0 master" 33 | exit 1 34 | fi 35 | 36 | conda run -n "$conda_env" true && { 37 | set +o xtrace 38 | echo "Conda environment $conda_env already exists." 39 | echo "Please specify a non-existent environment name!" 40 | echo "Or remove the environment by running: conda env remove -n $conda_env" 41 | exit 1 42 | } 43 | 44 | rm -rf .pkg_build || true 45 | mkdir .pkg_build 46 | 47 | # Create .zip file out of the whole source code. 48 | # 49 | # However, we copy a potentially "dirty" version of this script 50 | # (build_linux.sh) to run inside Docker, because otherwise it's just too 51 | # painful to debug this script itself. 52 | ( 53 | cd "$root_dir" 54 | git archive --format=zip "$git_tag" 55 | ) > .pkg_build/croquis_src.zip 56 | cp "$0" .pkg_build 57 | 58 | conda create --yes -n "$conda_env" python="$py_version" cmake pybind11 59 | conda run -n "$conda_env" --no-capture-output \ 60 | --cwd "$PWD/.pkg_build" \ 61 | bash -c "mkdir build ; cd build ; unzip ../croquis_src.zip ; 62 | INSIDE_BUILD_ENV=Y ../build_macos.sh \ 63 | '$py_version' '$pkg_version'" 64 | 65 | else 66 | echo $PATH 67 | echo prefix = $CONDA_PREFIX 68 | 69 | # We're inside the build environment now. 70 | py_version="$1" # E.g., "3.8" 71 | py_short_version=$(echo "$py_version" | sed -e 's/\.//') # E.g., 38 72 | pkg_version="$2" 73 | 74 | python -V 2>&1 | grep -qF "Python $py_version" || { 75 | echo "Python version mismatch: expected $py_version, got $(python -V)" 76 | exit 1 77 | } 78 | 79 | # Now build! 80 | mkdir build.make 81 | cd build.make 82 | # We need -DPython3_FIND_VIRTUALENV=ONLY because otherwise CMake tries to 83 | # find other random Python versions hanging around in the system... -_-;; 84 | cmake -G'Unix Makefiles' -DCMAKE_BUILD_TYPE=Release \ 85 | -DPython3_FIND_VIRTUALENV=ONLY \ 86 | ../src 87 | CR_PKG_VERSION="$pkg_version" make -j8 VERBOSE=1 wheel 88 | 89 | exit 0 90 | fi 91 | 92 | # We're back outside. 93 | conda env remove -n "$conda_env" 94 | pkg_file=$(find .pkg_build/build/dist -name "croquis-$pkg_version-*.whl") 95 | cp "$pkg_file" . 96 | rm -rf .pkg_build 97 | 98 | set +o xtrace 99 | echo "Created package file: "$(basename "$pkg_file") 100 | -------------------------------------------------------------------------------- /croquis_fe/README: -------------------------------------------------------------------------------- 1 | Support module for registering js/css files to Jupyter on package installation. 2 | Files are copied when building the package. 3 | 4 | (We may not need a separate module to do this, but I couldn't figure out a 5 | better way.) 6 | 7 | See setup.py for details. 8 | -------------------------------------------------------------------------------- /croquis_fe/__init__.py: -------------------------------------------------------------------------------- 1 | def _jupyter_nbextension_paths(): 2 | return [ 3 | { 4 | 'section': 'notebook', 5 | 'src': 'static', 6 | 'dest': 'croquis_fe', 7 | 'require': 'croquis_fe/static', 8 | } 9 | ] 10 | -------------------------------------------------------------------------------- /croquis_fe/croquis_fe.json: -------------------------------------------------------------------------------- 1 | { 2 | "load_extensions": { 3 | "croquis_fe/croquis_fe": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /croquis_fe/static/croquis_fe.css: -------------------------------------------------------------------------------- 1 | ../../src/js/croquis_fe.css -------------------------------------------------------------------------------- /croquis_fe/static/croquis_loader.js: -------------------------------------------------------------------------------- 1 | ../../src/js/croquis_loader.js -------------------------------------------------------------------------------- /doc/croquis2_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/croquis2_1.png -------------------------------------------------------------------------------- /doc/ex1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/ex1.png -------------------------------------------------------------------------------- /doc/ex2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/ex2.png -------------------------------------------------------------------------------- /doc/ex3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/ex3.png -------------------------------------------------------------------------------- /doc/ex4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/ex4.png -------------------------------------------------------------------------------- /doc/ex5.csv: -------------------------------------------------------------------------------- 1 | x,y1,y2 2 | 0,2,4 3 | 1,2.2,3.4 4 | 2,2.4,3.2 5 | 3,2.6,2.8 6 | 4,2.8,2.4 7 | 5,3,2 8 | -------------------------------------------------------------------------------- /doc/ex5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/ex5.png -------------------------------------------------------------------------------- /doc/ex6.csv: -------------------------------------------------------------------------------- 1 | date,location,sales 2 | 2021-05-01,Seoul,12 3 | 2021-05-02,Seoul,6 4 | 2021-05-03,Seoul,3 5 | 2021-05-04,Seoul,8 6 | 2021-05-05,Seoul,8 7 | 2021-05-06,Seoul,4 8 | 2021-05-07,Seoul,11 9 | 2021-05-08,Seoul,10 10 | 2021-05-09,Seoul,4 11 | 2021-05-10,Seoul,6 12 | 2021-05-11,Seoul,12 13 | 2021-05-12,Seoul,14 14 | 2021-05-13,Seoul,14 15 | 2021-05-14,Seoul,14 16 | 2021-05-15,Seoul,7 17 | 2021-05-16,Seoul,9 18 | 2021-05-17,Seoul,7 19 | 2021-05-18,Seoul,7 20 | 2021-05-19,Seoul,7 21 | 2021-05-20,Seoul,7 22 | 2021-05-21,Seoul,11 23 | 2021-05-22,Seoul,3 24 | 2021-05-23,Seoul,5 25 | 2021-05-24,Seoul,7 26 | 2021-05-25,Seoul,13 27 | 2021-05-26,Seoul,11 28 | 2021-05-27,Seoul,3 29 | 2021-05-28,Seoul,9 30 | 2021-05-29,Seoul,6 31 | 2021-05-30,Seoul,12 32 | 2021-05-31,Seoul,5 33 | 2021-05-10,New York,18 34 | 2021-05-11,New York,23 35 | 2021-05-12,New York,12 36 | 2021-05-13,New York,22 37 | 2021-05-14,New York,10 38 | 2021-05-15,New York,16 39 | 2021-05-16,New York,25 40 | 2021-05-17,New York,17 41 | 2021-05-18,New York,13 42 | 2021-05-19,New York,10 43 | 2021-05-20,New York,27 44 | 2021-05-21,New York,17 45 | 2021-05-22,New York,25 46 | 2021-05-23,New York,12 47 | 2021-05-24,New York,16 48 | 2021-05-25,New York,9 49 | 2021-05-26,New York,20 50 | 2021-05-27,New York,13 51 | 2021-05-28,New York,9 52 | 2021-05-29,New York,23 53 | 2021-05-30,New York,27 54 | 2021-05-31,New York,12 55 | 2021-04-25,Tokyo,15 56 | 2021-04-26,Tokyo,16 57 | 2021-04-27,Tokyo,13 58 | 2021-04-28,Tokyo,21 59 | 2021-04-29,Tokyo,23 60 | 2021-04-30,Tokyo,16 61 | 2021-05-01,Tokyo,12 62 | 2021-05-02,Tokyo,9 63 | 2021-05-03,Tokyo,9 64 | 2021-05-04,Tokyo,9 65 | 2021-05-05,Tokyo,24 66 | -------------------------------------------------------------------------------- /doc/ex6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yongjik/croquis/c9f66db19fcf018c9655d957d110bcf8d1263a99/doc/ex6.png -------------------------------------------------------------------------------- /doc/reference.md: -------------------------------------------------------------------------------- 1 | # Croquis API 2 | 3 | Since it's currently very simple, there's not much to document. Every graph is 4 | created by running the following commands: 5 | 6 | ``` 7 | # Create the figure object. 8 | fig = croquis.plot(...) 9 | 10 | # Add one or more data sets. 11 | fig.add(x1, y1, ...) 12 | fig.add(x2, y2, ...) 13 | 14 | # Generate the figure. 15 | fig.show() 16 | ``` 17 | 18 | ### `croquis.plot()` 19 | 20 | This function currently has only two optional arguments: 21 | 22 | * `x_axis='linear'` (**default**) interprets the x axis as ordinary numbers. 23 | * `x_axis='timestamp'` interprets it as [POSIX timestamps](https://en.wikipedia.org/wiki/Unix_time). 24 | Timestamps are interpreted as UTC but displayed using the local timezone. 25 | * `y_axis` behaves similarly. 26 | 27 | ### `fig.add(X, Y, colors=None, **kwargs)` 28 | 29 | Each call may add a set of lines - scatter plots can be generated by adding 30 | "lines" with a single point each, or by setting `line_width` to zero. You can 31 | call this function as many times as you want, but for best performance, it's 32 | better to batch multiple lines into a single call to `fig.add()`. 33 | 34 | * `X`, `Y`: Data points. 35 | 36 | Can be a Numpy array or something that can be transformed into a Numpy array. 37 | Data types can be any of: 8/16/32/64-bit signed/unsigned integer, `float` 38 | (i.e., `numpy.float32`), `double` (i.e., `numpy.float64`), or `np.datetime64` 39 | (which is converted to POSIX timestamps - specify `x_axis='linear'` to view 40 | them as time). 41 | 42 | If dimension is `N` or `(1, N)` (for some `N`), then it is considered `N` 43 | points of a single line. If dimension is `(M, N)` then it is considered `M` 44 | different lines, each with `N` data points. 45 | 46 | Broadcasting is supported, that is, one of `X` or `Y` can have dimension `N` 47 | or `(1, N)` and the other can have dimension `(M, N)` (where `M > 1`). 48 | 49 | For example, the following are supported combinations. 50 | 51 | ``` 52 | fig.add([1, 2, 3], [4, 5, 6]) # Adds a single line with three points. 53 | 54 | X = np.linespace(0, 10, 0.1) 55 | fig.add(X, np.sin(X)) # Adds a single line. 56 | 57 | X = np.array([[1, 2, 3, 4, 5], [2, 4, 6, 8, 10]) 58 | fig.add(X, np.sin(X)) # Adds two lines, each with five points. 59 | 60 | # Broadcasting: adds 100 lines, each with 200 points. 61 | # All lines share the same 200 x-coordinates. 62 | X = np.linspace(0, 2 * np.pi, 200) 63 | freqs = np.linspace(0, 1, 100).reshape(100, 1) 64 | Y = np.sin(freqs * X) # matrix of size 100 x 200 65 | fig.add(X, Y) 66 | ``` 67 | 68 | * `colors` (optional) 69 | 70 | If specified, must have dimension `(M, 3)`, where `M` is the number of lines. 71 | Can be either integer (range [0, 255]), or float (range [0.0, 1.0]). 72 | 73 | Currently, markers (if present) must have the same color as lines. 74 | 75 | * `marker_size`, `line_width`, `highlight_line_width` (optional) 76 | 77 | Specifies the size of the marker, line width, and line width when a line is 78 | highlighted (by hovering). Note that only circles are supported so far. 79 | 80 | * `copy_data` (optional: default is `True`) 81 | 82 | If `False`, then croquis does not copy the arguments and instead holds a 83 | reference to the data. Can reduce memory usage for huge data, but if the 84 | underlying data is modified while the graph is being used, bad things may 85 | happen. 86 | 87 | * `start_idxs` (optional) 88 | 89 | Used for adding a batch of lines with an irregular number of points. If 90 | specified, changes how we interpret `X` and `Y`: `X` and `Y` must be 1-D 91 | arrays with the same length, and `start_idxs` must be a monotonically 92 | increasing integer array indicating where each line starts. For example: 93 | 94 | ``` 95 | # Contains three lines, with 3, 5, and 4 points each. 96 | X = [0, 1, 2, 0, 1, 2, 3, 4, 0, 1, 2, 3] 97 | Y = [1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3] 98 | # | | | 99 | # line #0: 0 1 2 | | 100 | # #1: 3 4 5 6 7 | 101 | # #2: 8 9 10 11 102 | start_idxs = [0, 3, 8] 103 | 104 | fig.add(X, Y, start_idxs=start_idxs) 105 | ``` 106 | 107 | * `groupby` (optional) 108 | 109 | (Not included in 0.1.0 release.) 110 | 111 | An alternative for `start_idxs`: when given, `X`, `Y`, and `groupby` must be 112 | 1-D arrays with the same size. The elements of `groupby` can be of any type, 113 | including string. Each line is made of points that share the same value of 114 | `groupby` - lines are arranged in the order of the first occurrence of the 115 | point. 116 | 117 | In addition, if `label`/`labels` are not given, the values of `groupby` are 118 | also used as labels. 119 | 120 | For example: 121 | 122 | ``` 123 | # Contains four lines, appearing in this order: 124 | # 'ICN' (3 points) 125 | # 'SFO' (1 point) 126 | # 'GMP' (2 points) 127 | # 'RDU' (3 points) 128 | X = [ 0, 1, 2, 3, 4, 5, 6, 7, 8] 129 | Y = [ 10, 11, 20, 12, 30, 40, 41, 31, 42] 130 | airport = ['ICN', 'ICN', 'SFO', 'ICN', 'GMP', 'RDU', 'RDU', 'GMP', 'RDU'] 131 | 132 | fig.add(X, Y, groupby=airport, marker_size=10) 133 | ``` 134 | 135 | NOTE: `groupby` internally works by calling `numpy.unique()` and re-arranging 136 | the data, so it is less efficient than calling `start_idxs`. 137 | 138 | * `labels` (optional) 139 | 140 | If given, specifies the name of each line. Must be a list of strings (or 141 | something convertible to that) and have the same number of strings as lines. 142 | 143 | * `label` (optional) 144 | 145 | Shorthand for the case when there's only one line: must be a string. 146 | (Obviously, cannot be used together with `labels`.) 147 | -------------------------------------------------------------------------------- /doc/vscode.md: -------------------------------------------------------------------------------- 1 | # Using VS Code 2 | 3 | I just started learning [VS code](https://code.visualstudio.com/), so this is a 4 | compilation of random stuff I found: some of them may be useful. 5 | 6 | ## Javascript 7 | 8 | Apparently, it seems impossible to run multiple debuggers (say, JS and Python) 9 | in one VS code window at the same time: see 10 | https://github.com/microsoft/vscode/issues/4507 11 | 12 | So, I guess the best solution is to create two workspaces, one under `src/js` 13 | and another for everything else. Then you can open two VS code windows side by 14 | side. 15 | 16 | For style check, you can install ESLint extension - default `.eslintrc.js` is 17 | provided. 18 | 19 | For debugging, start a separate terminal, go to any directory containing your 20 | notebook files, and run: `jupyter-notebook --no-browser`. (`--no-browser` is 21 | not necessary but it's cleaner that way.) It will print something like: 22 | 23 | ``` 24 | $ jupyter-notebook --no-browser 25 | ...... 26 | To access the notebook, open this file in a browser: 27 | file:///...... 28 | Or copy and paste one of these URLs: 29 | http://localhost:8888/?token=5a279ac7f6b052b30a9905d665d8a36542218501d4e2c3f3 30 | or http://127.0.0.1:8888/?token=5a279ac7f6b052b30a9905d665d8a36542218501d4e2c3f3 31 | ``` 32 | 33 | Inside VS code, choose "Start Debugging", choose "Chrome", and it will start a 34 | new instance of Chrome connected to the IDE. 35 | 36 | Inside VS Code, add the following `pathMapping` to `launch.json`, and restart 37 | debugging so that the change takes effect: 38 | 39 | ``` 40 | "configurations": [ 41 | { 42 | "type": "pwa-chrome", 43 | "request": "launch", 44 | "name": "Launch Chrome against localhost", 45 | ...... 46 | "pathMapping": { 47 | "/nbextensions": "${workspaceFolder}", 48 | "/nbextensions/croquis_loader_dev.js": 49 | "${workspaceFolder}/croquis_loader.js", 50 | } 51 | } 52 | ] 53 | ``` 54 | 55 | Now open the Notebook URL inside the debug Chrome. 56 | 57 | ## Python 58 | 59 | VS code can attach to a running process. First launch Jupyter notebook, run `ps 60 | -ef | grep ipykernel` to find out the process ID, and inside VS code, choose 61 | "Start Debugging" and then "Attach using Process ID". Add a breakpoint, execute 62 | a Jupyter cell in the browser, and the breakpoint will trigger. 63 | 64 | Inside `launch.json`, add `"justMyCode": false` so that you can see the full 65 | stack trace including Jupyter internals. 66 | 67 | For some reason, I found that "Go to Symbol in Workspace" is pretty unreliable - 68 | it started working only after I disabled pylance extension. YMMV. 69 | 70 | ## C++ 71 | 72 | (TODO) 73 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Intended to be invoked from cmake - don't run directly. 2 | 3 | import os 4 | import pathlib 5 | import setuptools 6 | import setuptools.command.build_ext 7 | import setuptools.command.build_py 8 | import shutil 9 | import subprocess 10 | import sys 11 | 12 | MODE = None 13 | 14 | curpath = pathlib.Path(os.path.dirname(os.path.realpath(__file__))) 15 | os.chdir(curpath) 16 | 17 | libpaths = None 18 | js_bundle_dir = None 19 | 20 | # Create my own build_ext class, which simply copies the file to the correct 21 | # location. 22 | # Inspired by: https://github.com/pybind/cmake_example/blob/master/setup.py 23 | class MyBuildExt(setuptools.command.build_ext.build_ext): 24 | def run(self): 25 | ext, = self.extensions 26 | assert len(libpaths) == 1, \ 27 | 'Expected exactly one shared library file but found ' \ 28 | f'{len(libpaths)}: ' + str(libpaths) 29 | 30 | destpath = self.get_ext_fullpath(ext.name) 31 | destdir = pathlib.Path(destpath).parent 32 | destdir.mkdir(parents=True, exist_ok=True) 33 | shutil.copy(src=libpaths[0], dst=destpath) 34 | 35 | # Also populate 'is_nodev.py' to let the library know that this is not a 36 | # development environment. 37 | with open(destdir / 'is_nodev.py', 'wt') as f: 38 | f.write( 39 | '''# This file doesn't exist in the source tree, and is generated by setup.py so 40 | # that the croquis library can tell if it's running inside a dev environment. 41 | pass 42 | ''') 43 | 44 | # Copy the generated js bundle files. 45 | # Copied from: https://digip.org/blog/2011/01/generating-data-files-in-setup.py.html 46 | class MyBuildPy(setuptools.command.build_py.build_py): 47 | def run(self): 48 | static_dir = pathlib.Path(self.build_lib) / 'croquis_fe' / 'static' 49 | self.mkpath(str(static_dir)) 50 | for fn in 'croquis_fe.js', 'croquis_fe.js.map': 51 | shutil.copy(src=(js_bundle_dir / fn), dst=(static_dir / fn)) 52 | 53 | super().run() 54 | 55 | if __name__ == '__main__': 56 | MODE = sys.argv[1] if len(sys.argv) >= 2 else None 57 | 58 | if MODE == '-CMAKE': 59 | print('*** setup.py invoked from cmake: ', ' '.join(sys.argv)) 60 | del sys.argv[1] 61 | print('*** Cleaning build directory ...') 62 | if (curpath / 'build').exists(): 63 | shutil.rmtree(curpath / 'build') 64 | libpaths = [sys.argv.pop(1)] 65 | js_bundle_dir = pathlib.Path(sys.argv.pop(1)) 66 | 67 | elif MODE == '-f': 68 | print('*** setup.py invoked directly: ' + 69 | "I assume you know what you're doing ...") 70 | del sys.argv[1] 71 | libpaths = list(curpath.glob('src/croquis/lib/_csrc*')) 72 | js_bundle_dir = curpath / 'src' / 'js' / 'dist' 73 | 74 | else: 75 | print(''' 76 | *** This setup.py is not supposed to be invoked directly. 77 | *** Instead, please generate the package via cmake: 78 | 79 | # You can use any directory for cmake, but don't use "build" because 80 | # setuptools will also try to use it! 81 | mkdir build.cmake 82 | cd build.cmake 83 | cmake -DCMAKE_BUILD_TYPE=Release ../src 84 | make wheel # or make VERBOSE=1 wheel 85 | 86 | *** If you *really* want to run setup.py directly, first build the C library via 87 | *** cmake, put the library file under src/croquis/lib/, and invoke setup.py with 88 | *** -f: 89 | 90 | python setup.py -f bdist_wheel # or some other command you want to run. 91 | ''') 92 | sys.exit(1) 93 | 94 | with open(curpath / 'README.md', 'rt') as f: 95 | long_description = f.read() 96 | 97 | csrc = setuptools.Extension(name='croquis.lib._csrc', sources=[]) 98 | 99 | setuptools.setup( 100 | name='croquis', 101 | version=os.environ.get('CR_PKG_VERSION', '0.1.0'), 102 | author='Yongjik Kim', 103 | description='Library for plotting interactive graphs on Jupyter Notebook', 104 | long_description=long_description, 105 | long_description_content_type='text/markdown', 106 | url='https://github.com/yongjik/croquis', 107 | classifiers=[ 108 | "Framework :: Jupyter", 109 | "Programming Language :: Python :: 3", 110 | "Programming Language :: Python :: 3.6", 111 | "Programming Language :: Python :: 3.7", 112 | "Programming Language :: Python :: 3.8", 113 | "Programming Language :: Python :: 3.9", 114 | "Programming Language :: Python :: Implementation :: CPython", 115 | "License :: OSI Approved :: MIT License", 116 | "Operating System :: MacOS :: MacOS X", 117 | "Operating System :: POSIX :: Linux", 118 | # "Operating System :: OS Independent", 119 | ], 120 | python_requires='>=3.6', 121 | install_requires=[ 122 | 'jinja2', 123 | 124 | # Minimum version to support np.argsort(kind='stable'). 125 | 'numpy>=1.15.0', 126 | ], 127 | 128 | ext_modules=[csrc], 129 | # package_data={'': ['lib/_csrc*']}, 130 | cmdclass={ 131 | 'build_ext': MyBuildExt, 132 | 'build_py': MyBuildPy, 133 | }, 134 | 135 | #----------------------------------------- 136 | # Install nbextension at install time. 137 | # Basic idea is stolen from plotly/setup.py (version 4.14.3) 138 | # See also: https://jupyter-notebook.readthedocs.io/en/stable/examples/Notebook/Distributing%20Jupyter%20Extensions%20as%20Python%20Packages.html 139 | 140 | # Not completely sure, but it seems like we need a separate __init__.py 141 | # for Jupyter to run, so it's probably better to have a separate "fe" 142 | # module. (Plotly similary has `plotlywidget`.) 143 | packages=['croquis', 'croquis_fe'], 144 | 145 | package_dir={ 146 | 'croquis': 'src/croquis', 147 | 'croquis_fe': 'croquis_fe', 148 | }, 149 | package_data={ 150 | 'croquis_fe': [ 151 | # Symlinked to src/js. 152 | 'static/croquis_fe.css', 153 | 'static/croquis_loader.js', 154 | 155 | # croquis_fe.js & croquis_fe.js.map cannot be added like this, 156 | # because they're not in the correct directory. Instead, they 157 | # are added by MyBuildPy class. 158 | ], 159 | }, 160 | 161 | # TODO: We're duplicating these files? (Once under croquis_fe/, and 162 | # then under share/jupyter/...) 163 | data_files=[ 164 | ( 165 | 'share/jupyter/nbextensions/croquis_fe', [ 166 | 'src/js/croquis_fe.css', 167 | 'src/js/croquis_loader.js', 168 | str(js_bundle_dir / 'croquis_fe.js'), 169 | str(js_bundle_dir / 'croquis_fe.js.map'), 170 | ], 171 | ), 172 | ( 173 | 'etc/jupyter/nbconfig/notebook.d', 174 | ['croquis_fe/croquis_fe.json'], 175 | ), 176 | ], 177 | zip_safe=False, 178 | ) 179 | -------------------------------------------------------------------------------- /src/croquis/__init__.py: -------------------------------------------------------------------------------- 1 | # Initial test module to test client-server communication. 2 | 3 | import sys 4 | 5 | # We use f-strings, so 3.6 is the minimum version. 6 | # TODO: Check if we can actually run on 3.6! 7 | assert sys.version_info[:2] >= (3, 6), 'Croquis requires Python 3.6 and higher!' 8 | 9 | import logging 10 | import os 11 | 12 | from . import env_helper, log_util 13 | 14 | # Check if we want to enable debug logging. 15 | def maybe_enable_dbglog(): 16 | filename = None 17 | 18 | if env_helper.ENV in ['dev', 'unittest']: 19 | filename = 'dbg.log' 20 | 21 | s = os.environ.get('CROQUIS_DBGLOG') 22 | if s is not None: filename = s 23 | 24 | if filename is not None: 25 | log_util.begin_console_logging(logging.DEBUG, filename) 26 | 27 | maybe_enable_dbglog() 28 | 29 | if env_helper.ENV in ['dev', 'unittest']: 30 | # Enable Python stack dump in case we hit segfault. 31 | import faulthandler 32 | faulthandler.enable(file=open('stack.dump', 'a')) 33 | 34 | if env_helper.ENV in ['dev', 'deployed']: 35 | logger = logging.getLogger(__name__) 36 | logger.info('Loading croquis module ...') 37 | 38 | # Initialize worker threads. 39 | from .lib import _csrc 40 | from . import thr_manager 41 | 42 | # Now initialzie other parts. 43 | from . import comm 44 | if env_helper.has_ipython(): comm.comm_manager.open() 45 | 46 | # Import public API. 47 | # TODO: More functions here!! 48 | from .plot import plot 49 | -------------------------------------------------------------------------------- /src/croquis/buf_util.py: -------------------------------------------------------------------------------- 1 | # Utility functions for handling buffers. 2 | 3 | import types 4 | 5 | import numpy as np 6 | 7 | # Return a memoryview object that represents `data`. 8 | # If `copy_data` is true, make a copy first. 9 | # 10 | # If a list is given, convert to numpy array first, using `dtype`. Otherwise, 11 | # `dtype` is unused. 12 | # 13 | # TODO: I'm not sure if using memoryview is useful. Since we need numpy anyway, 14 | # maybe we should just move everything to numpy arrays? 15 | def ensure_buffer(data, dtype=None, copy_data=True): 16 | # Sanity check. 17 | if isinstance(data, types.GeneratorType): 18 | raise TypeError( 19 | f'Cannot convert a generator {type(data)} to a buffer: ' 20 | 'did you use (... for ... in ...) instead of [...]?') 21 | 22 | try: 23 | data = memoryview(data) 24 | except TypeError: 25 | # If `data` does not support Python buffer protocol (e.g., it is a 26 | # list), then we first convert it to a numpy array. 27 | # 28 | # cf. https://docs.python.org/3/c-api/buffer.html 29 | data = np.array(data, copy=copy_data, dtype=dtype) 30 | 31 | # Convert numpy datetime to unix timestamp. 32 | if np.issubdtype(data.dtype, np.datetime64): 33 | data = (data - np.datetime64(0, 's')) / np.timedelta64(1, 's') 34 | 35 | return memoryview(data) 36 | 37 | if copy_data: 38 | data = memoryview(np.copy(data)) 39 | return data 40 | -------------------------------------------------------------------------------- /src/croquis/color_util.py: -------------------------------------------------------------------------------- 1 | # Generator for the default color scheme. 2 | # 3 | # We may go "fancy" later, but for now, let's stick to the 90's feel. 4 | 5 | import numpy as np 6 | 7 | # The default color was created in a very "scientific" way by manually 8 | # generating a sequence of points in the HSV coordinate and tweaking it until it 9 | # sort of looked alright. I.e., 10 | # 11 | #------------------------------------------------- 12 | # import matplotlib.colors 13 | # import numpy as np 14 | # 15 | # N = 20 16 | # mult = 7 17 | # 18 | # HSV = np.zeros((N, 3)) 19 | # 20 | # # Give larger distance around green & purple. 21 | # HSV[:, 0] = [ 22 | # 0, 1, 2, 3, 4, 5, 23 | # 7, 9, 11, 13, 15, 24 | # 16, 17, 18, 19, 20, 25 | # 22, 24, 26, 28, 26 | # ] 27 | # HSV[:, 0] /= 30 28 | # 29 | # HSV[:, 1] = 1 30 | # HSV[:, 2] = 1 31 | # 32 | # HSV[[1, 4, 6, 9, 12, 15, 18], 1] = 0.8 33 | # HSV[[1, 4, 6, 9, 12, 15, 18], 2] = 0.8 34 | # HSV[[2, 5, 7, 10, 13, 16, 19], 2] = 0.5 35 | # HSV[8, 2] = 0.3 36 | # 37 | # RGB = matplotlib.colors.hsv_to_rgb(HSV) 38 | # 39 | # for idx in range(N): 40 | # color = (RGB[(idx * mult) % N] * 255).astype(np.uint8) 41 | # print(' 0x%02x, 0x%02x, 0x%02x, # %d' % (tuple(color) + (idx,))) 42 | #------------------------------------------------- 43 | 44 | BLK_SIZE = 20 45 | DEFAULT_COLORS = np.array([ 46 | 0xff, 0x00, 0x00, # 0 47 | 0x19, 0x7f, 0x00, # 1 48 | 0x00, 0x33, 0xff, # 2 49 | 0xcc, 0x49, 0x28, # 3 50 | 0x00, 0x4c, 0x0f, # 4 51 | 0x28, 0x28, 0xcc, # 5 52 | 0x7f, 0x33, 0x00, # 6 53 | 0x28, 0xcc, 0x8a, # 7 54 | 0x32, 0x00, 0x7f, # 8 55 | 0xff, 0x99, 0x00, # 9 56 | 0x00, 0x7f, 0x7f, # 10 57 | 0xcc, 0x00, 0xff, # 11 58 | 0xcc, 0xab, 0x28, # 12 59 | 0x00, 0xcb, 0xff, # 13 60 | 0xcc, 0x28, 0xab, # 14 61 | 0x7f, 0x7f, 0x00, # 15 62 | 0x28, 0x8a, 0xcc, # 16 63 | 0x7f, 0x00, 0x33, # 17 64 | 0x8a, 0xcc, 0x28, # 18 65 | 0x00, 0x33, 0x7f, # 19 66 | ], dtype=np.uint8).reshape(BLK_SIZE, 3) 67 | 68 | def default_colors(start_item_id, item_cnt): 69 | if item_cnt == 0: return np.zeros((0, 3), dtype=np.uint8) 70 | 71 | start_block = start_item_id // BLK_SIZE 72 | end_block = (start_item_id + item_cnt + BLK_SIZE - 1) // BLK_SIZE 73 | offset = start_item_id - start_block * BLK_SIZE 74 | 75 | C = np.tile(DEFAULT_COLORS, (end_block - start_block, 1)) 76 | return C[offset:(offset + item_cnt), :] 77 | -------------------------------------------------------------------------------- /src/croquis/comm.py: -------------------------------------------------------------------------------- 1 | # Communications support: see 2 | # https://jupyter-notebook.readthedocs.io/en/stable/comms.html 3 | 4 | import collections 5 | import logging 6 | import uuid 7 | import weakref 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | # Create a unique UUID so that FE can find out when BE restarts. 12 | BE_uuid = str(uuid.uuid4()) 13 | 14 | class CommManager(object): 15 | def __init__(self): 16 | self.is_open = False 17 | self.comm = None 18 | 19 | self.plots = {} 20 | 21 | # handlers[canvas_id][msgtype] = weakref.WeakMethod(callback) 22 | self.handlers = collections.defaultdict(dict) 23 | 24 | # Called only if we're inside IPython: otherwise obviously there's no 25 | # communications channel! 26 | def open(self): 27 | logger.info('Opening communication channel ...') 28 | assert not self.is_open, 'Already opened!' 29 | get_ipython().kernel.comm_manager.register_target( 30 | 'croquis', self._open_handler) 31 | self.is_open = True 32 | 33 | # Register a Plotter object so that it's not garbage-collected when the user 34 | # loses the reference to it - which is quite common because people will just 35 | # reuse the variable `fig` in different cells. 36 | def register_plot(self, canvas_id, plot): 37 | self.plots[canvas_id] = plot 38 | 39 | # Register a handler for message received from FE. The callback is called 40 | # with `canvas_id`, `msgtype`, and json contents of the message. 41 | # 42 | # I'm not sure if I have to worry, but to make sure that objects are 43 | # properly garbage collected, we assume `callback` is a "bound method" and 44 | # create a weak reference to it, so that we don't accidentally hold a 45 | # reference to the underlying object. 46 | # 47 | # For sending messages out, see thr_manager.register_cpp_callback(). 48 | def register_handler(self, canvas_id, msgtype, callback): 49 | ref = weakref.WeakMethod(callback) 50 | if msgtype in self.handlers[canvas_id]: 51 | logger.warn('Handler already registered for %s (%s) ...', 52 | canvas_id, msgtype) 53 | self.handlers[canvas_id][msgtype] = ref 54 | 55 | # if `msgtype` is None, deregister the whole cell. 56 | def deregister_handler(self, canvas_id, msgtype=None): 57 | if msgtype is None: 58 | del self.handlers[canvas_id] 59 | del self.plots[canvas_id] 60 | else: 61 | del self.handlers[canvas_id][msgtype] 62 | 63 | # Send a message. `attachments` is an optional list of memoryview objects 64 | # (or some objects that supports the buffer protocol). 65 | def send(self, canvas_id, msgtype, attachments=None, **kwargs): 66 | kwargs.update(canvas_id=canvas_id, msg=msgtype) 67 | if attachments is not None: 68 | if attachments == []: 69 | attachments = None 70 | elif type(attachments) != list: 71 | attachments = [attachments] 72 | logger.debug('CommManager.send() sending message: %s', kwargs) 73 | self.comm.send(kwargs, buffers=attachments) 74 | 75 | # Handler for the initial message. 76 | def _open_handler(self, comm, open_msg): 77 | logger.debug('Initial open packet received from client: ', open_msg) 78 | self.comm = comm 79 | 80 | comm.on_msg(self._msg_handler) 81 | 82 | logger.debug('Sending BE_ready packet ...') 83 | logger.debug(type(comm)) 84 | comm.send({'msg': 'BE_ready'}) 85 | logger.debug('Sent BE_ready packet ...') 86 | 87 | # Handler for all subsequent messages. 88 | # See also ThrManager._callback_entry_point(). 89 | def _msg_handler(self, msg): 90 | logger.debug('Data received from client: %s', msg) 91 | try: 92 | data = msg['content']['data'] 93 | canvas_id = data['canvas_id'] 94 | msgtype = data['msg'] 95 | except KeyError: 96 | logger.error('Malformed message: %s', msg) 97 | return 98 | 99 | if canvas_id not in self.handlers: 100 | logger.warning('Unrecognized canvas_id %s', canvas_id) 101 | return 102 | 103 | if msgtype not in self.handlers[canvas_id]: 104 | logger.warning('Missing handler for canvas_id=%s, msgtype=%s', 105 | canvas_id, msgtype) 106 | return 107 | 108 | callback = self.handlers[canvas_id][msgtype] 109 | cb = callback() 110 | if cb is None: 111 | logger.warning('Handler gone for canvas_id=%s, msgtype=%s', 112 | canvas_id, msgtype) 113 | del self.handlers[canvas_id][msgtype] 114 | return 115 | 116 | cb(canvas_id, msgtype, msg) 117 | 118 | comm_manager = CommManager() 119 | -------------------------------------------------------------------------------- /src/croquis/data_util.py: -------------------------------------------------------------------------------- 1 | # Miscellaneous utility functions for shuffling data. 2 | 3 | import numpy as np 4 | 5 | # Given a numpy array `keys` where the same key may be repeated multiple times, 6 | # construct the following: 7 | # 8 | # - `unique_keys`: contains each key just once, in the order they appear first. 9 | # - `idxs`: a map for shuffling data in `keys` so that key values are grouped 10 | # together. 11 | # - `start_idxs`: contains the start index of each unique key in keys[idxs]. 12 | # 13 | # For example if 14 | # keys == ['foo', 'foo', 'bar', 'foo', 'qux', 'baz', 'baz', 'qux', 'baz'], 15 | # 0 1 2 3 4 5 6 7 8 16 | # (total 9 rows with 4 unique keys) 17 | # 18 | # then the intermediate variables below are: 19 | # unique == ['bar', 'baz', 'qoo', 'qux'] 20 | # idxs0 == [2 5 0 4] 21 | # inverse == [2 2 0 2 3 1 1 3 1] 22 | # counts == [1 3 3 2] 23 | # 24 | # order = [2 0 3 1] 25 | # 26 | # and the return values are: 27 | # unique_keys = ['foo', 'bar', 'qux', 'baz'] # in the order they appear 28 | # idxs = [0 1 3 2 4 7 5 6 8] # 0 1 3 == foo 29 | # # 2 == bar 30 | # # 4 7 == qux 31 | # # 5 6 8 == baz 32 | # start_idxs = [0, 3, 4, 6, (9)] # Only returns the first 4 numbers. 33 | def compute_groupby(keys): 34 | unique, idxs0, inverse, counts = np.unique( 35 | keys, return_index=True, return_inverse=True, return_counts=True) 36 | 37 | order = np.argsort(idxs0, kind='stable') 38 | 39 | try: 40 | # This will succeed if `keys` is a numpy array. 41 | unique_keys = keys[idxs0[order]] 42 | except TypeError: 43 | unique_keys = [keys[idxs0[i]] for i in order] 44 | 45 | idxs = np.argsort(idxs0[inverse], kind='stable') 46 | 47 | start_idxs = np.zeros(dtype=np.int64, shape=(len(unique_keys) + 1)) 48 | np.cumsum(counts[order], out=start_idxs[1:]) 49 | 50 | return unique_keys, idxs, start_idxs[:-1] 51 | -------------------------------------------------------------------------------- /src/croquis/datatype_util.py: -------------------------------------------------------------------------------- 1 | # Utility functions for handling different input data types. 2 | 3 | import collections 4 | import types 5 | 6 | import numpy as np 7 | 8 | # Helper function to check if the given object is a pandas DataFrame. 9 | def is_dataframe(obj): 10 | try: 11 | import pandas as pd 12 | return isinstance(obj, pd.DataFrame) 13 | except ImportError: 14 | return False 15 | 16 | # Return a numpy array object that represents `data`. 17 | # If `copy_data` is true, always make a copy; otherwise, copy only if necessary. 18 | # 19 | # If a list is given, convert to a numpy array first, using `dtype`. Otherwise, 20 | # `dtype` is unused. 21 | def convert_to_numpy(data, dtype=None, copy_data=True): 22 | # Sanity check. 23 | if isinstance(data, types.GeneratorType): 24 | raise TypeError( 25 | f'Cannot convert a generator {type(data)} to a buffer: ' 26 | 'did you use (... for ... in ...) instead of [...]?') 27 | 28 | data = np.array(data, copy=copy_data, dtype=dtype) 29 | if data.dtype.hasobject: 30 | raise TypeError('Object-type numpy arrays are not supported.') 31 | 32 | return data 33 | 34 | # Helper class to check that various numpy arrays we receive are in the correct 35 | # dimension. 36 | class DimensionChecker(): 37 | def __init__(self): 38 | self.dim_info = [] 39 | self.dims = collections.defaultdict(dict) 40 | 41 | # Record the dimensions of parameter `data` (with name `name`), where each 42 | # dimension is named using `dim_names`. 43 | # 44 | # If multi-dimensional lists are possible (e.g., X = [[1,2],[3,4]]), the 45 | # caller must first call ensure_buffer() or convert_to_numpy() so that we 46 | # can infer the correct dimension. 47 | # 48 | # Due to broadcast logic, not all dimensions need to be present. E.g., if 49 | # dim_names == ('lines', 'pts'), then `data` can have shape (200,), in which 50 | # case we only record `pts == 200` (`lines` will be absent). 51 | def add(self, name, data, dim_names : tuple, min_dims=1): 52 | assert type(dim_names) == tuple 53 | try: 54 | shape = data.shape 55 | except AttributeError: 56 | shape = (len(data),) 57 | 58 | self.dim_info.append((name, shape)) 59 | 60 | assert min_dims <= len(shape) <= len(dim_names) 61 | dim_names = dim_names[-len(shape):] 62 | for dim_name, sz in zip(dim_names, shape): 63 | self.dims[dim_name][name] = sz 64 | 65 | # Verify that the recorded dimensions are consistent. 66 | def verify(self, dim_name, default=None, must=None): 67 | def _raise(msg): 68 | raise ValueError( 69 | msg + ': please check if arguments have correct shapes:' + 70 | str(self.dim_info)) 71 | 72 | if dim_name not in self.dims: 73 | if default is None: _raise(f'Missing dimension ({dim_name})') 74 | return default 75 | 76 | s = set(self.dims[dim_name].values()) 77 | if len(s) != 1: _raise(f'Inconsistent dimension ({dim_name})') 78 | dim, = s 79 | if must is not None and dim != must: 80 | _raise(f'Dimension ({dim_name}) must be {must}') 81 | return dim 82 | -------------------------------------------------------------------------------- /src/croquis/display.py: -------------------------------------------------------------------------------- 1 | # Helper class for interfacing with IPython display API. 2 | 3 | import uuid 4 | 5 | import IPython.display 6 | import jinja2 7 | 8 | from . import comm, env_helper 9 | 10 | comm_reopened = False 11 | 12 | # Encapsulates a canvas output, with a unique "canvas ID" (which is our own - 13 | # shouldn't be confused with anything IPython does). 14 | class DisplayObj(object): 15 | # `debug=True` enables these buttons for FE debugging: 16 | # 17 | # * Record: Start recording events and logs into the internal buffer. 18 | # * Stop: Stop recording. 19 | # * Save: Save the recorded events and logs to a file. 20 | # * Load: Load the saved events from a previous run. 21 | # * Replay: Start replaying the events generated by either "Record" or 22 | # "Load". 23 | # * Clear: Clear the internal buffer. 24 | # 25 | # Load/Replay buttons were added to debug the highlight algorithm (i.e., 26 | # "Why does it not highlight line #3 when the cursor is on top of it?"), but 27 | # the code is very haphazard - there's no guarantee that they may work. 28 | # 29 | # See `EventReplayer` in croquis_fe.js for details. 30 | DEFAULT_CONFIG = { 31 | 'debug': False, 32 | 'reload_fe': False, # Force reload frontend js, for debugging. 33 | 34 | # Where to find our js modules: see env_helper.py and setup.py. 35 | '_js_loader_module': 36 | 'croquis_loader_dev' if env_helper.is_dev() 37 | else 'croquis_fe/croquis_loader', 38 | '_js_main_module': 39 | 'croquis_fe_dev' if env_helper.is_dev() 40 | else 'croquis_fe/croquis_fe', 41 | } 42 | 43 | def __init__(self, **kwargs): 44 | # Create a random "unique" ID. 45 | # 46 | # We add a prefix, because document.querySelector() in js doesn't like 47 | # ID's starting with a digit. (Well, weirdly, jquery seems to work just 48 | # fine with these IDs, so YMMV...) 49 | self.canvas_id = 'v-' + str(uuid.uuid4()) 50 | 51 | self.config = {'canvas_id': self.canvas_id} 52 | for key, default_val in self.DEFAULT_CONFIG.items(): 53 | self.config[key] = kwargs.pop(key, default_val) 54 | assert not kwargs, 'Unknown options: ' + str(kwargs) 55 | 56 | def register_handler(self, msgtype, callback): 57 | comm.comm_manager.register_handler(self.canvas_id, msgtype, callback) 58 | 59 | def show(self): 60 | # In dev environment, always force reload frontend after kernel restart. 61 | global comm_reopened 62 | if not comm_reopened: 63 | comm_reopened = True 64 | if env_helper.is_dev(): 65 | self.config['reload_fe'] = True 66 | 67 | self.config['BE_uuid'] = comm.BE_uuid 68 | 69 | # NOTE: Please keep this in sync with comments on Ctxt (croquis_fe.js). 70 | html = jinja2.Template(''' 71 |
72 | {% if debug %} 73 |
74 | 75 | 76 | 77 | ... 78 |
79 | {% endif %} 80 |
81 |
82 | 83 | 84 | 85 | 86 | 87 |   Drag mouse to: 88 | 90 | 91 | 93 | 94 |
95 |
96 | 97 |
98 | {% if debug %} 99 | 100 | {% endif %} 101 |
102 | {% if debug %} 103 |
Debug logging
104 | {% endif %} 105 |
106 | ''') 131 | 132 | IPython.display.display( 133 | IPython.display.HTML(html.render(self.config))) 134 | -------------------------------------------------------------------------------- /src/croquis/env_helper.py: -------------------------------------------------------------------------------- 1 | # Helper for handling different environments. 2 | # 3 | # We're in the "dev environment", if we're running directly on the source tree. 4 | # We distinguish it by trying to find lib/is_nodev.py, which is dynamically 5 | # generated by setup.py (hence is missing in the dev environment). Inside the 6 | # dev environment, we enable extra logging to help debugging. 7 | 8 | import os 9 | import subprocess 10 | import sys 11 | 12 | if os.environ.get('CROQUIS_UNITTEST'): 13 | ENV = 'unittest' 14 | else: 15 | try: 16 | from .lib import is_nodev 17 | ENV = 'deployed' 18 | except ImportError: 19 | ENV = 'dev' 20 | 21 | # Check that we're running inside ipython. 22 | # (For now, running outside ipython is only for internal testing.) 23 | if ENV == 'unittest': 24 | HAS_IPYTHON = False 25 | else: 26 | try: 27 | get_ipython() 28 | HAS_IPYTHON = True 29 | except NameError: 30 | HAS_IPYTHON = False 31 | print( 32 | '''*** IPython not detected: croquis requires IPython to run. 33 | *** Most functionality will not work outside of IPython. 34 | ''', 35 | file=sys.stderr) 36 | 37 | if ENV == 'dev': 38 | # Tell Jupyter to import our nbextension. 39 | # (In non-dev environment, this should have been handled when the package 40 | # was built: see CMakeLists.txt and setup.py.) 41 | # 42 | # We also build the js "bundle" by calling webpack - takes <1s, so it seems 43 | # OK for now. (Otherwise we would have to add the "js build" step 44 | # somewhere...) 45 | 46 | import notebook.nbextensions 47 | 48 | curdir = os.path.dirname(os.path.realpath(__file__)) 49 | jsdir = os.path.join(curdir, '../js') 50 | 51 | try: 52 | subprocess.check_call(['webpack', '--mode=development'], cwd=jsdir) 53 | except: 54 | print('''Failed to generate the frontend javascript file: 55 | please check if webpack is available. You can install webpack by: 56 | npm install -g --save-dev webpack webpack-cli 57 | ''', file=sys.stderr) 58 | raise 59 | 60 | def _install(src, dest): 61 | notebook.nbextensions.install_nbextension( 62 | os.path.join(jsdir, src), destination=dest, 63 | user=True, symlink=True, overwrite=True) 64 | 65 | _install('croquis_fe.css', 'croquis_fe_dev.css') 66 | _install('croquis_loader.js', 'croquis_loader_dev.js') 67 | _install('dist/croquis_fe_dev.js', 'croquis_fe_dev.js') 68 | _install('dist/croquis_fe_dev.js.map', 'croquis_fe_dev.js.map') 69 | 70 | def is_dev(): return ENV == 'dev' 71 | def has_ipython(): return HAS_IPYTHON 72 | -------------------------------------------------------------------------------- /src/croquis/lib/.gitignore: -------------------------------------------------------------------------------- 1 | _csrc* 2 | -------------------------------------------------------------------------------- /src/croquis/lib/README: -------------------------------------------------------------------------------- 1 | This directory will contain the C++ shared library file, so that we can import 2 | it from python during development. 3 | -------------------------------------------------------------------------------- /src/croquis/log_util.py: -------------------------------------------------------------------------------- 1 | # Helper for logging - mostly for debugging. 2 | 3 | import datetime 4 | import logging 5 | import math 6 | import os 7 | import sys 8 | import threading 9 | import time 10 | 11 | parent = __name__[:__name__.rfind('.')] 12 | parent_logger = logging.getLogger(parent) 13 | 14 | start_time = time.time() 15 | log_fd = -1 # Used by C++ code so that we can share the same log file. 16 | 17 | # Formats the log message with indicating how much time has passed since the 18 | # last log in this thread. Useful for debugging. 19 | class ThreadLocalTimestampFormatter(logging.Formatter): 20 | def __init__(self): 21 | super().__init__() 22 | self.thread_local = threading.local() 23 | 24 | def formatMessage(self, record): 25 | T = record.created 26 | try: 27 | elapsed = T - self.thread_local.last_T 28 | except AttributeError: 29 | elapsed = 0 30 | self.thread_local.last_T = T 31 | 32 | # Relative time since the beginning (but only the last ten seconds). 33 | relative = (T - start_time) 34 | usec = ((relative * 0.01) - math.floor(relative * 0.01)) * 100 35 | usec = '%9.6f' % usec 36 | 37 | if -1.0 < elapsed < 1.0: 38 | elapsed_str = '%+10d' % int(elapsed * 1e6) 39 | else: 40 | elapsed_str = '%+10.6f' % elapsed 41 | 42 | timestamp = datetime.datetime.fromtimestamp(T).strftime('%H:%M:%S.%f') 43 | return '%s%s %-15s %s %s %s:%d %s' % \ 44 | (record.levelname[0], timestamp, 45 | record.threadName, usec, elapsed_str, 46 | record.filename, record.lineno, record.message) 47 | 48 | class LoggingHandler(logging.Handler): 49 | def __init__(self, filename): 50 | super().__init__() 51 | self.f = None 52 | if filename is not None: 53 | self.f = open(filename, 'at', buffering=1) 54 | self.f.write(f'\n===== Dbglog started (PID {os.getpid()}) =====\n') 55 | 56 | global log_fd 57 | log_fd = self.f.fileno() 58 | 59 | def emit(self, record): 60 | try: 61 | msg = self.format(record) + '\n' 62 | if self.f is not None: 63 | self.f.write(msg) 64 | if record.levelno >= logging.INFO: 65 | # The iPython kernel intercepts stderr (fd 2) and sends every 66 | # output as ZMQ message for the notebook, but we don't want to 67 | # spam the notebook. For some reason, `os.stderr.fileno()` 68 | # points to the original stderr, so we can use that. 69 | # 70 | # For details, see the `OutStream` class inside `ipykernel`. 71 | os.write(sys.stderr.fileno(), bytes(msg, 'utf-8')) 72 | except Exception: 73 | self.handleError(record) 74 | 75 | # Enable logging to stderr. 76 | # Called by __init__.py in case of dev environment. 77 | def begin_console_logging(level=logging.INFO, filename='dbg.log'): 78 | parent_logger.propagate = False 79 | parent_logger.setLevel(level) 80 | 81 | handler = LoggingHandler(filename) 82 | handler.setLevel(level) 83 | handler.setFormatter(ThreadLocalTimestampFormatter()) 84 | parent_logger.addHandler(handler) 85 | -------------------------------------------------------------------------------- /src/croquis/misc_util.py: -------------------------------------------------------------------------------- 1 | # Miscellaneous functions. 2 | 3 | # Check that `kwargs` is empty. Used to dynamically parse named arguments. 4 | def check_empty(kwargs): 5 | if len(kwargs) > 0: 6 | raise TypeError(f'Unexpected keyword arguments: {list(kwargs)}') 7 | -------------------------------------------------------------------------------- /src/croquis/png_util.py: -------------------------------------------------------------------------------- 1 | # Utility function for creating PNG image data. 2 | # 3 | # Since we only generate exactly one kind of PNG, we take a number of shortcuts 4 | # to make it easier for us. (Alternatively, we could import something like 5 | # Pillow, but it seems like overkill.) 6 | # 7 | # In particular, we always assume that the given data is 256x256x3, and the PNG 8 | # "filtering" algorithm has been already applied: see RgbBuffer::make_png_data() 9 | # for more explanation. 10 | 11 | import struct 12 | import zlib 13 | 14 | # PNG image header, 256x256x3. 15 | PNG_HEADER_RGB = ( 16 | b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a' # PNG file header 17 | b'\x00\x00\x00\x0d' # IHDR chunk size 18 | b'\x49\x48\x44\x52' # "IHDR" 19 | b'\x00\x00\x01\x00\x00\x00\x01\x00' # width 256, height 256 20 | b'\x08\x02\x00\x00\x00' # 8-bit RGB, no interlacing 21 | b'\xd3\x10\x3f\x31' # CRC 22 | ) 23 | 24 | # PNG image header, 256x256x4. 25 | PNG_HEADER_RGBA = ( 26 | b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a' # PNG file header 27 | b'\x00\x00\x00\x0d' # IHDR chunk size 28 | b'\x49\x48\x44\x52' # "IHDR" 29 | b'\x00\x00\x01\x00\x00\x00\x01\x00' # width 256, height 256 30 | b'\x08\x06\x00\x00\x00' # 8-bit RGBA, no interlacing 31 | b'\x5c\x72\xa8\x66' # CRC 32 | ) 33 | 34 | PNG_FOOTER= b'\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82' 35 | 36 | def generate_png(png_data, is_transparent): 37 | if is_transparent: 38 | header = PNG_HEADER_RGBA 39 | assert len(png_data) == 256 * (256 * 4 + 1) 40 | else: 41 | header = PNG_HEADER_RGB 42 | assert len(png_data) == 256 * (256 * 3 + 1) 43 | compressed = zlib.compress(png_data) 44 | crc = zlib.crc32(compressed, zlib.crc32(b'IDAT')) 45 | 46 | buf = bytearray(len(header) + len(compressed) + 12 + len(PNG_FOOTER)) 47 | pos = 0 48 | 49 | def _append(data): 50 | nonlocal pos 51 | buf[pos:pos+len(data)] = data 52 | pos += len(data) 53 | 54 | _append(header) 55 | _append(struct.pack('>I', len(compressed))) 56 | _append(b'IDAT') 57 | _append(compressed) 58 | _append(struct.pack('>I', crc)) 59 | _append(PNG_FOOTER) 60 | assert pos == len(buf) 61 | 62 | return buf 63 | -------------------------------------------------------------------------------- /src/croquis/tests/axis_util_test.py: -------------------------------------------------------------------------------- 1 | # Use pytest to run. 2 | 3 | import datetime 4 | import re 5 | import os 6 | import sys 7 | import time 8 | 9 | curdir = os.path.dirname(os.path.realpath(__file__)) 10 | sys.path.insert(0, f'{curdir}/../..') 11 | 12 | os.environ['CROQUIS_UNITTEST'] = 'Y' 13 | from croquis import axis_util 14 | 15 | # TODO: test offset as well !! 16 | 17 | def linear_helper(axis, x0, x1, width, expected): 18 | builder = axis_util._create_tick_builder('linear', axis, x0, x1, width, 0, 0) 19 | ticks = builder.run() 20 | 21 | print(ticks) 22 | assert len(ticks) >= 2 23 | 24 | for px, label in ticks: 25 | assert re.fullmatch(expected, label) 26 | coord = float(label) 27 | if axis == 'x': 28 | px1 = (coord - x0) / (x1 - x0) * (width - 1) 29 | else: 30 | px1 = (x1 - coord) / (x1 - x0) * (width - 1) 31 | assert abs(px - px1) <= 0.5 + 1e-6 32 | 33 | def test_linear(): 34 | linear_helper('x', 0.0223, 0.02244, 700, r'0\.022\d\d') 35 | linear_helper('x', 2.23e-5, 2.24e-5, 800, r'2\.2\d\de-05') 36 | linear_helper('x', -12345678, 12345679, 600, r'(0|-?\d+000000)') 37 | linear_helper('y', -500, 800, 300, r'(0|-?\d00)') 38 | linear_helper('y', 12345678, 12345679, 2000, r'1\.234567\d+e\+07') 39 | 40 | # TODO: This function's behavior depends on the current timezone! 41 | def timestamp_helper(axis, d0, d1, width, expected): 42 | _unix_ts = lambda d: time.mktime(time.strptime(d, '%Y%m%d-%H%M%S')) 43 | builder = axis_util._create_tick_builder( 44 | 'timestamp', axis, _unix_ts(d0), _unix_ts(d1), width, 0, 0) 45 | ticks = builder.run() 46 | 47 | print(ticks) 48 | assert len(ticks) >= 2 49 | 50 | for px, label in ticks: 51 | assert re.fullmatch(expected, label) 52 | # TODO: verify that the time string actually matches the coordinate. 53 | 54 | def test_timestamp(): 55 | timestamp_helper('x', '20200101-000000', '20210101-000000', 800, r'202[01]-\d\d-01') 56 | timestamp_helper('x', '19000101-000000', '20210101-000000', 800, r'\d\d\d0') 57 | timestamp_helper('x', '20210401-000000', '20210601-000000', 800, r'2021-0[456]-\d\d') 58 | timestamp_helper('x', '20201231-210000', '20210101-120000', 800, 59 | r'(202[01]-\d\d-\d\d )?\d\d:00') 60 | timestamp_helper('x', '20210131-235000', '20210201-000500', 800, 61 | r'(2021-\d\d-\d\d )?\d\d:\d\d') 62 | timestamp_helper('x', '20210131-235955', '20210201-000005', 800, 63 | r'(2021-\d\d-\d\d )?\d\d:\d\d:\d\d') 64 | timestamp_helper('x', '20191213-084904', '20210118-231056', 1000, 65 | r'202[01]-\d\d-01') 66 | -------------------------------------------------------------------------------- /src/croquis/tests/data_util_test.py: -------------------------------------------------------------------------------- 1 | # Use pytest to run. 2 | 3 | import os 4 | import sys 5 | 6 | curdir = os.path.dirname(os.path.realpath(__file__)) 7 | sys.path.insert(0, f'{curdir}/../..') 8 | 9 | os.environ['CROQUIS_UNITTEST'] = 'Y' 10 | from croquis import data_util 11 | 12 | import numpy as np 13 | 14 | gen = np.random.default_rng(12345) 15 | 16 | def run_test(nkeys, nrows): 17 | key_idxs = gen.integers(0, nkeys, size=nrows) 18 | keys = ['key%04d' % k for k in key_idxs] 19 | 20 | unique_keys, idxs, start_idxs = data_util.compute_groupby(keys) 21 | 22 | assert len(unique_keys) == len(set(unique_keys)) 23 | assert set(unique_keys) == set(keys) 24 | assert set(idxs) == set(range(nrows)) 25 | assert len(start_idxs) == len(unique_keys) 26 | 27 | assert start_idxs[0] == 0 28 | for key_idx in range(len(start_idxs) - 1): 29 | assert start_idxs[key_idx] < start_idxs[key_idx + 1] 30 | 31 | # Verify that keys are added in the order of their first occurrence. 32 | # (I.e., idxs[start_idxs[key_idx]] is the first occurrence of the key 33 | # unique_keys[key_idx].) 34 | assert idxs[start_idxs[key_idx]] < idxs[start_idxs[key_idx + 1]] 35 | 36 | for key_idx, start_idx in enumerate(start_idxs): 37 | next_idx = start_idxs[key_idx + 1] if key_idx < len(start_idxs) - 1 \ 38 | else nrows 39 | 40 | for idx in range(start_idx + 1, next_idx): 41 | assert idxs[idx] > idxs[idx - 1] 42 | 43 | for idx in range(start_idx, next_idx): 44 | assert keys[idxs[idx]] == unique_keys[key_idx] 45 | 46 | def test_simple(): 47 | run_test(1, 1) 48 | run_test(1, 10000) 49 | run_test(10000, 10000) 50 | 51 | def test_loop(): 52 | for nkeys in range(1, 100): 53 | run_test(nkeys, 1000) 54 | -------------------------------------------------------------------------------- /src/croquis/thr_manager.py: -------------------------------------------------------------------------------- 1 | # Wrapper around C++ thread manager. 2 | 3 | import logging 4 | import os 5 | import threading 6 | import traceback 7 | import weakref 8 | 9 | from . import log_util 10 | from .lib import _csrc 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | # Sanity check - we probably don't need more threads. 15 | MAX_THREAD = 50 16 | 17 | # A simple wrapper around threads so that we can log any unhandled exception and 18 | # then *do* shut down the process. 19 | def create_thread(target, name, daemon=False, args=(), kwargs={}): 20 | assert callable(target) 21 | 22 | def _thr_main(): 23 | try: 24 | logger.debug('Thread %s started.', name) 25 | target(*args, **kwargs) 26 | # These threads normally never finish. 27 | logger.warning('Thread %s exiting ...', name) 28 | except: 29 | logger.exception('Unhandled exception, terminating the process!') 30 | # import pdb; pdb.post_mortem() 31 | os._exit(1) 32 | 33 | thr = threading.Thread(target=_thr_main, name=name, daemon=daemon) 34 | return thr 35 | 36 | class ThrManager(object): 37 | def __init__(self, nthreads=None): 38 | self.callbacks = {} 39 | 40 | nthreads = nthreads or min(MAX_THREAD, os.cpu_count()) 41 | logger.info('Creating %d worker threads ...', nthreads) 42 | self._C = _csrc.ThrManager( 43 | nthreads, ThrManager._callback_entry_point, 44 | log_util.start_time, log_util.log_fd) 45 | 46 | # Instead of creating threads inside the C++ ThrManager, we create 47 | # threads here and hand them over to C++, for better error reporting. 48 | def _thr(idx): 49 | thr = create_thread(self._C.wthr_entry_point, f'Croquis#{idx}', 50 | daemon=True, args=(idx,)) 51 | thr.start() 52 | return thr 53 | 54 | self.threads = [_thr(i) for i in range(nthreads)] 55 | 56 | # Register callbacks so that C++ code can easily call Python code. 57 | # 58 | # I'm paranoid, so to avoid circular references, we'll use weak references. 59 | # (See CommManager.register_handler for discussion.) 60 | def _register_cpp_callback(self, C_obj, callback): 61 | obj_id = C_obj.get_address() 62 | ref = weakref.WeakMethod(callback) 63 | self.callbacks[obj_id] = ref 64 | 65 | # Callback function called by C++ code. 66 | # (We use "staticmethod" to avoid holding reference to `self`.) 67 | # 68 | # See also CommManager._msg_handler(). 69 | @staticmethod 70 | def _callback_entry_point(obj_id, str_data, data1, data2): 71 | if data1 is not None: data1 = memoryview(data1) 72 | if data2 is not None: data2 = memoryview(data2) 73 | 74 | global thr_manager 75 | self = thr_manager 76 | callback = self.callbacks.get(obj_id) 77 | if callback is None: 78 | logger.error('Cannot find callback for obj_id=%x', obj_id) 79 | return False 80 | 81 | cb = callback() 82 | if cb is None: 83 | logger.error('Handler gone for obj_id=%x', obj_id) 84 | del self.callbacks[obj_id] 85 | return False 86 | 87 | # `str_data` contains key-value pairs in the format "x=y", e.g., 88 | # {"msg=test_message", "foo=hello", "#bar=3"}. 89 | assert type(str_data) == list 90 | d = {} 91 | for kv in str_data: 92 | key, val = kv.split('=', 1) 93 | if key.startswith('#'): key, val = key[1:], int(val) 94 | d[key] = val 95 | cb(d, data1, data2) 96 | return True 97 | 98 | # Start the thread manager. 99 | # Currently there's no support for "shutting down" ThrManager: it will keep 100 | # running as long as the process is alive. 101 | thr_manager = ThrManager() 102 | register_cpp_callback = thr_manager._register_cpp_callback 103 | -------------------------------------------------------------------------------- /src/csrc/croquis/bitmap_buffer.h: -------------------------------------------------------------------------------- 1 | // Bitmap buffer for drawing lines with AVX intrinsics. 2 | 3 | #pragma once 4 | 5 | #include // uint64_t 6 | #include // memset 7 | 8 | #include // __m256i 9 | 10 | namespace croquis { 11 | 12 | class BitmapBuffer { 13 | public: 14 | // Each BitmapBuffer contains 512x512 pixels, and each block is 8x8. 15 | enum { BLK_CNT = 4096 }; // = (512 * 512) / (8 * 8) 16 | 17 | // Comprised of 64bit blocks, where each block is an 8x8 area. 18 | alignas(32) uint64_t buf[BLK_CNT]; 19 | 20 | // List of blocks that are changed so far: we have 4 extra entries, because 21 | // store_blks() may write up to 4 entries past the end. 22 | uint16_t blklist[BLK_CNT + 4]; 23 | int blk_cnt = 0; // Number of blocks stored in `blklist`. 24 | 25 | BitmapBuffer() { memset(buf, 0x00, BLK_CNT * sizeof(uint64_t)); } 26 | 27 | void reset() { 28 | for (int i = 0; i < blk_cnt; i++) buf[blklist[i]] = 0; 29 | blk_cnt = 0; 30 | } 31 | 32 | private: 33 | // Helper function. 34 | inline void store_blks(uint16_t offset, __m256i blks); 35 | 36 | public: 37 | void draw_line(float x0, float y0, float x1, float y1, float width); 38 | 39 | // Helper function to get a pixel for testing. 40 | inline bool get_pixel(int x, int y) const { 41 | int idx1 = (y / 8) * 64 + (x / 8); 42 | int idx2 = (y % 8) * 8 + (x % 8); 43 | uint64_t blk = buf[idx1]; 44 | return (bool) ((blk >> idx2) & 0x01); 45 | } 46 | }; 47 | 48 | } // namespace croquis 49 | -------------------------------------------------------------------------------- /src/csrc/croquis/constants.h: -------------------------------------------------------------------------------- 1 | // Constants used in the source. 2 | 3 | #pragma once 4 | 5 | namespace croquis { 6 | 7 | static const int TILE_SIZE = 256; 8 | 9 | // How much zoom we do per each step: must match croquis_fe.js. 10 | static const float ZOOM_FACTOR = 1.5; 11 | 12 | } // namespace croquis 13 | -------------------------------------------------------------------------------- /src/csrc/croquis/grayscale_buffer.h: -------------------------------------------------------------------------------- 1 | // A buffer in 256-color grayscle for drawing lines fast (hopefully). 2 | 3 | #pragma once 4 | 5 | #include // uint64_t 6 | #include // memset 7 | 8 | #include // __m128i 9 | 10 | namespace croquis { 11 | 12 | class GrayscaleBuffer { 13 | public: 14 | // Each BitmapBuffer contains 256x256 pixels, and each block is 4x4. 15 | enum { BLK_CNT = 4096 }; // = (256 * 256) / (4 * 4) 16 | 17 | // Comprised of 16-byte blocks, where each block is a 4x4 area. 18 | alignas(16) __m128i buf[BLK_CNT]; 19 | 20 | // List of blocks that are changed so far: we have 2 extra entries, because 21 | // store_blks() may write up to 2 entries past the end. 22 | uint16_t blklist[BLK_CNT + 2]; 23 | int blk_cnt = 0; // Number of blocks stored in `blklist`. 24 | 25 | GrayscaleBuffer() { memset(buf, 0x00, BLK_CNT * sizeof(__m128i)); } 26 | 27 | void reset() { 28 | for (int i = 0; i < blk_cnt; i++) 29 | _mm_store_si128(&buf[blklist[i]], _mm_setzero_si128()); 30 | blk_cnt = 0; 31 | } 32 | 33 | private: 34 | // Helper function. 35 | inline void store_blk(int offset, __m128i blks); 36 | 37 | public: 38 | void draw_line(float x0, float y0, float x1, float y1, float width); 39 | void draw_circle(float x0, float y0, float radius); 40 | 41 | // Helper function to get a pixel for testing. 42 | inline uint8_t get_pixel(int x, int y) const { 43 | int idx1 = (y / 4) * 64 + (x / 4); 44 | int idx2 = (y % 4) * 4 + (x % 4); 45 | 46 | // I think this *should* be `char` to work around the "strict aliasing" 47 | // rule. 48 | return ((const char *) buf)[idx1 * 16 + idx2]; 49 | } 50 | }; 51 | 52 | } // namespace croquis 53 | -------------------------------------------------------------------------------- /src/csrc/croquis/intersection_finder.cc: -------------------------------------------------------------------------------- 1 | // A task that finds intersections between tiles and line segments. 2 | 3 | #include "croquis/intersection_finder.h" 4 | 5 | #include // INT_MAX 6 | 7 | #include // min 8 | #include // make_unique 9 | 10 | #include "croquis/canvas.h" // CanvasConfig 11 | #include "croquis/util/macros.h" // CHECK 12 | #include "croquis/util/stl_container_util.h" // util::push_back 13 | 14 | namespace croquis { 15 | 16 | template 17 | IntersectionResult::IntersectionResult(int tile_cnt, 18 | DType start_id, DType end_id) 19 | : tile_cnt(tile_cnt), start_id(start_id), end_id(end_id) 20 | { 21 | // Reserve some extra: `strip_cnt_` is the number of strips in the initial 22 | // chunk. 23 | int extra = std::max(5, int(tile_cnt * 0.2)); 24 | strip_cnt_ = tile_cnt + extra; 25 | 26 | chunks_.push_back(std::make_unique(strip_cnt_ * STRIP_SZ)); 27 | strips_ = std::make_unique(tile_cnt); 28 | idxs_ = std::make_unique(tile_cnt); 29 | 30 | // Initialize pointers. 31 | DType *ptr = chunks_[0].get(); 32 | for (int i = 0; i < tile_cnt; i++) { 33 | strips_[i] = ptr; 34 | idxs_[i] = 0; 35 | *ptr = -1; // Sentinel value. 36 | ptr += STRIP_SZ; 37 | } 38 | 39 | // Put the extra buffer into `freelist_.`. 40 | freelist_ = nullptr; 41 | for (int i = 0; i < extra; i++) { 42 | *(void **) ptr = freelist_; 43 | freelist_ = (void *) ptr; 44 | ptr += STRIP_SZ; 45 | } 46 | } 47 | 48 | template 49 | DType *IntersectionResult::allocate_chunk() 50 | { 51 | CHECK(freelist_ == nullptr); 52 | 53 | // printf("allocate_chunk called !!!\n"); 54 | 55 | int chunksize = std::min(std::max(20, strip_cnt_ / 2), 1024); 56 | DType *chunk = 57 | util::push_back( 58 | chunks_, 59 | std::make_unique(chunksize * STRIP_SZ)).get(); 60 | 61 | DType *ptr = chunk + STRIP_SZ; 62 | for (int i = 1; i < chunksize; i++) { 63 | *(void **) ptr = freelist_; 64 | freelist_ = (void *) ptr; 65 | ptr += STRIP_SZ; 66 | } 67 | 68 | return chunk; 69 | } 70 | 71 | template 72 | IntersectionResultSet::IntersectionResultSet( 73 | const std::vector &prio_coords, 74 | const std::vector ®_coords, 75 | DType start, DType end, DType batch_size) 76 | { 77 | // printf("**********************************\n"); 78 | // printf("IntersectionResultSet created %p\n", this); 79 | 80 | CHECK(prio_coords.size() + reg_coords.size() > 0); 81 | CHECK(prio_coords.size() % 2 == 0); 82 | CHECK(reg_coords.size() % 2 == 0); 83 | 84 | int row_min = INT_MAX, row_max = INT_MIN; 85 | int col_min = INT_MAX, col_max = INT_MIN; 86 | 87 | for (size_t i = 0; i < prio_coords.size(); i += 2) { 88 | row_min = std::min(row_min, prio_coords[i]); 89 | row_max = std::max(row_max, prio_coords[i]); 90 | col_min = std::min(col_min, prio_coords[i + 1]); 91 | col_max = std::max(col_max, prio_coords[i + 1]); 92 | } 93 | 94 | for (size_t i = 0; i < reg_coords.size(); i += 2) { 95 | row_min = std::min(row_min, reg_coords[i]); 96 | row_max = std::max(row_max, reg_coords[i]); 97 | col_min = std::min(col_min, reg_coords[i + 1]); 98 | col_max = std::max(col_max, reg_coords[i + 1]); 99 | } 100 | 101 | // Now fill in the coordinate specification. 102 | // (We assume that `prio_coords` and `reg_coords` don't intersect.) 103 | tile_cnt_ = (prio_coords.size() + reg_coords.size()) / 2; 104 | row_start_ = row_min; 105 | nrows_ = row_max - row_min + 1; 106 | col_start_ = col_min; 107 | ncols_ = col_max - col_min + 1; 108 | 109 | // Compute the mapping from tile coordinate (row, col) to buffer ID for each 110 | // tile we're using. 111 | int area_size = nrows_ * ncols_; 112 | tile_map_ = std::make_unique(area_size); 113 | is_prio_ = std::make_unique(area_size); 114 | for (int i = 0; i < area_size; i++) { 115 | tile_map_[i] = -1; 116 | is_prio_[i] = false; 117 | } 118 | 119 | for (size_t i = 0; i < prio_coords.size(); i += 2) { 120 | int idx = (prio_coords[i] - row_start_) * ncols_ + 121 | (prio_coords[i + 1] - col_start_); 122 | CHECK(idx >= 0 && idx < area_size); // Sanity check. 123 | CHECK(tile_map_[idx] == -1); // Sanity check. 124 | tile_map_[idx] = 0; 125 | is_prio_[idx] = true; 126 | } 127 | 128 | for (size_t i = 0; i < reg_coords.size(); i += 2) { 129 | int idx = (reg_coords[i] - row_start_) * ncols_ + 130 | (reg_coords[i + 1] - col_start_); 131 | CHECK(idx >= 0 && idx < area_size); // Sanity check. 132 | CHECK(tile_map_[idx] == -1); // Sanity check. 133 | tile_map_[idx] = 0; 134 | } 135 | 136 | { 137 | int c = 0; 138 | for (int i = 0; i < area_size; i++) { 139 | if (tile_map_[i] == 0) tile_map_[i] = c++; 140 | } 141 | CHECK(c == tile_cnt_); 142 | } 143 | 144 | // Now create the necessary number of IntersectionResult instances. 145 | CHECK(start <= end); // Sanity check. 146 | while (start < end) { 147 | DType this_size = std::min(end - start, batch_size); 148 | util::emplace_back_unique(results, tile_cnt_, start, start + this_size); 149 | start += this_size; 150 | } 151 | } 152 | 153 | // Instantiate necessary templates. 154 | template class IntersectionResult; 155 | template class IntersectionResultSet; 156 | 157 | } // namespace croquis 158 | -------------------------------------------------------------------------------- /src/csrc/croquis/line_algorithm.h: -------------------------------------------------------------------------------- 1 | // Template helper functions for drawing lines. 2 | 3 | #pragma once 4 | 5 | #include // isnan 6 | #include // printf (for debugging) 7 | 8 | #include // max 9 | 10 | #include "croquis/util/logging.h" // DBG_LOG1 11 | 12 | namespace croquis { 13 | 14 | // Draw a straigh line from (x0, y0) to (x1, y1), and "visit" all pixels on the 15 | // line by calling the visitor function. 16 | // 17 | // In order to "reuse" the same algorithm as GrayscaleBuffer, we assume each 18 | // pixel is centered at integer coordinates: for example, the pixel centered at 19 | // the origin (0, 0) contains points [-0.5, 0.5] x [0.5, 0.5]. 20 | // 21 | // We will not be too concerned about what happens when the line segment passes 22 | // through the corner or stops exactly at the edge. 23 | // 24 | // TODO: Expand templates and support `double` as well? 25 | template 26 | class StraightLineVisitor { 27 | private: 28 | const int xmin_, ymin_, xmax_, ymax_; 29 | const F fn_; 30 | 31 | public: 32 | StraightLineVisitor(int xmin, int ymin, int xmax, int ymax, F fn) 33 | : xmin_(xmin), ymin_(ymin), xmax_(xmax), ymax_(ymax), 34 | fn_(fn) { } 35 | 36 | inline void visit(float x0, float y0, float x1, float y1, float width); 37 | }; 38 | 39 | // Most of the logic is copied from GrayscaleBuffer::draw_line(). 40 | // See there for detailed comments. 41 | template 42 | inline void StraightLineVisitor::visit( 43 | float x0, float y0, float x1, float y1, float width) 44 | { 45 | #if 0 46 | printf("visit(x0 y0 x1 y1 width = %.3f %.3f %.3f %.3f %.3f\n", 47 | x0, y0, x1, y1, width); 48 | #endif 49 | 50 | if (isnan(x0) || isnan(y0) || isnan(x1) || isnan(y1)) return; 51 | 52 | const float dx = x1 - x0; 53 | const float dy = y1 - y0; 54 | 55 | // Shift by (xmin_, ymin_) so that the boundary starts at the origin. 56 | // If we "flip" the coordinate, shift by (xmax_, ymax_) so that, again, the 57 | // boundary starts at the origin. 58 | float coords0[8] = { 59 | x0 - xmin_, x1 - xmin_, y0 - ymin_, y1 - ymin_, 60 | xmax_ - x0, xmax_ - x1, ymax_ - y0, ymax_ - y1, 61 | }; 62 | int coord_type = 63 | 4 * (fabsf(dy) > fabsf(dx)) + // bit 2: steep slope 64 | 2 * (y0 > y1) + // bit 1: y0 > y1 65 | 1 * (x0 > x1); // bit 0: x0 > x1 66 | 67 | const int FLIP = 4; 68 | static const int coord_shuffle_map[] = { 69 | 0, 1, 2, 3, 70 | 1, 0, FLIP+3, FLIP+2, 71 | 0, 1, FLIP+2, FLIP+3, 72 | 1, 0, 3, 2, 73 | 74 | 2, 3, 0, 1, 75 | 2, 3, FLIP+0, FLIP+1, 76 | 3, 2, FLIP+1, FLIP+0, 77 | 3, 2, 1, 0, 78 | }; 79 | 80 | float coords[4]; 81 | for (int i = 0; i < 4; i++) 82 | coords[i] = coords0[coord_shuffle_map[coord_type * 4 + i]]; 83 | 84 | const float u0 = coords[0]; 85 | const float u1 = coords[1]; 86 | const float v0 = coords[2]; 87 | const float v1 = coords[3]; 88 | const float du = u1 - u0; 89 | const float dv = v1 - v0; 90 | 91 | // 0: no transformation (u = x - xmin_, v = y - ymin_) 92 | // 1: flip y (u = x - xmin_, v = ymax_ - y) 93 | // 2: transpose (u = y - ymin_, v = x - xmin_) 94 | // 3: flip x, and then transpose (u = y - ymin_, v = xmax_ - x) 95 | int shuffle_type = (coord_type >> 1) ^ (coord_type & 0x01); 96 | 97 | // Draw area limit. 98 | const int area_width = 99 | (shuffle_type >= 2) ? ymax_ - ymin_ + 1 : xmax_ - xmin_ + 1; 100 | const int area_height = 101 | (shuffle_type >= 2) ? xmax_ - xmin_ + 1 : ymax_ - ymin_ + 1; 102 | 103 | const float len = sqrt(dx * dx + dy * dy); 104 | if (len == 0.0) return; // No line to draw. 105 | const float invlen = 1.0f / len; 106 | const float wu = dv * (invlen * width / 2); 107 | const float wv = du * (invlen * width / 2); 108 | 109 | const float umin = u0 - wu; 110 | const float vmin = v0 - wv; 111 | 112 | const float slope = dv / du; 113 | 114 | // vL0: where the lower line intersects the left side of the leftmost pixel 115 | // (u = -0.5). 116 | // vH0: where the higher line intersects the *right* side of the leftmost 117 | // pixel (u = +0.5). 118 | float vL0 = (v0 - wv) + slope * (-0.5 - (u0 + wu)); 119 | float vH0 = (v0 + wv) + slope * (+0.5 - (u0 - wu)); 120 | 121 | // Find the first column to visit. 122 | int u; 123 | if (umin > -0.5f && vmin > -0.5f) { 124 | // If (umin, vmin) is inside the drawing area, we can start from there. 125 | u = nearbyintf(umin); 126 | } 127 | else { 128 | // The line starts outside the drawing area. 129 | if (vH0 > -0.5f) { 130 | // If vH0 >= -0.5, then the higher line passes at or above the pixel 131 | // (0, 0). Hence, we can start drawing at u=0. 132 | u = 0; 133 | } 134 | else { 135 | // If the higher line passes below (0, 0), then find the 136 | // u-coordinate when v equals -0.5 (i.e., when it enters a pixel in 137 | // the bottom row). To guard against overflow, let's first check if 138 | // the pixel is to the right of the drawing area, in which case 139 | // there's nothing to draw. 140 | if (slope * (area_width + 1 - (u0 - wu)) < -0.5f - (v0 + wv)) return; 141 | const float uH = (u0 - wu) + (-0.5f - (v0 + wv)) / (slope + 1e-8); 142 | u = nearbyintf(uH); 143 | } 144 | } 145 | 146 | const int umax_int = std::min((int) nearbyintf(u1 + wu), area_width - 1); 147 | const int vmin_int = std::max((int) nearbyintf(v0 - wv), 0); 148 | const int vmax_int = std::min((int) nearbyintf(v1 + wv), area_height - 1); 149 | 150 | // if (umax_int - u > 10) 151 | // DBG_LOG1(1, "u = %d umax_int = %d\n", u, umax_int); 152 | 153 | for (; u <= umax_int; u++) { 154 | int vL = std::max((int) nearbyintf(vL0 + slope * u), vmin_int); 155 | int vH = std::min((int) nearbyintf(vH0 + slope * u), vmax_int); 156 | if (vL > vH) return; // Only happens if (vL >= area_height). 157 | 158 | // if (vL <= vH - 2) DBG_LOG1(1, " u = %d vL = %d vH = %d\n", u, vL, vH); 159 | 160 | for (int v = vL; v <= vH; v++) { 161 | // TODO: Change to branch-less code? 162 | switch (shuffle_type) { 163 | case 0: fn_(u + xmin_, v + ymin_); break; 164 | case 1: fn_(u + xmin_, ymax_ - v); break; 165 | case 2: fn_(v + xmin_, u + ymin_); break; 166 | case 3: fn_(xmax_ - v, u + ymin_); break; 167 | } 168 | } 169 | } 170 | } 171 | 172 | // Helper function. 173 | template 174 | StraightLineVisitor create_straight_line_visitor( 175 | int xmin, int ymin, int xmax, int ymax, F fn) { 176 | return StraightLineVisitor(xmin, ymin, xmax, ymax, fn); 177 | } 178 | 179 | } // namespace croquis 180 | -------------------------------------------------------------------------------- /src/csrc/croquis/message.cc: -------------------------------------------------------------------------------- 1 | // Messages between frontend and backend. 2 | 3 | #include "croquis/message.h" 4 | 5 | #include 6 | 7 | #include "croquis/thr_manager.h" 8 | 9 | namespace croquis { 10 | 11 | #if 0 12 | // Called when Python code is done with this data - we call the "unpin task". 13 | MessageData::~MessageData() 14 | { 15 | if (unpin_task_ != nullptr) ThrManager::enqueue(std::move(unpin_task_)); 16 | 17 | // printf("MessageData freed: %p\n", this); 18 | } 19 | #endif 20 | 21 | } // namespace croquis 22 | -------------------------------------------------------------------------------- /src/csrc/croquis/message.h: -------------------------------------------------------------------------------- 1 | // Messages between frontend and backend. 2 | 3 | #pragma once 4 | 5 | #include 6 | 7 | #include // unique_ptr 8 | #include 9 | 10 | namespace croquis { 11 | 12 | #if 0 13 | // A simple buffer that uses shared_ptr implementation. 14 | struct SharedBuf { 15 | // Unfortunately, shared_ptr of an array requires C++17, so let's use a 16 | // wrapper. 17 | std::unique_ptr ptr; 18 | size_t sz; 19 | }; 20 | #endif 21 | 22 | // Manages binary data buffer used to send data back to frontend. 23 | class MessageData { 24 | public: 25 | const std::string name; // For debugging. 26 | 27 | protected: 28 | const size_t size_; 29 | 30 | MessageData(const std::string &name, size_t sz) : name(name), size_(sz) { } 31 | 32 | public: 33 | virtual ~MessageData() { } 34 | 35 | virtual void *get() = 0; 36 | const void *get() const { return const_cast(this)->get(); } 37 | size_t size() const { return size_; } 38 | }; 39 | 40 | // Simplest implementation using unique_ptr. 41 | class UniqueMessageData final : public MessageData { 42 | private: 43 | std::unique_ptr ptr; 44 | 45 | public: 46 | UniqueMessageData(const std::string &name, size_t sz) 47 | : MessageData(name, sz), ptr(std::make_unique(sz)) { } 48 | 49 | virtual void *get() override { return (void *) ptr.get(); } 50 | }; 51 | 52 | } // namespace croquis 53 | -------------------------------------------------------------------------------- /src/csrc/croquis/pybind11_shim.cc: -------------------------------------------------------------------------------- 1 | // Testing pybind11 !! 2 | // https://pybind11.readthedocs.io/en/master/basics.html 3 | 4 | #include "croquis/buffer.h" 5 | #include "croquis/canvas.h" 6 | #include "croquis/figure_data.h" 7 | #include "croquis/message.h" 8 | #include "croquis/plotter.h" 9 | #include "croquis/thr_manager.h" 10 | #include "croquis/util/string_printf.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | namespace py = pybind11; 17 | 18 | PYBIND11_MODULE(_csrc, m) { 19 | using croquis::util::string_printf; 20 | 21 | m.doc() = "Internal module for croquis"; 22 | 23 | py::class_(m, "ThrManager") 24 | .def(py::init(), 25 | py::return_value_policy::reference) 26 | .def("wthr_entry_point", &croquis::ThrManager::wthr_entry_point, 27 | py::call_guard()); 28 | 29 | // TODO: Do we need this? 30 | #if 0 31 | // Helper function to shut down thread manager. 32 | m.def("shutdown_tmgr", &croquis::ThrManager::shutdown, 33 | "Helper function to shut down existing ThrManager."); 34 | #endif 35 | 36 | py::class_(m, "MessageData", py::buffer_protocol()) 37 | .def_property_readonly( 38 | "name", [](const croquis::MessageData &data) { 39 | return data.name; 40 | } 41 | ) 42 | .def_buffer([](croquis::MessageData &data) { 43 | // Better to use `unsigned char` here, because otherwise the buffer 44 | // will be `signed char` and it won't be compatible with Python 45 | // `bytes` type. 46 | // 47 | // Can check in Python by: 48 | // m = memoryview(data) 49 | // print(m.format) # 'b' for (char *) 50 | // # 'B' for (unsigned char *) 51 | // m[:3] = b'foo' # Needs 'B'. 52 | return py::buffer_info((unsigned char *) data.get(), data.size()); 53 | }) 54 | .def("__repr__", [](const croquis::MessageData &data) { 55 | return string_printf( 56 | "", 57 | data.name.c_str(), data.get(), data.size()); 58 | }); 59 | 60 | py::class_(m, "CanvasConfig") 61 | .def(py::init()) 62 | .def(py::init()) 64 | .def_readonly("id", &croquis::CanvasConfig::id) 65 | .def_readonly("w", &croquis::CanvasConfig::w) 66 | .def_readonly("h", &croquis::CanvasConfig::h) 67 | .def_readonly("x0", &croquis::CanvasConfig::x0) 68 | .def_readonly("y0", &croquis::CanvasConfig::y0) 69 | .def_readonly("x1", &croquis::CanvasConfig::x1) 70 | .def_readonly("y1", &croquis::CanvasConfig::y1) 71 | .def_readonly("zoom_level", &croquis::CanvasConfig::zoom_level) 72 | .def_readonly("x_offset", &croquis::CanvasConfig::x_offset) 73 | .def_readonly("y_offset", &croquis::CanvasConfig::y_offset); 74 | 75 | py::class_(m, "Plotter") 76 | .def(py::init<>()) 77 | .def("add_rectangular_line_data", 78 | [](croquis::Plotter &p, 79 | py::buffer X, py::buffer Y, py::buffer colors, 80 | int line_cnt, int pts_cnt, 81 | float marker_size, float line_width, 82 | float highlight_line_width) { 83 | p.add_figure_data( 84 | X.request(), Y.request(), colors.request(), line_cnt, pts_cnt, 85 | marker_size, line_width, highlight_line_width); 86 | }, 87 | py::call_guard() 88 | ) 89 | .def("add_freeform_line_data", 90 | [](croquis::Plotter &p, 91 | py::buffer X, py::buffer Y, py::buffer start_idxs, 92 | py::buffer colors, 93 | int item_cnt, int total_pts_cnt, 94 | float marker_size, float line_width, 95 | float highlight_line_width) { 96 | p.add_figure_data( 97 | X.request(), Y.request(), start_idxs.request(), 98 | colors.request(), 99 | item_cnt, total_pts_cnt, 100 | marker_size, line_width, highlight_line_width); 101 | }, 102 | py::call_guard() 103 | ) 104 | .def("get_address", 105 | [](const croquis::Plotter &p) { return (uintptr_t) &p; }) 106 | .def_property_readonly( 107 | "sm_version", [](const croquis::Plotter &p) { 108 | return p.get_sm_version(); 109 | } 110 | ) 111 | .def("create_canvas_config", &croquis::Plotter::create_canvas_config, 112 | py::call_guard()) 113 | .def("init_selection_map", [](croquis::Plotter &p) { 114 | auto result = p.init_selection_map(); 115 | return py::memoryview::from_buffer( 116 | result.first, { result.second }, { 1 /* stride */ }); 117 | }) 118 | .def("start_selection_update", 119 | &croquis::Plotter::start_selection_update) 120 | .def("end_selection_update", &croquis::Plotter::end_selection_update) 121 | .def("acknowledge_seqs", &croquis::Plotter::acknowledge_seqs, 122 | py::call_guard()) 123 | .def("tile_req_handler", &croquis::Plotter::tile_req_handler, 124 | py::call_guard()) 125 | .def("check_error", &croquis::Plotter::check_error); 126 | } 127 | -------------------------------------------------------------------------------- /src/csrc/croquis/rgb_buffer.h: -------------------------------------------------------------------------------- 1 | // A buffer for RGB image tile. 2 | 3 | #pragma once 4 | 5 | #include // uint32_t 6 | #include // memset 7 | 8 | #include // __m128i 9 | 10 | #include // unique_ptr 11 | #include 12 | 13 | #include "croquis/constants.h" // TILE_SIZE 14 | #include "croquis/message.h" // UniqueMessageData 15 | #include "croquis/util/macros.h" // DIE_MSG 16 | 17 | namespace croquis { 18 | 19 | class GrayscaleBuffer; 20 | 21 | // An abstract base class for RgbBuffer or RgbaBuffer. 22 | class ColoredBufferBase { 23 | public: 24 | // Each BitmapBuffer contains 256x256 pixels, and each block is 4x4. 25 | enum { BLK_CNT = 4096 }; // = (256 * 256) / (4 * 4) 26 | static_assert(BLK_CNT == (TILE_SIZE * TILE_SIZE) / 16, "Sanity check"); 27 | 28 | virtual ~ColoredBufferBase() { } 29 | 30 | // Merge the data from GrayscaleBuffer with the given color, and clears 31 | // GrayscaleBuffer. 32 | // `line_id` is used in RgbBuffer to update `hovermap` on affected pixels. 33 | virtual void merge(GrayscaleBuffer *buf, int line_id, 34 | uint32_t color /* 0xaarrggbb */) = 0; 35 | 36 | // Create a buffer of pixels organized according to PNG spec: it can be 37 | // compressed (via zlib) to generate a PNG IDAT chunk. 38 | // 39 | // (For simplicity, we're going to use python's `zlib` module to handle 40 | // compression for us, for now.) 41 | virtual std::unique_ptr 42 | make_png_data(const std::string &name) const = 0; 43 | 44 | // Only available for RgbBuffer. 45 | virtual std::unique_ptr 46 | make_hovermap_data(const std::string &name) const = 0; 47 | 48 | // Helper function for debugging. 49 | virtual uint32_t get_pixel(int x, int y) const = 0; 50 | }; 51 | 52 | // RgbBuffer internally contains two buffers: 53 | // - `buf` contains RGB values in successive 4x4 blocks. 54 | // - `hovermap` contains the ID of the last merge() operation that touched the 55 | // pixel: it is used by FE to figure out which "line" is currently under the 56 | // mouse cursor. 57 | class RgbBuffer final : public ColoredBufferBase { 58 | public: 59 | // Comprised of 16-byte blocks, where each block is a 4x4 area. 60 | // Block #0: (0..3, 0..3), R 61 | // Block #1: (0..3, 0..3), G 62 | // Block #2: (0..3, 0..3), B 63 | // Block #3: (4..7, 0..3), R 64 | // ... 65 | alignas(16) __m128i buf[BLK_CNT * 3]; 66 | 67 | // Comprised of 64-byte blocks, where each block is a 4x4 area of 32-bit 68 | // integers. Initialized to -1. 69 | // 70 | // It seems like C++ doesn't correctly support 32-byte alignment until 71 | // C++17 - so let's use posix_memalign for now, so that we can compile on 72 | // C++14. 73 | __m256i *hovermap; // alignas(32) __m256i hovermap[BLK_CNT * 2]; 74 | 75 | RgbBuffer(uint32_t color); // color = 0x??rrggbb 76 | ~RgbBuffer(); 77 | 78 | void merge(GrayscaleBuffer *buf, int line_id, 79 | uint32_t color /* 0xaarrggbb */) override; 80 | 81 | std::unique_ptr 82 | make_png_data(const std::string &name) const override; 83 | 84 | // Create a buffer of hovermap data arranged in the ordinary pixel order. 85 | std::unique_ptr 86 | make_hovermap_data(const std::string &name) const override; 87 | 88 | uint32_t get_pixel(int x, int y) const override { 89 | int idx1 = (y / 4) * 64 + (x / 4); 90 | int idx2 = (y % 4) * 4 + (x % 4); 91 | 92 | uint8_t r = ((const char *) buf)[idx1 * 48 + idx2]; 93 | uint8_t g = ((const char *) buf)[idx1 * 48 + 16 + idx2]; 94 | uint8_t b = ((const char *) buf)[idx1 * 48 + 32 + idx2]; 95 | return ((uint32_t) r << 16) + ((uint32_t) g << 8) + b; 96 | } 97 | }; 98 | 99 | // For highlight tiles: similar as above, but also contains the alpha channel, 100 | // and does not contain the hovermap. 101 | // 102 | // The formula for correctly combining RGBA channels is rather complicated [1]. 103 | // To simplify computation, and because I'm lazy, we store intermediate data in 104 | // a non-transparent "4-color" format: let's call it RGBW. 105 | // 106 | // - At start, everything is black. 107 | // 108 | // - At each merge(), RGB colors are added to the black background just like 109 | // RgbBuffer. A pseudo-color, "white", is added as if it is always 255. 110 | // I.e., if the given color is (r, g, b), then we treat it as (R=r, G=g, B=b, 111 | // W=255). 112 | // 113 | // - At the end, (r, g, b, w) can be converted to RGBA as: 114 | // R = r * (255 / w) 115 | // G = g * (255 / w) 116 | // B = b * (255 / w) 117 | // A = w 118 | // 119 | // (This does not overflow because the construction guarantees r<=w, etc.) 120 | // 121 | // This obviously loses some information: e.g., if w=3, then r can be only 122 | // between [0, 3], so the final red color has only two bits of depth. On the 123 | // other hand, who really cares about fidelity of red when alpha channel is 3. 124 | // 125 | // [1] See: https://en.wikipedia.org/wiki/Alpha_compositing 126 | // 127 | // TODO: Refactor? 128 | class RgbaBuffer final : public ColoredBufferBase { 129 | public: 130 | // Comprised of 16-byte blocks, where each block is a 4x4 area. 131 | // 132 | // Block #0: (0..3, 0..3), R 133 | // Block #1: (0..3, 0..3), G 134 | // Block #2: (0..3, 0..3), B 135 | // Block #3: (0..3, 0..3), W 136 | // Block #4: (4..7, 0..3), R 137 | // ... 138 | alignas(16) __m128i buf[BLK_CNT * 4]; 139 | 140 | RgbaBuffer() { memset(buf, 0x00, sizeof(buf)); } 141 | 142 | // `line_id` is unused here. 143 | void merge(GrayscaleBuffer *buf, int line_id, 144 | uint32_t color /* 0xaarrggbb */) override; 145 | 146 | std::unique_ptr 147 | make_png_data(const std::string &name) const override; 148 | 149 | std::unique_ptr 150 | make_hovermap_data(const std::string &name) const override { 151 | DIE_MSG("RgbaBuffer doesn't support make_hovermap_data()!\n"); 152 | } 153 | 154 | uint32_t get_pixel(int x, int y) const override { 155 | int idx1 = (y / 4) * 64 + (x / 4); 156 | int idx2 = (y % 4) * 4 + (x % 4); 157 | 158 | uint8_t r = ((const char *) buf)[idx1 * 64 + idx2]; 159 | uint8_t g = ((const char *) buf)[idx1 * 64 + 16 + idx2]; 160 | uint8_t b = ((const char *) buf)[idx1 * 64 + 32 + idx2]; 161 | uint8_t w = ((const char *) buf)[idx1 * 64 + 48 + idx2]; 162 | return ((uint32_t) w << 24) + 163 | ((uint32_t) r << 16) + ((uint32_t) g << 8) + b; 164 | } 165 | }; 166 | 167 | } // namespace croquis 168 | -------------------------------------------------------------------------------- /src/csrc/croquis/task.h: -------------------------------------------------------------------------------- 1 | // A unit of work that can run in any worker thread. 2 | 3 | #pragma once 4 | 5 | #include // int64_t 6 | 7 | #include 8 | #include // unique_ptr 9 | 10 | #include "croquis/util/clock.h" // microtime 11 | #include "croquis/util/macros.h" // CHECK 12 | 13 | namespace croquis { 14 | 15 | class ThrManager; 16 | class WorkThr; 17 | 18 | // Forward declaration. 19 | class Task; 20 | 21 | // TODO: The task object is destroyed in the worker thread, which is most likely 22 | // a different thread than the one that created it. It might increase CPU 23 | // overhead. Maybe we could use a task object pool...? 24 | class Task { 25 | public: 26 | // Currently we support three scheduling classes for Tasks. 27 | // 28 | // - SCHD_FIFO is regular tasks (highest priority), served FIFO. 29 | // 30 | // - SCHD_LIFO is used for tiles, served *LIFO*, because more recent tile 31 | // requests are usually more relevant. 32 | // 33 | // - SCHD_LIFO_LOW is similar to SCHD_LIFO but has lower priority: they are 34 | // used for low-priority highlight tiles. 35 | // 36 | // Tasks using SCHD_LIFO/SCHD_LIFO_LOW can be "expedited" by calling 37 | // ThrManager::expedite_task() - we then update `enqueue_time_` to the 38 | // current time, so that it has the highest priority. 39 | // 40 | // To avoid starvation, we reserve some scheduling slot for executing tasks 41 | // in SCHD_LIFO/SCHD_LIFO_LOW in FIFO order. These tasks should check 42 | // how long they stayed in the queue and abort if they are "stale". 43 | enum ScheduleClass { 44 | SCHD_FIFO = 0, 45 | SCHD_LIFO = 1, 46 | SCHD_LIFO_LOW = 2, 47 | }; 48 | 49 | // Determines whether the task is "owned" by ThrManager. 50 | // 51 | // A task owned by ThrManager (TMGR_OWNED) is deleted upon completion. 52 | // 53 | // An externally owned task (EXTERNAL_OWNED) is *not* automatically deleted 54 | // upon completion: the caller that created this task should own the 55 | // pointer. This is the expected behavior for tasks using 56 | // SCHD_LIFO/SCHD_LIFO_LOW, so that they can be "expedited" while in the 57 | // queue: otherwise there will be a race condition between 58 | // ThrManager::expedite_task() and ThrManager actually running the task and 59 | // deleting it. 60 | // 61 | // An EXTERNAL_OWNED task may transition to TMGR_OWNED if the owner 62 | // relinquishes ownership, or DONE if it is finished. 63 | enum Status : int { 64 | TMGR_OWNED = 0, 65 | EXTERNAL_OWNED = 1, 66 | DONE = 2, 67 | }; 68 | 69 | private: 70 | friend class WorkThr; 71 | friend class ThrManager; 72 | friend class ThrHelper; 73 | 74 | const ScheduleClass sched_class_; 75 | 76 | // Time when this task is enqueued: used for SCHD_LIFO and SCHD_LIFO_LOW. 77 | int64_t enqueue_time_; 78 | 79 | // Pointers to construct task queues and priority queues: managed by WorkThr 80 | // and ThrManager. 81 | Task *next_ = nullptr; 82 | Task *prev_ = nullptr; 83 | int heap_idx_ = -1; 84 | 85 | // Prerequisite count: counts the number of unfinished tasks that are its 86 | // own prerequisites. If this becomes zero, we can start. 87 | // 88 | // Accessed via ThrManager. 89 | // 90 | // We actually start with count 1, which is later decremented by 91 | // ThrManager::enqueue(). This way, a Task cannot prematurely start before 92 | // it is officially enqueued. 93 | std::atomic prereq_cnt_{1}; 94 | 95 | std::atomic status_{EXTERNAL_OWNED}; 96 | 97 | // Dependent task: an optional task for which this task is a prerequisite. 98 | Task *dep_; 99 | 100 | protected: 101 | // Specify that this is a prerequiste task of another task `dep`. 102 | // `dep` must not have been enqueued yet. 103 | explicit Task(ScheduleClass sched_class = SCHD_FIFO, Task *dep = nullptr) 104 | : sched_class_(sched_class), enqueue_time_(util::microtime()), dep_(dep) 105 | { 106 | if (dep != nullptr) dep->prereq_cnt_.fetch_add(1); 107 | } 108 | 109 | public: 110 | virtual ~Task() { } 111 | virtual void run() = 0; 112 | 113 | // Safely relinquish ownership of a task that may or may not be finished 114 | // yet. 115 | static void relinquish_ownership(std::unique_ptr task) { 116 | Status s = EXTERNAL_OWNED; 117 | 118 | // If CAS succeeds, then the thread is now owned by ThrManager: it will 119 | // be freed when it is complete. 120 | // 121 | // If it fails, then the thread must be already complete (DONE), so we 122 | // should free it here. 123 | if (task->status_.compare_exchange_strong(s, TMGR_OWNED)) 124 | task.release(); 125 | else { 126 | CHECK(s == DONE); 127 | task.reset(); 128 | } 129 | } 130 | 131 | DISALLOW_COPY_AND_MOVE(Task); 132 | }; 133 | 134 | // A simple task to run a function. 135 | template class LambdaTask : public Task { 136 | private: 137 | T fn_; 138 | 139 | public: 140 | explicit LambdaTask(T &&fn, 141 | ScheduleClass sched_class = SCHD_FIFO, 142 | Task *dep = nullptr) 143 | : Task(sched_class, dep), fn_(std::move(fn)) { } 144 | 145 | public: 146 | virtual void run() override { fn_(); } 147 | }; 148 | 149 | template std::unique_ptr> 150 | make_lambda_task(T &&fn, 151 | Task::ScheduleClass sched_class = Task::SCHD_FIFO, 152 | Task *dep = nullptr) 153 | { 154 | return std::make_unique>(std::move(fn), sched_class, dep); 155 | } 156 | 157 | } // namespace croquis 158 | -------------------------------------------------------------------------------- /src/csrc/croquis/tests/bitmap_buffer_test.cc: -------------------------------------------------------------------------------- 1 | // Test BitmapBuffer. 2 | // 3 | // TODO: Use a proper test framework! 4 | 5 | #include "croquis/bitmap_buffer.h" 6 | 7 | #include 8 | #include // sqrtf 9 | #include // fopen 10 | #include // uint64_t 11 | #include // memcpy 12 | 13 | #include 14 | 15 | namespace croquis { 16 | 17 | // Helper function for examining the contents of the buffer. 18 | static void write_bitmap(const BitmapBuffer &buf, const char *filename) 19 | { 20 | FILE *fp = fopen(filename, "wb"); 21 | for (int y = 0; y < 512; y++) { 22 | for (int x = 0; x < 512; x++) { 23 | uint8_t pixel = buf.get_pixel(x, y) ? 255 : 0; 24 | fwrite(&pixel, 1, 1, fp); 25 | } 26 | } 27 | fclose(fp); 28 | } 29 | 30 | static void do_test(BitmapBuffer *buf, 31 | float x0, float y0, float x1, float y1, float width) 32 | { 33 | BitmapBuffer before; 34 | memcpy(before.buf, buf->buf, sizeof(buf->buf)); 35 | 36 | buf->draw_line(x0, y0, x1, y1, width); 37 | 38 | // Now check if the line was drawn correctly. 39 | const float dx = x1 - x0; 40 | const float dy = y1 - y0; 41 | const float len = sqrtf(dx * dx + dy * dy); 42 | const float wx = dy * (width / (2 * len)); 43 | const float wy = dx * (width / (2 * len)); 44 | 45 | const int xmin = nearbyintf(x0 - wx); 46 | const int xmax = nearbyintf(x1 + wx); 47 | const int ymin = nearbyintf(y0 - wy); 48 | const int ymax = nearbyintf(y1 + wy); 49 | 50 | const float slope = dy / dx; 51 | 52 | for (int y = 0; y < 512; y++) { 53 | for (int x = 0; x < 512; x++) { 54 | bool orig = before.get_pixel(x, y); 55 | bool pixel = buf->get_pixel(x, y); 56 | 57 | float yL = slope * (x - (x0 + wx)) + (y0 - wy); 58 | float yH = slope * (x - (x0 - wx)) + (y0 + wy); 59 | bool is_line = (xmin <= x) && (x <= xmax) && 60 | (ymin <= y) && (y <= ymax) && 61 | (y >= yL) && (y <= yH); 62 | 63 | // TODO: We need some macro for testing! 64 | if (is_line) { 65 | assert(pixel); 66 | if (!pixel) { 67 | printf("missing pixel: x=%d y=%d yL=%.4f yH=%.4f\n", 68 | x, y, yL, yH); 69 | } 70 | } 71 | else { 72 | assert(orig == pixel); 73 | if (orig != pixel) { 74 | printf("should be same but orig %d pixel %d: x=%d y=%d\n", 75 | orig, pixel, x, y); 76 | } 77 | } 78 | } 79 | } 80 | 81 | // Check that blklist is correct. 82 | std::unordered_set blks; 83 | for (int i = 0; i < buf->blk_cnt; i++) { 84 | int blk_id = buf->blklist[i]; 85 | assert(blks.count(blk_id) == 0); 86 | blks.insert(blk_id); 87 | } 88 | 89 | for (int blk_id = 0; blk_id < BitmapBuffer::BLK_CNT; blk_id++) 90 | assert(blks.count(blk_id) == (buf->buf[blk_id] != 0ULL)); 91 | } 92 | 93 | static void run_test() 94 | { 95 | BitmapBuffer buf; 96 | 97 | do_test(&buf, -20.0, 10.0, 300.0, 150.0, 4.5); 98 | do_test(&buf, 20.0, 10.0, 300.0, 250.0, 3.0); 99 | do_test(&buf, 50.0, 125.0, 700.0, 300.0, 15.0); 100 | 101 | // Test flat line. 102 | do_test(&buf, 40.0, 350.0, 600.0, 350.0, 8.1); 103 | do_test(&buf, 30.0, 450.0, 600.0, 451.7, 2.5); 104 | 105 | write_bitmap(buf, "test1.dat"); 106 | } 107 | 108 | } // namespace croquis 109 | 110 | int main() 111 | { 112 | croquis::run_test(); 113 | return 0; 114 | } 115 | -------------------------------------------------------------------------------- /src/csrc/croquis/tests/grayscale_buffer_test.cc: -------------------------------------------------------------------------------- 1 | // Test GrayscaleBuffer. 2 | // 3 | // TODO: Use a proper test framework! 4 | 5 | #include "croquis/grayscale_buffer.h" 6 | 7 | #include 8 | #include // sqrtf 9 | #include // fopen 10 | #include // uint64_t 11 | #include // memcpy 12 | 13 | #include // _mm_testz_si128 14 | 15 | #include // max 16 | #include 17 | #include 18 | 19 | #include "croquis/util/string_printf.h" 20 | 21 | namespace croquis { 22 | 23 | // Helper function for examining the contents of the buffer. 24 | static void write_bitmap(const GrayscaleBuffer &buf, const char *filename) 25 | { 26 | FILE *fp = fopen(filename, "wb"); 27 | for (int y = 0; y < 256; y++) { 28 | for (int x = 0; x < 256; x++) { 29 | uint8_t pixel = buf.get_pixel(x, y); 30 | fwrite(&pixel, 1, 1, fp); 31 | } 32 | } 33 | fclose(fp); 34 | } 35 | 36 | static void do_test(GrayscaleBuffer *buf, 37 | float x0, float y0, float x1, float y1, float width) 38 | { 39 | #if 0 40 | printf("!!! x0 y0 x1 y1 w = %.3f %.3f %.3f %.3f %.3f\n", 41 | x0, y0, x1, y1, width); 42 | #endif 43 | 44 | GrayscaleBuffer before; 45 | memcpy(before.buf, buf->buf, sizeof(buf->buf)); 46 | 47 | buf->draw_line(x0, y0, x1, y1, width); 48 | 49 | bool xyflip = (fabsf(y1 - y0) > fabsf(x1 - x0)); 50 | if (xyflip) { 51 | float tmp; 52 | tmp = x0; x0 = y0; y0 = tmp; 53 | tmp = x1; x1 = y1; y1 = tmp; 54 | } 55 | 56 | if (x1 < x0) { 57 | float tmp; 58 | tmp = x0; x0 = x1; x1 = tmp; 59 | tmp = y0; y0 = y1; y1 = tmp; 60 | } 61 | 62 | // Now check if the line was drawn correctly. 63 | const float dx = x1 - x0; 64 | const float dy = y1 - y0; 65 | const float len = sqrtf(dx * dx + dy * dy); 66 | const float wx = dy * (width / (2 * len)); 67 | const float wy = dx * (width / (2 * len)); 68 | 69 | // Guard against random numerical inconsistency. 70 | const float EPSILON = 0.001; 71 | 72 | const int xmin0 = nearbyintf(fminf(x0, x1) - fabsf(wx) - EPSILON); 73 | const int xmax0 = nearbyintf(fmaxf(x0, x1) + fabsf(wx) - EPSILON); 74 | const int ymin0 = nearbyintf(fminf(y0, y1) - fabsf(wy) - EPSILON); 75 | const int ymax0 = nearbyintf(fmaxf(y0, y1) + fabsf(wy) - EPSILON); 76 | 77 | const int xmin1 = nearbyintf(fminf(x0, x1) - fabsf(wx) + EPSILON); 78 | const int xmax1 = nearbyintf(fmaxf(x0, x1) + fabsf(wx) + EPSILON); 79 | const int ymin1 = nearbyintf(fminf(y0, y1) - fabsf(wy) + EPSILON); 80 | const int ymax1 = nearbyintf(fmaxf(y0, y1) + fabsf(wy) + EPSILON); 81 | 82 | const float slope = dy / dx; 83 | 84 | #if 0 85 | printf(" %sx0 y0 x1 y1 w = %.3f %.3f %.3f %.3f %.3f\n", 86 | (xyflip) ? "(xy flipped) " : "", 87 | x0, y0, x1, y1, width); 88 | printf(" dx dy len wx wy = %.3f %.3f %.3f %.4f %.4f\n", 89 | dx, dy, len, wx, wy); 90 | printf(" xmin xmax ymin ymax = %d(%d) %d(%d) %d(%d) %d(%d)\n", 91 | xmin0, xmin1, xmax0, xmax1, ymin0, ymin1, ymax0, ymax1); 92 | #endif 93 | 94 | for (int y = 0; y < 256; y++) { 95 | for (int x = 0; x < 256; x++) { 96 | uint8_t orig, pixel; 97 | if (xyflip) { 98 | orig = before.get_pixel(y, x); 99 | pixel = buf->get_pixel(y, x); 100 | } 101 | else { 102 | orig = before.get_pixel(x, y); 103 | pixel = buf->get_pixel(x, y); 104 | } 105 | 106 | float yL = slope * (x - (x0 + wx)) + (y0 - wy); 107 | float yH = slope * (x - (x0 - wx)) + (y0 + wy); 108 | 109 | // allowed0/allowed1: slightly smaller/larger bounding box. 110 | bool allowed0 = 111 | (xmin1 <= x) && (x <= xmax0) && (ymin1 <= y) && (y <= ymax0); 112 | bool allowed1 = 113 | (xmin0 <= x) && (x <= xmax1) && (ymin0 <= y) && (y <= ymax1); 114 | 115 | float frac = fminf(y + 0.5f - yL, 1.0f); 116 | if (yH < y + 0.5f) frac -= y + 0.5f - yH; 117 | frac = fmaxf(frac, 0.0f); 118 | 119 | uint8_t color0 = nearbyintf(frac * 255.f) * allowed0; 120 | uint8_t expected0 = std::max(orig, color0); 121 | uint8_t color1 = nearbyintf(frac * 255.f) * allowed1; 122 | uint8_t expected1 = std::max(orig, color1); 123 | 124 | // TODO: We need some macro for testing! 125 | int threshold = 1 + (int) (width / 40); 126 | // assert(abs(expected - pixel) <= threshold); 127 | if (abs(expected0 - pixel) > threshold && 128 | abs(expected1 - pixel) > threshold) { 129 | printf("(%d, %d) orig=%d expected color=%d(%d) actual=%d: " 130 | "yL=%.3f yH=%.3f\n", 131 | x, y, orig, color0, color1, pixel, yL, yH); 132 | } 133 | } 134 | } 135 | 136 | // Check that blklist is correct. 137 | std::unordered_set blks; 138 | for (int i = 0; i < buf->blk_cnt; i++) { 139 | int blk_id = buf->blklist[i]; 140 | // printf("Found blk #%d\n", blk_id); 141 | if (blks.count(blk_id)) printf("blk #%d duplicate!\n", blk_id); 142 | assert(blks.count(blk_id) == 0); 143 | blks.insert(blk_id); 144 | } 145 | 146 | for (int blk_id = 0; blk_id < GrayscaleBuffer::BLK_CNT; blk_id++) { 147 | __m128i blk = buf->buf[blk_id]; 148 | bool is_zero = _mm_testz_si128(blk, blk); 149 | assert(blks.count(blk_id) == !is_zero); 150 | } 151 | } 152 | 153 | static void test_lines() 154 | { 155 | GrayscaleBuffer buf; 156 | 157 | do_test(&buf, -20.0, 10.0, 250.0, 150.0, 4.5); 158 | do_test(&buf, 20.0, 10.0, 250.0, 220.0, 3.0); 159 | do_test(&buf, 50.0, 125.0, 500.0, 200.0, 15.0); 160 | 161 | // Test flat line. 162 | do_test(&buf, 40.0, 150.0, 300.0, 150.0, 8.1); 163 | do_test(&buf, 30.0, 200.0, 300.0, 202.5, 2.5); 164 | 165 | // Some random lines from previous tests. 166 | do_test(&buf, -63.78, 289.14, 225.55, 131.13, 3.29); 167 | do_test(&buf, -170.27, 185.94, 249.37, 93.87, 38.43); 168 | do_test(&buf, 278.843, -1.208, -205.838, 307.298, 1.794); 169 | do_test(&buf, 484.980, 276.463, 23.283, 113.903, 2.975); 170 | 171 | write_bitmap(buf, "test1.dat"); 172 | } 173 | 174 | static void test_random_lines() 175 | { 176 | std::mt19937 gen(12345678); // Random number generator. 177 | std::normal_distribution coord_dist(128.0, 200.0); 178 | std::uniform_real_distribution width_dist(0.0, 5.0); 179 | 180 | for (int n = 0; n < 20; n++) { 181 | printf("Running random test iteration #%d ...\n", n); 182 | GrayscaleBuffer buf; 183 | 184 | for (int i = 0; i < 200; i++) { 185 | float x0 = coord_dist(gen); 186 | float y0 = coord_dist(gen); 187 | float x1 = coord_dist(gen); 188 | float y1 = coord_dist(gen); 189 | 190 | // Skew toward reasonable (<5) width, but leave a few large values 191 | // for testing edge cases. 192 | float width = width_dist(gen); 193 | if (width_dist(gen) < 0.1) 194 | width *= 10; 195 | if (width_dist(gen) < 0.001 * n) 196 | width *= 100; 197 | 198 | // printf("%.3f %.3f %.3f %.3f %.3f\n", x0, y0, x1, y1, width); 199 | do_test(&buf, x0, y0, x1, y1, width); 200 | } 201 | 202 | write_bitmap(buf, util::string_printf("random-%d.dat", n).c_str()); 203 | } 204 | } 205 | 206 | static void run_test() 207 | { 208 | test_lines(); 209 | test_random_lines(); 210 | } 211 | 212 | } // namespace croquis 213 | 214 | int main() 215 | { 216 | croquis::run_test(); 217 | return 0; 218 | } 219 | -------------------------------------------------------------------------------- /src/csrc/croquis/tests/show_bitmap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # A utility script to visualize the given bitmap. 4 | 5 | import sys 6 | 7 | import numpy as np 8 | import matplotlib.pyplot as plt 9 | 10 | with open(sys.argv[1], 'rb') as f: 11 | dat = f.read() 12 | 13 | dat = np.frombuffer(dat, dtype=np.uint8).reshape(256, 256) 14 | 15 | plt.imshow(dat) 16 | plt.show() 17 | -------------------------------------------------------------------------------- /src/csrc/croquis/thr_manager.h: -------------------------------------------------------------------------------- 1 | // Thread pool manager. 2 | // The threads are shared by all plots inside the same process. 3 | 4 | #pragma once 5 | 6 | #include 7 | #include // uintptr_t 8 | 9 | #include 10 | #include 11 | #include // unique_ptr 12 | #include 13 | #include // mt19937 14 | #include 15 | #include // forward 16 | #include 17 | 18 | #include "croquis/message.h" 19 | #include "croquis/task.h" 20 | #include "croquis/util/macros.h" // CHECK 21 | 22 | namespace croquis { 23 | 24 | class ThrManager; 25 | extern ThrManager *tmgr_; // Singleton. 26 | 27 | class WorkThr { 28 | private: 29 | friend class ThrManager; 30 | 31 | const int idx_; 32 | const std::thread::id tid_; 33 | 34 | // Random number generator for scheduling. (This is probably overkill, but 35 | // it's hard to find a better RNG that's easy to use...) 36 | std::mt19937 gen_; 37 | 38 | public: 39 | explicit WorkThr(int idx); 40 | void run(); 41 | }; 42 | 43 | class ThrManager { 44 | public: 45 | // A generic callback, telling the Python code to send the data back to FE. 46 | // 47 | // Arguments are: key (address of the sender object); 48 | // data as json string; 49 | // optional binary data (up to two). 50 | typedef std::function &, 51 | std::unique_ptr, 52 | std::unique_ptr)> 53 | PyCallback_t; 54 | 55 | const int nthreads; 56 | 57 | private: 58 | friend class WorkThr; 59 | 60 | const std::thread::id mgr_tid_; 61 | std::vector> wthrs_; 62 | 63 | PyCallback_t py_callback_; 64 | 65 | // Mutex and condition variable to guard task_queue_. 66 | std::mutex m_; 67 | std::condition_variable cv_; 68 | std::condition_variable shutdown_cv_; 69 | 70 | bool shutdown_ = false; 71 | 72 | // All tasks with SCHD_FIFO constitute a circular doubly-linked list. This 73 | // points to the head (i.e., the next task to execute). 74 | Task *fifo_queue_ = nullptr; 75 | int fifo_queue_size_ = 0; // For debugging. 76 | 77 | // All remaining tasks also constitute another circular doubly-linked list, 78 | // in order to avoid starvation. This points to the head. 79 | Task *low_prio_queue_ = nullptr; 80 | 81 | // Tasks in SCHD_LIFO are stored here as a max-heap on enqueue_time_. 82 | // TODO: Actually, since an "expedited" task can only move to the head of 83 | // the queue, we don't need a full-fledged max-heap. We can simply 84 | // use a doubly linked list! -_- 85 | std::vector lifo_heap_; 86 | std::vector lifo_low_heap_; 87 | 88 | public: 89 | ThrManager(int nthreads, PyCallback_t py_callback, 90 | double start_time, int log_fd); 91 | 92 | #if 0 93 | // Shuts down the existing thread manager. 94 | // TODO: Currently not used. Do we need this? 95 | static void shutdown(); 96 | #endif 97 | 98 | // Called by Python for each worker thread. 99 | void wthr_entry_point(int idx); 100 | 101 | // Enqueue a task: can be called by any thread. 102 | // 103 | // If the task has any prerequiste tasks, it won't be actually "enqueued" in 104 | // any data structure - it will be enqueued when `prereq_cnt_` becomes zero. 105 | static void enqueue(std::unique_ptr task); 106 | 107 | // Enqueue a task, but do not transfer ownership. 108 | static void enqueue_no_delete(Task *task); 109 | 110 | // Convenience functions. 111 | template 112 | static void enqueue(Args&&... args) { 113 | enqueue(std::make_unique(std::forward(args)...)); 114 | } 115 | 116 | template 117 | static Task *enqueue_lambda( 118 | T &&fn, 119 | Task::ScheduleClass sched_class = Task::SCHD_FIFO, 120 | Task *dep = nullptr) { 121 | std::unique_ptr task = 122 | make_lambda_task(std::move(fn), sched_class, dep); 123 | Task *t = task.get(); 124 | enqueue(std::move(task)); 125 | return t; 126 | } 127 | 128 | template 129 | static std::unique_ptr enqueue_lambda_no_delete( 130 | T &&fn, 131 | Task::ScheduleClass sched_class, 132 | Task *dep = nullptr) { 133 | std::unique_ptr task = 134 | make_lambda_task(std::move(fn), sched_class, dep); 135 | enqueue_no_delete(task.get()); 136 | return task; 137 | } 138 | 139 | // Expedite a LIFO task so that it has the highest priority among its 140 | // scheduling class. See Task::ScheduleClass for discussion. 141 | static void expedite_task(Task *t) { tmgr_->do_expedite_task(t); } 142 | 143 | private: 144 | // Internal helper functions. 145 | void do_enqueue(Task *t); 146 | void do_expedite_task(Task *t); 147 | 148 | // Called by WorkThr: may return nullptr if we're shutting down. 149 | Task *dequeue_task(WorkThr *wthr); 150 | 151 | friend void set_task_ready(Task *t); 152 | 153 | public: 154 | // Call the python callback to send a message: can be called by any thread. 155 | // 156 | // (Not really relevant to ThrManager, but this seems the most convenient 157 | // place to have this...) 158 | // 159 | // `dict` contains key-value pairs in the format "x=y", e.g., 160 | // {"msg=test_message", "foo=hello", "#bar=3"}. 161 | // (Use '#' in front of the key to create a numeric value.) 162 | bool send_msg(uintptr_t obj_id, const std::vector &dict, 163 | std::unique_ptr data1 = nullptr, 164 | std::unique_ptr data2 = nullptr); 165 | 166 | template 167 | bool send_msg(const T *obj, const std::vector &dict, 168 | std::unique_ptr data1 = nullptr, 169 | std::unique_ptr data2 = nullptr) { 170 | return send_msg((uintptr_t) obj, dict, 171 | std::move(data1), std::move(data2)); 172 | } 173 | }; 174 | 175 | } // namespace croquis 176 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/avx_util.h: -------------------------------------------------------------------------------- 1 | // Utility functions to be used with AVX intrinsics. 2 | 3 | #include // memcpy 4 | 5 | #include 6 | 7 | #include 8 | 9 | #include "croquis/util/string_printf.h" 10 | 11 | namespace croquis { 12 | namespace util { 13 | 14 | static inline std::string to_string(__m256i a) 15 | { 16 | int d[8]; 17 | memcpy((void *) d, (void *) &a, 32); 18 | return string_printf("%08x_%08x_%08x_%08x_%08x_%08x_%08x_%08x", 19 | d[7], d[6], d[5], d[4], d[3], d[2], d[1], d[0]); 20 | } 21 | 22 | } // namespace util 23 | } // namespace croquis 24 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/clock.h: -------------------------------------------------------------------------------- 1 | // Utility function for getting a monotonic clock. 2 | 3 | #pragma once 4 | 5 | #include // int64_t 6 | #include // clock_gettime 7 | 8 | namespace croquis { 9 | namespace util { 10 | 11 | // Return a monotonic timestamp in microsecond resolution. 12 | // 13 | // Not sure exactly how it is computed, but assuming it starts at zero at boot, 14 | // we can run for ~292k years before overflow, so I think we're good. 15 | // 16 | // TODO: What about Windows? 17 | inline int64_t microtime() 18 | { 19 | struct timespec ts; 20 | 21 | // Let's ignore rc: hopefully it shouldn't fail ... 22 | int rc = clock_gettime(CLOCK_MONOTONIC, &ts); 23 | return ((int64_t) (ts.tv_sec) * 1000000) + (int64_t) (ts.tv_nsec / 1000); 24 | } 25 | 26 | } // namespace util 27 | } // namespace croquis 28 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/color_util.h: -------------------------------------------------------------------------------- 1 | // Miscellaneous helper functions for RGB handling. 2 | 3 | #pragma once 4 | 5 | #include // fmaxf 6 | #include // uint32_t 7 | 8 | namespace croquis { 9 | namespace util { 10 | 11 | // Convert floating-point RGB values to a 3-byte value. 12 | static inline uint32_t get_rgb(float r, float g, float b) 13 | { 14 | int r1 = nearbyintf(fmaxf(0.0f, fminf(r, 1.0f)) * 255.0); 15 | int g1 = nearbyintf(fmaxf(0.0f, fminf(g, 1.0f)) * 255.0); 16 | int b1 = nearbyintf(fmaxf(0.0f, fminf(b, 1.0f)) * 255.0); 17 | 18 | return (r1 << 16) + (g1 << 8) + b1; 19 | } 20 | 21 | static inline uint32_t get_argb(float r, float g, float b) 22 | { 23 | return 0xff000000 + get_rgb(r, g, b); 24 | } 25 | 26 | } // namespace util 27 | } // namespace croquis 28 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/error_helper.cc: -------------------------------------------------------------------------------- 1 | // Helper for creating Python exceptions. 2 | 3 | #include "croquis/util/error_helper.h" 4 | 5 | #include // va_start 6 | 7 | #include 8 | 9 | #include "croquis/util/string_printf.h" 10 | 11 | namespace croquis { 12 | namespace util { 13 | 14 | namespace py = pybind11; 15 | 16 | [[noreturn]] void throw_value_error(const char *fmt, ...) 17 | { 18 | va_list va; 19 | va_start(va, fmt); 20 | std::string msg = string_vprintf(fmt, va); 21 | va_end(va); 22 | 23 | throw py::value_error(msg); 24 | } 25 | 26 | } // namespace util 27 | } // namespace croquis 28 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/error_helper.h: -------------------------------------------------------------------------------- 1 | // Helper for creating Python exceptions. 2 | 3 | #pragma once 4 | 5 | namespace croquis { 6 | namespace util { 7 | 8 | [[noreturn]] void throw_value_error(const char *fmt, ...) 9 | __attribute__((format(printf, 1, 2))); 10 | 11 | } // namespace util 12 | } // namespace croquis 13 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/logging.cc: -------------------------------------------------------------------------------- 1 | // Logging support. 2 | 3 | #include "croquis/util/logging.h" 4 | 5 | #include // floor 6 | #include // strrchr 7 | #include // clock_gettime, strftime 8 | #include // write 9 | 10 | #include 11 | #include 12 | 13 | #include "croquis/util/string_printf.h" 14 | 15 | namespace croquis { 16 | namespace util { 17 | 18 | static double start_time_; 19 | static int log_fd_ = -1; 20 | 21 | static thread_local const char *thr_name_ = ""; 22 | 23 | // Keep thread names here: never freed. 24 | static std::vector thr_names_; 25 | 26 | void init_logging(double start_time, int log_fd) 27 | { 28 | start_time_ = start_time; 29 | log_fd_ = log_fd; 30 | } 31 | 32 | void set_thread_name(const std::string &name) 33 | { 34 | std::string *s = new std::string(name); 35 | thr_names_.push_back(s); 36 | thr_name_ = s->c_str(); 37 | } 38 | 39 | void log(const char *file, int line, const std::string &s) 40 | { 41 | // Remove directory from `file`. 42 | const char *p = strrchr(file, '/'); 43 | if (p != nullptr) file = p + 1; 44 | 45 | // Find the current time (HH:MM:SS). 46 | struct timespec ts; 47 | clock_gettime(CLOCK_REALTIME, &ts); 48 | time_t tv_sec = ts.tv_sec; 49 | struct tm tm; 50 | localtime_r(&tv_sec, &tm); 51 | 52 | char time_str[20]; 53 | strftime(time_str, 20, "%H:%M:%S", &tm); 54 | 55 | double T = (double) ts.tv_sec + (ts.tv_nsec / 1e9); 56 | double relative = T - start_time_; 57 | double usec = ((relative * 0.01) - floor(relative * 0.01)) * 100; 58 | 59 | // TODO: Python version also shows "elapsed_str", that is, time elapsed 60 | // since the last log by this thread. 61 | 62 | std::string log_line = util::string_printf( 63 | ">%s.%06d %-15s %9.6f %s:%d ", 64 | time_str, (int) (ts.tv_nsec / 1000), thr_name_, usec, file, line); 65 | log_line += s; 66 | if (log_line.back() != '\n') log_line += '\n'; 67 | 68 | // TODO: Support level, and write important messages to stderr! 69 | // write(2, log_line.c_str(), log_line.size()); 70 | 71 | if (log_fd_ != -1) { 72 | // Write to log_fd_, ignore any write errors. 73 | #if defined(__GNUC__) && !defined(__clang__) 74 | # pragma GCC diagnostic ignored "-Wunused-result" 75 | #endif 76 | write(log_fd_, log_line.c_str(), log_line.size()); 77 | } 78 | } 79 | 80 | } // namespace util 81 | } // namespace croquis 82 | 83 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/logging.h: -------------------------------------------------------------------------------- 1 | // Logging support. 2 | 3 | #pragma once 4 | 5 | #include // va_list 6 | 7 | #include 8 | 9 | #include "croquis/util/string_printf.h" 10 | 11 | namespace croquis { 12 | namespace util { 13 | 14 | // Set up the logging so that it matches Python code. 15 | void init_logging(double start_time, int log_fd); 16 | 17 | // Set up the current thread's name for logging. 18 | void set_thread_name(const std::string &name); 19 | 20 | void log(const char *file, int line, const std::string &s); 21 | 22 | #define DBG_LOG(enabled, msg) \ 23 | do { \ 24 | if (enabled) ::croquis::util::log(__FILE__, __LINE__, msg); \ 25 | } while (0) 26 | 27 | #define DBG_LOG1(enabled, ...) \ 28 | do { \ 29 | if (enabled) \ 30 | ::croquis::util::log(__FILE__, __LINE__, \ 31 | ::croquis::util::string_printf(__VA_ARGS__)); \ 32 | } while (0) 33 | 34 | } // namespace util 35 | } // namespace croquis 36 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/macros.h: -------------------------------------------------------------------------------- 1 | // Miscellaneous macros. 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | #define DISALLOW_COPY_AND_MOVE(T) \ 9 | T(const T &) = delete; \ 10 | T(T &&) = delete; \ 11 | T &operator=(const T &) = delete; \ 12 | T &operator=(T &&) = delete 13 | 14 | // Google-style. 15 | #define CHECK(cond) \ 16 | do { \ 17 | if (!(cond)) { \ 18 | fprintf(stderr, "Assertion failed (%s:%d): %s\n", \ 19 | __FILE__, __LINE__, #cond); \ 20 | abort(); \ 21 | } \ 22 | } while (0) 23 | 24 | #define DIE_MSG(msg) \ 25 | do { \ 26 | fprintf(stderr, "Assertion failed (%s:%d): %s\n", \ 27 | __FILE__, __LINE__, msg); \ 28 | abort(); \ 29 | } while (0) 30 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/math.h: -------------------------------------------------------------------------------- 1 | // Mathematical utility functions. 2 | // TODO: Vectorize? 3 | 4 | #pragma once 5 | 6 | #include // isnan 7 | 8 | #include // pair 9 | 10 | namespace croquis { 11 | namespace util { 12 | 13 | // Utility function to decide the initial coordinate range. 14 | template 15 | static inline std::pair initial_range(T m, T M) 16 | { 17 | T diff = M - m; 18 | T margin = (diff == 0.0) ? 1.0 : diff * 0.05; 19 | return std::make_pair(m - margin, M + margin); 20 | } 21 | 22 | #if 0 23 | float min(const float *data, size_t sz) { 24 | float m = data[0]; 25 | 26 | for (const float *p = data; p < data + sz; p++) { 27 | m = std::min(m, *p); 28 | } 29 | 30 | return m; 31 | } 32 | #endif 33 | 34 | // Find min/max values among non-NaN values. 35 | // If `data` is all NaN, just return some NaN value. 36 | static inline std::pair minmax(const float *data, size_t sz) 37 | { 38 | const float *ptr = data; 39 | const float *end = data + sz; 40 | 41 | float m, M; 42 | while (ptr < end) { 43 | m = M = *(ptr++); 44 | if (!isnan(m)) break; 45 | } 46 | 47 | while (ptr < end) { 48 | float v = *(ptr++); 49 | if (v < m) m = v; 50 | if (v > M) M = v; 51 | ptr++; 52 | } 53 | 54 | return std::make_pair(m, M); 55 | } 56 | 57 | } // namespace util 58 | } // namespace croquis 59 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/myhash.h: -------------------------------------------------------------------------------- 1 | // Utility functions for hashing. 2 | // 3 | // NOTE: libstdc++ uses identity function for std::hash (Yikes!). 4 | 5 | #pragma once 6 | 7 | #include // uint64_t 8 | #include // memcpy 9 | 10 | #include // hash 11 | #include // pair 12 | #include 13 | 14 | namespace croquis { 15 | namespace util { 16 | 17 | template 18 | inline uint32_t unaligned_load32(const T *p) 19 | { 20 | uint32_t v; 21 | memcpy(&v, p, sizeof(uint32_t)); 22 | return v; 23 | } 24 | 25 | template 26 | inline uint64_t unaligned_load64(const T *p) 27 | { 28 | uint64_t v; 29 | memcpy(&v, p, sizeof(uint64_t)); 30 | return v; 31 | } 32 | 33 | template 34 | struct myhash : public std::hash { }; 35 | 36 | // Logic stolen from Hash128to64() inside CityHash. 37 | inline size_t hash_combine(size_t hash1, size_t hash2) 38 | { 39 | const size_t mul = 0x9ddfea08eb382d69ULL; 40 | size_t a = (hash1 ^ hash2) * mul; 41 | a ^= (a >> 47); 42 | size_t b = (hash2 ^ a) * mul; 43 | b ^= (b >> 47); 44 | b *= mul; 45 | return b; 46 | } 47 | 48 | template 49 | struct myhash> { 50 | size_t operator()(const std::pair& p) const 51 | { 52 | const size_t hash1 = myhash()(p.first); 53 | const size_t hash2 = myhash()(p.second); 54 | return hash_combine(hash1, hash2); 55 | } 56 | }; 57 | 58 | template 59 | struct myhash> { 60 | size_t operator()(const std::vector& v) const 61 | { 62 | size_t hash = 0L; 63 | for (const auto &elem : v) 64 | hash = hash_combine(hash, myhash()(elem)); 65 | return hash; 66 | } 67 | }; 68 | 69 | } // namespace util 70 | } // namespace croquis 71 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/optional.h: -------------------------------------------------------------------------------- 1 | // Apparently clang++ in Mac OS does not support std::experimental::optional, 2 | // even when I invoke it with -std=c++14 !! 3 | // So, I've decided to just use C++17 - see CMakeLists.txt for discussion. 4 | // 5 | // On the other hand, in Linux we only have std::experimental::optional if I use 6 | // -std=c++14. 7 | 8 | #if defined(USE_EXPERIMENTAL_OPTIONAL) 9 | 10 | #include 11 | 12 | namespace croquis { 13 | template using optional = std::experimental::optional; 14 | } // namespace croquis 15 | 16 | #elif defined(USE_STD_OPTIONAL) 17 | 18 | #include 19 | 20 | namespace croquis { 21 | template using optional = std::optional; 22 | } // namespace croquis 23 | 24 | #else 25 | 26 | #error "Expecting either USE_EXPERIMENTAL_OPTIONAL or USE_STD_OPTIONAL !!" 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/stl_container_util.h: -------------------------------------------------------------------------------- 1 | // STL container helper functions. 2 | 3 | #pragma once 4 | 5 | #include 6 | #include // forward_as_tuple 7 | #include // move, piecewise_construct 8 | 9 | namespace croquis { 10 | namespace util { 11 | 12 | template 13 | inline void insert_if_nonexistent(T *container, 14 | const typename T::key_type &key, 15 | const typename T::mapped_type &value) 16 | { 17 | if (container->count(key) == 0) { 18 | container->insert(make_pair(key, value)); 19 | } 20 | } 21 | 22 | // Run push_back and return the reference to the added item. 23 | template 24 | inline typename Container::value_type &push_back(Container &v, T elem) 25 | { 26 | v.push_back(std::forward(elem)); 27 | return v.back(); 28 | } 29 | 30 | // Run emplace_back and return the reference to the emplaced item. 31 | template 32 | inline typename Container::value_type &emplace_back(Container &v, Args&&... args) 33 | { 34 | v.emplace_back(std::forward(args)...); 35 | return v.back(); 36 | } 37 | 38 | // Run make_unique + emplace_back and return the reference to the unique_ptr. 39 | template 40 | inline typename Container::value_type & 41 | emplace_back_unique(Container &v, Args&&... args) 42 | { 43 | v.emplace_back(std::make_unique 44 | (std::forward(args)...)); 45 | return v.back(); 46 | } 47 | 48 | // Run emplace and return the reference to the emplaced item. 49 | // (If the key already exists, return the existing element.) 50 | // 51 | // TODO: I added "std::move" below so that it can correctly(???) forward rvalue 52 | // references, but I have no idea how this works. It may break some day... :/ 53 | template 54 | inline typename Container::mapped_type &get_or_emplace( 55 | Container &v, 56 | const typename Container::key_type &key, 57 | Args&&... args) 58 | { 59 | // I guess there was some *very* good reason, but seriously... 60 | // WTF C++11, who thought this is an acceptable syntax? 61 | std::pair ret = 62 | v.emplace(std::piecewise_construct, 63 | std::forward_as_tuple(key), 64 | std::forward_as_tuple(std::move(args)...)); 65 | return ret.first->second; 66 | } 67 | 68 | // Sometimes we do want to move the key. 69 | template 70 | inline typename Container::mapped_type &get_or_emplace( 71 | Container &v, 72 | typename Container::key_type &&key, 73 | Args&&... args) 74 | { 75 | std::pair ret = 76 | v.emplace(std::piecewise_construct, 77 | std::forward_as_tuple(std::move(key)), 78 | std::forward_as_tuple(std::move(args)...)); 79 | return ret.first->second; 80 | } 81 | 82 | // Convenience function to construct strings with a delimiter. 83 | template 84 | static inline void append_str(std::string *result, const T1 &delim, const T2 &s) 85 | { 86 | if (result->empty()) { 87 | *result = s; 88 | } 89 | else { 90 | *result += delim; 91 | *result += s; 92 | } 93 | } 94 | 95 | template 96 | inline std::string join_strings(const Container &v, const T &delim) 97 | { 98 | std::string buf; 99 | for (const std::string &s : v) { 100 | if (buf != "") buf += delim; 101 | buf += s; 102 | } 103 | return buf; 104 | } 105 | 106 | template 107 | inline std::string join_elems(const Container &v, F f, const T &delim) 108 | { 109 | std::string buf; 110 | for (const auto &elem : v) { 111 | if (buf != "") buf += delim; 112 | buf += f(elem); 113 | } 114 | return buf; 115 | } 116 | 117 | template 118 | inline std::string join_to_string(const Container &v, const T &delim) 119 | { 120 | std::string buf; 121 | for (const auto &elem : v) { 122 | if (buf != "") buf += delim; 123 | buf += std::to_string(elem); 124 | } 125 | return buf; 126 | } 127 | 128 | } // namespace util 129 | } // namespace croquis 130 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/string_printf.cc: -------------------------------------------------------------------------------- 1 | #include "croquis/util/string_printf.h" 2 | 3 | #include // va_start 4 | #include // sprintf 5 | 6 | using std::string; 7 | 8 | namespace croquis { 9 | namespace util { 10 | 11 | string string_printf(const char *fmt, ...) 12 | { 13 | std::string retval; 14 | 15 | va_list va; 16 | va_start(va, fmt); 17 | retval = string_vprintf(fmt, va); 18 | va_end(va); 19 | 20 | return retval; 21 | } 22 | 23 | string string_vprintf(const char *fmt, va_list ap) 24 | { 25 | va_list ap2; 26 | va_copy(ap2, ap); 27 | 28 | size_t sz = 64; 29 | std::string buf(sz, ' '); 30 | 31 | int new_sz = vsnprintf(&(buf[0]), sz, fmt, ap); 32 | 33 | // Need one more byte because vsnprintf always emits the final '\0'. 34 | buf.resize(new_sz + 1, ' '); 35 | if (new_sz >= sz) { 36 | vsnprintf(&(buf[0]), new_sz + 1, fmt, ap2); 37 | va_end(ap2); 38 | } 39 | buf.resize(new_sz, ' '); 40 | 41 | return buf; 42 | } 43 | 44 | } // namespace util 45 | } // namespace croquis 46 | -------------------------------------------------------------------------------- /src/csrc/croquis/util/string_printf.h: -------------------------------------------------------------------------------- 1 | // String utility functions. 2 | 3 | #pragma once 4 | 5 | #include // va_list 6 | 7 | #include 8 | 9 | namespace croquis { 10 | namespace util { 11 | 12 | std::string string_printf(const char *fmt, ...) 13 | __attribute__((format(printf, 1, 2))); 14 | 15 | std::string string_vprintf(const char *fmt, va_list ap); 16 | 17 | // Utility function for converting a double exactly. 18 | // According to Wikipedia, 17 digits should be enough. 19 | inline std::string double_to_string(double d) 20 | { return string_printf("%.17g", d); } 21 | 22 | } // namespace util 23 | } // namespace croquis 24 | -------------------------------------------------------------------------------- /src/js/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | amd: true, 4 | browser: true, 5 | 6 | // Let's use an older standard for now - 7 | // I don't think we really need es2021. 8 | es2017: true, 9 | // es2021: true 10 | }, 11 | extends: "eslint:recommended", 12 | parserOptions: { 13 | sourceType: "module", 14 | "ecmaVersion": 2017 // default was 12 15 | }, 16 | rules: { 17 | "no-unused-vars": 0, // Too many warnings. 18 | "no-constant-condition": 0, // while (true) is valid code! Sheesh... 19 | }, 20 | overrides: [ 21 | { 22 | files: ["croquis_loader.js"], 23 | parserOptions: {sourceType: "script"}, 24 | }, 25 | ], 26 | }; 27 | -------------------------------------------------------------------------------- /src/js/.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | package.json 3 | package-lock.json 4 | jsconfig.json 5 | node_modules/ 6 | -------------------------------------------------------------------------------- /src/js/README.md: -------------------------------------------------------------------------------- 1 | Javascript code lives here. 2 | 3 | You can run ESLint by: 4 | 5 | ``` 6 | cd (top dir)/src/js 7 | npm install -g eslint 8 | eslint croquis_fe.js 9 | ``` 10 | 11 | (This will use `.eslintrc.js` which contains minimal rules.) 12 | -------------------------------------------------------------------------------- /src/js/axis_handler.js: -------------------------------------------------------------------------------- 1 | // The axis handler. 2 | 3 | import { assert, INFLIGHT_REQ_EXPIRE_MSEC, sqr } from './util.js'; 4 | 5 | // Helper class used by AxisHandler. 6 | class AxisTick { 7 | // `coord`: in pixels. 8 | constructor(handler, axis, coord, label_str) { 9 | this.handler = handler; 10 | this.axis = axis; 11 | this.coord = coord; 12 | 13 | if (axis == 'x') { 14 | this.axis_cell = this.handler.x_axis; 15 | this.css_field = 'left'; 16 | } 17 | else if (axis == 'y') { 18 | this.axis_cell = this.handler.y_axis; 19 | this.css_field = 'top'; 20 | } 21 | else 22 | assert(null); 23 | 24 | // Add the new tick, label, and grid line. 25 | this.tick = document.createElement('div'); 26 | this.tick.classList.add('cr_tick'); 27 | 28 | this.label = document.createElement('div'); 29 | this.label.classList.add('cr_label'); 30 | this.label.textContent = label_str; 31 | 32 | this.line = document.createElement('div'); 33 | this.line.classList.add(`${axis}_grid`); 34 | 35 | this.update_location(); 36 | 37 | this.axis_cell.appendChild(this.tick); 38 | this.axis_cell.appendChild(this.label); 39 | this.handler.grid.appendChild(this.line); 40 | } 41 | 42 | // Acts as a "destructor" - remove myself from display. 43 | remove() { 44 | this.tick.remove(); 45 | this.label.remove(); 46 | this.line.remove(); 47 | } 48 | 49 | // Update location after panning. 50 | update_location() { 51 | let tile_set = this.handler.tile_handler.tile_set; 52 | 53 | const [offset, limit] = 54 | (this.axis == 'x') ? [tile_set.x_offset, tile_set.width] 55 | : [tile_set.y_offset, tile_set.height]; 56 | const screen_coord = Math.round(offset + this.coord); 57 | 58 | for (let obj of [this.tick, this.label, this.line]) { 59 | if (screen_coord >= 0 && screen_coord < limit) { 60 | obj.style[this.css_field] = screen_coord + 'px'; 61 | obj.style.visibility = 'visible'; 62 | } 63 | else 64 | obj.style.visibility = 'hidden'; 65 | } 66 | } 67 | } 68 | 69 | const REQUEST_NEW_LABELS_DIST_THRESHOLD = 30; 70 | 71 | export class AxisHandler { 72 | constructor(ctxt, tile_handler) { 73 | this.ctxt = ctxt; 74 | this.tile_handler = tile_handler; 75 | 76 | this.x_axis = ctxt.canvas_main.querySelector('.cr_x_axis'); 77 | this.y_axis = ctxt.canvas_main.querySelector('.cr_y_axis'); 78 | this.grid = ctxt.canvas_main.querySelector('.cr_grid'); 79 | 80 | this.ticks = []; 81 | 82 | this.next_seq = 0; // Used for `axis_req` message. 83 | 84 | // For flow control, we remember the number of "axis_req" messages we've 85 | // sent, and don't send more than two messages until we get a response. 86 | // 87 | // The logic is similar to TilehHandler.inflight_reqs (but simpler): see 88 | // there for more discussion. 89 | // 90 | // TODO: Refactor into a single helper class? 91 | this.inflight_reqs = new Map(); 92 | 93 | this.last_seq = -1; 94 | 95 | // x/y offsets used for our last "axis_req" message. 96 | this.last_x_offset = this.last_y_offset = 0; 97 | } 98 | 99 | // Remove all known ticks and re-create them, following new information sent 100 | // from the backend. 101 | // Message type is either 'canvas_config' or 'axis_ticks'. 102 | update(msg_dict) { 103 | if (msg_dict.msg == 'canvas_config') { 104 | // Reset sequence #. 105 | this.last_seq = -1; 106 | this.next_seq = 0; 107 | this.last_x_offset = this.last_y_offset = 0; 108 | } 109 | else if (msg_dict.msg == 'axis_ticks') { 110 | this.inflight_reqs.delete(msg_dict.axis_seq); 111 | 112 | // Check if this is the newest info - otherwise silently ignore. 113 | if (msg_dict.config_id != this.tile_handler.tile_set.config_id || 114 | msg_dict.axis_seq < this.last_seq) 115 | return; 116 | 117 | this.last_seq = msg_dict.axis_seq; 118 | this.last_x_offset = msg_dict.x_offset; 119 | this.last_y_offset = msg_dict.y_offset; 120 | } 121 | 122 | // Remove all existing ticks. 123 | for (let tick of this.ticks) tick.remove(); 124 | this.ticks.length = 0; 125 | 126 | for (let axis of ['x', 'y']) { 127 | for (let [coord, label] of msg_dict.axes[axis]) 128 | this.ticks.push(new AxisTick(this, axis, coord, label)); 129 | } 130 | } 131 | 132 | // Update ticks based on new location. 133 | update_location(zoom_udpated) { 134 | for (let tick of this.ticks) tick.update_location(); 135 | 136 | let tile_set = this.tile_handler.tile_set; 137 | 138 | if (!zoom_udpated) { 139 | // Check if we have drifted far enough to need new labels. 140 | let dist2 = sqr(tile_set.x_offset - this.last_x_offset) + 141 | sqr(tile_set.y_offset - this.last_y_offset); 142 | if (dist2 < sqr(REQUEST_NEW_LABELS_DIST_THRESHOLD)) return; 143 | } 144 | 145 | this.tile_handler.replayer.log('Creating new axis_req message ...'); 146 | 147 | if (this.inflight_reqs.size >= 2) { 148 | // TODO: If `zoom_updated` is true, we still need to send the 149 | // request! 150 | this.tile_handler.replayer.log( 151 | 'Too many in-flight axis requests, bailing out ...'); 152 | return; 153 | } 154 | 155 | const seq = this.next_seq++; 156 | this.inflight_reqs.set(seq, Date.now()); 157 | this.last_x_offset = Math.round(tile_set.x_offset); 158 | this.last_y_offset = Math.round(tile_set.y_offset); 159 | 160 | this.ctxt.send('axis_req', { 161 | config: tile_set.current_canvas_config(), 162 | axis_seq: seq, 163 | }); 164 | } 165 | 166 | // Forget in-flight requests that are too old. 167 | // TODO: Refactor and merge with TileHandler.expire_old_requests() ? 168 | expire_old_requests() { 169 | let deadline = Date.now() - INFLIGHT_REQ_EXPIRE_MSEC; 170 | for (let [seq_no, timestamp] of this.inflight_reqs) { 171 | if (timestamp < deadline) { 172 | this.tile_handler.replayer.log( 173 | `Forgetting old seq #${seq_no} ...`); 174 | this.inflight_reqs.delete(seq_no); 175 | } 176 | } 177 | } 178 | 179 | // Helper function for the crosshair ("nearest point"). 180 | update_crosshair(x, y) { 181 | if (x == null) { 182 | for (let hair of 183 | this.grid.querySelectorAll('.nearest_x, .nearest_y')) 184 | hair.remove(); 185 | return; 186 | } 187 | 188 | let x_hair = document.createElement('div'); 189 | x_hair.classList.add('nearest_x'); 190 | x_hair.style.left = x + 'px'; 191 | this.grid.appendChild(x_hair); 192 | 193 | let y_hair = document.createElement('div'); 194 | y_hair.classList.add('nearest_y'); 195 | y_hair.style.top = y + 'px'; 196 | this.grid.appendChild(y_hair); 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/js/croquis_fe.css: -------------------------------------------------------------------------------- 1 | /* CSS definition for the frontend. */ 2 | 3 | .croquis_nbext .cr_main { 4 | /* margin-left: 25px; */ 5 | /* margin-right: 25px; */ 6 | margin-bottom: 30px; 7 | } 8 | 9 | .croquis_nbext .cr_main .cr_ctrl_panel { 10 | margin-bottom: 10px; 11 | 12 | font-size: 15px; 13 | text-align: right; 14 | color: #950; 15 | } 16 | 17 | .croquis_nbext .cr_main .cr_ctrl_panel span.cr_ctrl_btns { 18 | margin-right: 30px; 19 | } 20 | 21 | .croquis_nbext .cr_main .cr_ctrl_panel input { 22 | margin: 0px 0px 0px 0px; 23 | vertical-align: middle; 24 | } 25 | 26 | .croquis_nbext .cr_main1 { 27 | margin: 0px 25px 30px 25px; 28 | } 29 | 30 | .croquis_nbext .cr_canvas_plus_x_axis { 31 | position: relative; 32 | 33 | /* We need this because cr_tooltip is under this and must be visible even 34 | * when it's partially out of the canvas. 35 | */ 36 | overflow: visible; 37 | } 38 | 39 | .croquis_nbext .cr_x_axis .cr_tick { 40 | background-color: black; 41 | width: 1px; 42 | height: 3px; 43 | 44 | position: absolute; 45 | top: 0px; 46 | } 47 | 48 | .croquis_nbext .cr_x_axis .cr_label { 49 | position: absolute; 50 | top: 5px; 51 | transform: translateX(-50%); 52 | } 53 | 54 | .croquis_nbext .cr_y_axis .cr_tick { 55 | background-color: black; 56 | width: 3px; 57 | height: 1px; 58 | 59 | position: absolute; 60 | right: 0px; 61 | } 62 | 63 | .croquis_nbext .cr_y_axis .cr_label { 64 | position: absolute; 65 | right: 5px; 66 | transform: translateY(-50%); 67 | } 68 | 69 | .croquis_nbext .cr_canvas { 70 | height: 50px; /* Initial height before anything is drawn. */ 71 | } 72 | 73 | .croquis_nbext .cr_inner { 74 | overflow: visible; 75 | 76 | width: 100%; 77 | height: 100%; 78 | 79 | user-select: none; 80 | } 81 | 82 | .croquis_nbext img.cr_tile { 83 | position: absolute; 84 | 85 | max-width: 256px; 86 | max-height: 256px; 87 | 88 | margin: 0px; 89 | border-style: none; 90 | /* border: thin solid red; */ 91 | 92 | user-select: none; 93 | } 94 | 95 | /* Contains highlight tiles. */ 96 | .croquis_nbext .cr_foreground { 97 | visibility: hidden; /* Hidden by default. */ 98 | background-color: rgba(255, 255, 255, 0.8); 99 | 100 | /* Must cover the whole area. */ 101 | width: 100%; 102 | height: 100%; 103 | 104 | user-select: none; 105 | } 106 | 107 | /* Contains grid lines. */ 108 | .croquis_nbext .cr_grid { 109 | width: 100%; 110 | height: 100%; 111 | user-select: none; 112 | } 113 | 114 | .croquis_nbext .cr_grid .x_grid { 115 | position: absolute; 116 | width: 1px; 117 | height: 100%; 118 | border-left: 1px dashed rgba(0, 45, 209, 0.5); 119 | } 120 | 121 | .croquis_nbext .cr_grid .y_grid { 122 | position: absolute; 123 | width: 100%; 124 | height: 1px; 125 | border-top: 1px dashed rgba(0, 45, 209, 0.5); 126 | } 127 | 128 | /* "Crosshair" for the nearest point. */ 129 | .croquis_nbext .cr_grid .nearest_x { 130 | position: absolute; 131 | width: 1px; 132 | height: 100%; 133 | border-left: 1px dashed rgba(196, 0, 95, 0.8); 134 | } 135 | 136 | .croquis_nbext .cr_grid .nearest_y { 137 | position: absolute; 138 | width: 100%; 139 | height: 1px; 140 | border-top: 1px dashed rgba(196, 0, 95, 0.8); 141 | } 142 | 143 | /* Used for select-to-zoom. */ 144 | .croquis_nbext .cr_select_area { 145 | visibility: hidden; /* Hidden by default. */ 146 | background-color: rgba(66, 57, 42, 0.7); 147 | border-style: dotted; 148 | border-width: 1px; 149 | border-color: rgba(0, 0, 0, 0.8); 150 | 151 | user-select: none; 152 | } 153 | 154 | .croquis_nbext .cr_tooltip { 155 | position: absolute; 156 | visibility: hidden; /* Hidden by default. */ 157 | background-color: #eec; 158 | border-style: solid; 159 | border-width: 2px; 160 | /* border-color: supplied by the code; follows item color. */ 161 | 162 | padding: 5px; 163 | 164 | max-width: 200px; 165 | width: fit-content; 166 | white-space: pre-wrap; 167 | } 168 | 169 | .croquis_nbext .cr_progressbar { 170 | height: 40px; 171 | 172 | /* Initially hidden: only shows up if the backend takes too long. */ 173 | visibility: hidden; 174 | } 175 | 176 | .croquis_nbext .cr_legend { 177 | flex: 0.1 1 160px; 178 | min-width: 120px; 179 | 180 | display: flex; 181 | flex-direction: column; 182 | 183 | padding: 20px 0px 0px 20px; 184 | 185 | font-size: 12px; 186 | line-height: normal; 187 | } 188 | 189 | /* https://stackoverflow.com/a/63890844 */ 190 | .croquis_nbext .cr_legend li + li { 191 | margin-top: 3px; 192 | } 193 | 194 | .croquis_nbext .cr_legend .cr_searchbox input { 195 | width: 100%; 196 | font-family: monospace; 197 | } 198 | 199 | .croquis_nbext .cr_legend .cr_search_ctrl { 200 | text-align: left; 201 | font-size: 90%; 202 | 203 | margin-top: 15px; 204 | 205 | position: relative; 206 | } 207 | 208 | .croquis_nbext .cr_legend .cr_search_ctrl ul.cr_btn_popup { 209 | visibility: hidden; 210 | 211 | background-color: #ffe; 212 | border: 1px solid #063; 213 | padding: 10px; 214 | 215 | list-style-type: none; 216 | 217 | position: absolute; 218 | top: 15px; 219 | left: 15px; 220 | } 221 | 222 | .croquis_nbext .cr_legend .cr_search_ctrl input[type="checkbox"] { 223 | margin: 0px; 224 | vertical-align: middle; 225 | } 226 | 227 | .croquis_nbext .cr_legend .cr_search_stat { 228 | text-align: left; 229 | font-size: 90%; 230 | color: #852400; 231 | } 232 | 233 | .croquis_nbext .cr_legend ul.cr_search_result { 234 | flex: 1 0 100px; 235 | overflow: auto; 236 | 237 | list-style-type: none; 238 | margin: 15px 0px 15px 0px; 239 | padding: 0px; 240 | } 241 | 242 | .croquis_nbext .cr_legend ul.cr_search_result li { 243 | text-align: left; 244 | font-family: monospace; 245 | white-space: nowrap; 246 | } 247 | 248 | /* When highlighted, e.g., by hovering over the line on canvas. */ 249 | .croquis_nbext .cr_legend ul.cr_search_result li.highlighted { 250 | background-color: #dff; 251 | font-weight: bold; 252 | } 253 | 254 | .croquis_nbext .cr_legend ul.cr_search_result li:hover { 255 | background-color: #dff; 256 | } 257 | 258 | .croquis_nbext .cr_legend ul.cr_search_result li input[type="checkbox"] { 259 | margin: 0px 0.5em 0px 0px; /* only 0.5em on right */ 260 | vertical-align: middle; 261 | } 262 | 263 | .croquis_nbext .cr_legend ul.cr_search_result li svg { 264 | vertical-align: middle; 265 | } 266 | 267 | .croquis_nbext .cr_legend .cr_info { 268 | flex: 0 0 70px; 269 | 270 | background-color: #dff; /* test */ 271 | } 272 | 273 | /******************************* 274 | * Debugging related elements. * 275 | *******************************/ 276 | 277 | .croquis_nbext .cr_dbg_btns { 278 | background-color: #cfd; 279 | color: #950; 280 | } 281 | 282 | .croquis_nbext .cr_dbgpt1 { 283 | color: red; 284 | font-size: 10px; 285 | position: absolute; 286 | transform: translateX(-50%) translateY(-50%); 287 | z-index: 10; 288 | } 289 | 290 | .croquis_nbext .cr_dbgpt2 { 291 | color: black; 292 | font-size: 10px; 293 | position: absolute; 294 | transform: translateX(-50%) translateY(-50%); 295 | z-index: 10; 296 | } 297 | 298 | .croquis_nbext .cr_dbg_status { 299 | background-color: #dfe; 300 | position: absolute; 301 | text-align: center; 302 | z-index: 5; 303 | 304 | left: 50px; 305 | top: 50px; 306 | width: 400px; 307 | } 308 | 309 | .croquis_nbext .cr_dbglog { 310 | background-color: #dfe; 311 | } 312 | -------------------------------------------------------------------------------- /src/js/croquis_loader.js: -------------------------------------------------------------------------------- 1 | // The entry point for the croquis frontend module. 2 | // 3 | // cf. https://mindtrove.info/4-ways-to-extend-jupyter-notebook/#nb-extensions 4 | // https://jupyter-notebook.readthedocs.io/en/stable/extending/frontend_extensions.html 5 | // 6 | // At first I tried bundling all the js code via webpack, but for some reason, 7 | // requirejs.toUrl() would stop working after bundling via webpack, so I 8 | // couldn't import CSS. (In other libraries such as ipympl, the CSS file itself 9 | // is also bundled together and dynamically "loaded" via webpack's 10 | // `style-loader`, but that seemed like too much magic.) 11 | // 12 | // So this is a compromise: this "loader" module is copied verbatim into 13 | // notebook's "nbextension" directory, so that we can use all the features of 14 | // requirejs without issues. The rest of the code is then bundled by webpack to 15 | // create `croquis_fe.js`. 16 | // 17 | // NOTE: Contrary to the name, this module doesn't actually *load* the main 18 | // module: it's loaded together by the code emitted by `display.py`. 19 | 20 | define([ 21 | 'module', // myself 22 | 'jquery', 23 | 'require', 24 | 'base/js/namespace' 25 | ], function(module, $, requirejs, Jupyter) { 26 | 27 | //------------------------------------------------ 28 | 29 | 'use strict'; 30 | 31 | let css_loaded = null; // Promise object. 32 | let comm = null; // Websocket communication channel provided by Jupyter. 33 | let comm_ready_resolve = null; 34 | let comm_ready = new Promise((resolve, reject) => { 35 | comm_ready_resolve = resolve; // Will be called when BE is ready. 36 | }); 37 | 38 | // Copied from `BE_uuid` of comm.py: used to detect BE restart. 39 | let last_BE_uuid = null; 40 | 41 | let ctxt_map = {}; // Map of contexts by canvas IDs. 42 | 43 | // Current time as string HH:MM:SS.mmm, for debug logging. 44 | function get_timestamp_str() { 45 | let T = new Date(); 46 | let fmt = (x, sz) => ('000' + x).slice(-sz); // Seriously, no printf?? 47 | return fmt(T.getHours(), 2) + ':' + fmt(T.getMinutes(), 2) + ':' + 48 | fmt(T.getSeconds(), 2) + '.' + fmt(T.getMilliseconds(), 3); 49 | } 50 | 51 | // Add a module-level resize handler. 52 | window.addEventListener('resize', resize_handler); 53 | 54 | // It's possible that the cell's output width is changed even without window 55 | // resizing. For example, a cell may execute something like: 56 | // from IPython.core.display import display, HTML 57 | // display(HTML("")) 58 | // 59 | // Apparently it's very hard to add a generic "resize handler" for individual 60 | // elements. (There's a library for it [https://github.com/marcj/css-element-queries] 61 | // but I don't want to add too much dependency.) So, let's just periodically 62 | // check if cell size changed. It doesn't need to be fast as such an event 63 | // should be pretty infrequent (for now). 64 | // 65 | // TODO: I need to clean up this handler once the module is unloaded? 66 | let resize_timer = window.setInterval(resize_handler, 500 /* ms */); 67 | 68 | function load_ipython_extension() { 69 | console.log('croquis_fe module loaded by Jupyter:', module.id); 70 | load_css(false /* !force_reload */); 71 | } 72 | 73 | // We need a separate function from load_ipython_extension() because during 74 | // development we want to be able to reload this module (triggered by Python 75 | // class `DisplayObj` via calling `require.undef`), but Jupyter only calls 76 | // load_ipython_extension() for the very first invocation. 77 | function init(force_reload, BE_uuid) { 78 | console.log(`${get_timestamp_str()} croquis_fe.init() called: ` + 79 | `force_reload=${force_reload} BE_uuid=${BE_uuid}`); 80 | // console.log('comm = ', comm); 81 | load_css(force_reload); 82 | if (BE_uuid != last_BE_uuid) { 83 | console.log(`${get_timestamp_str()} BE_uuid changed from ` + 84 | `${last_BE_uuid} to ${BE_uuid}: re-opening comm ...`); 85 | last_BE_uuid = BE_uuid; 86 | // See: https://jupyter-notebook.readthedocs.io/en/stable/comms.html 87 | comm = Jupyter.notebook.kernel.comm_manager.new_comm( 88 | 'croquis', {'msg': 'FE_loaded'}) 89 | comm.on_msg(msg_dispatcher); 90 | } 91 | } 92 | 93 | // module.id should be `nbextensions/croquis_loader_dev` (for dev environment) 94 | // or `nbextensions/croquis_fe/croquis_loader` (when installed via package). 95 | // 96 | // (Yeah, it would've been nice if they are at the same level, but 97 | // `notebook.nbextensions.install_nbextension` does not support subdirectories, 98 | // while it's probably a bad idea to install our js files at the top level 99 | // `nbextensions/` when we build a package.) 100 | // 101 | // In Linux, assuming Anaconda, the actual location would be: 102 | // dev: ~/.local/share/jupyter/nbextensions/croquis_loader_dev.js 103 | // installed: $CONDA_PREFIX/share/jupyter/nbextensions/croquis_fe/croquis_loader.js 104 | const is_dev = (module.id.search('croquis_loader_dev') != -1); 105 | 106 | // Load CSS if necessary. 107 | // cf. https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/internals.html 108 | function load_css(force_reload) { 109 | css_loaded = new Promise((resolve, reject) => { 110 | let existing_css = document.querySelector('head .croquis_fe_css'); 111 | 112 | if (existing_css != null) { 113 | if (!force_reload) { 114 | console.log('croquis_fe: CSS is already loaded.'); 115 | resolve(); 116 | return; 117 | } 118 | console.log('croquis_fe: reloading CSS ..'); 119 | existing_css.remove(); 120 | } 121 | 122 | const css_name = (is_dev) ? './croquis_fe_dev.css' : './croquis_fe.css'; 123 | let css_url = requirejs.toUrl(css_name); 124 | console.log(`Loading CSS (force_reload=${force_reload}): ${css_url}`); 125 | 126 | let link = document.createElement('link'); 127 | link.rel = 'stylesheet'; 128 | link.type = 'text/css'; 129 | link.classList.add('croquis_fe_css'); 130 | link.href = css_url; 131 | link.onload = function() { 132 | console.log('CSS loaded !!!'); 133 | resolve(); 134 | }; 135 | link.onerror = function() { 136 | // At least on Chrome, this seems to fire only when the CSS 137 | // link itself is unavailable. 138 | console.log('CSS load failure !!!'); 139 | reject(); 140 | }; 141 | 142 | document.querySelector('head').appendChild(link); 143 | }); 144 | } 145 | 146 | // Handles websocket message via Jupyter's channel. 147 | function msg_dispatcher(msg) { 148 | // console.log('Received comm: ', msg); 149 | let data = msg.content.data; 150 | 151 | if (data.msg == 'BE_ready') { 152 | console.log('Backend is ready!'); 153 | if (comm_ready_resolve != null) { 154 | comm_ready_resolve(); 155 | comm_ready_resolve = null; 156 | } 157 | return; 158 | } 159 | 160 | let canvas_id = msg.content.data.canvas_id; 161 | if (!(canvas_id in ctxt_map)) { 162 | console.log('Unknown canvas_id: ', canvas_id); 163 | return; 164 | } 165 | 166 | // If exists, msg.buffers holds binary data (e.g., PNG images). 167 | ctxt_map[canvas_id].msg_handler(msg.content.data, msg.buffers || []); 168 | } 169 | 170 | // Called by `Ctxt` to send message to BE. 171 | function send_msg(data) { 172 | comm_ready.then(() => { comm.send(data) }); 173 | } 174 | 175 | function resize_handler() { 176 | for (let canvas_id in ctxt_map) ctxt_map[canvas_id].resize_handler(); 177 | } 178 | 179 | function create_ctxt(main_module, canvas_id) { 180 | let env = { 181 | $: $, 182 | ctxt_map: ctxt_map, 183 | css_loaded: css_loaded, 184 | send_msg: send_msg, 185 | }; 186 | return new main_module.Ctxt(env, canvas_id); 187 | } 188 | 189 | return { 190 | load_ipython_extension: load_ipython_extension, 191 | init: init, 192 | create_ctxt: create_ctxt, 193 | }; 194 | 195 | //------------------------------------------------ 196 | 197 | }); 198 | -------------------------------------------------------------------------------- /src/js/css_helper.js: -------------------------------------------------------------------------------- 1 | // Helper for handling CSS properties. 2 | // 3 | // In theory, it might be a nice idea to "separate content (HTML) from 4 | // presentation (CSS)", but in reality, it doesn't really work for us. 5 | // The fact that, say, "div.cr_x_axis" has only "border-top" is not a 6 | // presentation detail; it's an essential part of the *content* because the 7 | // "border-top" is the freaking X axis! 8 | // 9 | // In other words, I found that a separate CSS file actually made it *harder* to 10 | // understand code. Since we're dynamically generating the HTML elements 11 | // anyway, let's also apply style here dynamically, so that it's easier to 12 | // understand what's going on. 13 | 14 | import { assert } from './util.js'; 15 | 16 | // Helper function. 17 | function apply_over_children(elem, selector, f) { 18 | let children = elem.querySelectorAll(selector); 19 | assert(children.length > 0, `No element matching ${selector}`); 20 | for (let child of children) f(child); 21 | } 22 | 23 | // Helper function for handling very simple "CSS-style" definitions. 24 | // I.e., "width: 50px; height: 30px;" is parsed into 25 | // [['width', '50px'], ['height', '30px']]. 26 | // 27 | // Will probably not work for generic cases. 28 | function parsePseudoCSS(s) { 29 | let keyvals = []; 30 | 31 | for (let kv of s.split(';')) { 32 | kv = kv.trim(); 33 | if (kv == '') continue; 34 | let match = kv.match(/([^ ]+)\s*:\s*([^ ].*)/); 35 | assert(match != null, `Cannot parse "CSS" string ${kv}`); 36 | keyvals.push([match[1], match[2]]); 37 | } 38 | 39 | return keyvals; 40 | } 41 | 42 | // Generic CSS update function for an element. 43 | export function apply_css(elem, property, value) { 44 | elem.style[property] = value; 45 | } 46 | 47 | // Generic CSS update function for descendants of an element. 48 | // `settings` is an array of pairs (selector, "property: value; (...)"). 49 | export function apply_css_tree(elem, settings) { 50 | for (let [selector, css] of settings) { 51 | let kv = parsePseudoCSS(css); 52 | apply_over_children(elem, selector, (child) => { 53 | for (let [k, v] of kv) child.style[k] = v; 54 | }); 55 | } 56 | } 57 | 58 | export function disable_drag(elem, selectors) { 59 | for (let selector of selectors) { 60 | apply_over_children(elem, selector, (child) => { 61 | child.draggable = false; 62 | }); 63 | } 64 | } 65 | 66 | // Set `elem` as a flex container, and apply "flex" parameters (for sizing) to 67 | // each child using `flex_settings`, which is an array of pairs (selector, flex 68 | // property). 69 | export function apply_flex(elem, dir, flex_settings) { 70 | assert(dir == 'row' || dir == 'column', `Invalid dir ${dir}`); 71 | 72 | elem.style.display = 'flex'; 73 | elem.style['flex-direction'] = dir; 74 | 75 | for (let [selector, flex] of flex_settings) { 76 | apply_over_children(elem, selector, (child) => { 77 | child.style.flex = flex; 78 | }); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/js/ctxt.js: -------------------------------------------------------------------------------- 1 | // The "Context" that is associated with each figure. 2 | 3 | import { Tile } from './tile.js' 4 | import { TileHandler } from './tile_handler.js'; 5 | import { PROGRESSBAR_TIMEOUT } from './util.js'; 6 | 7 | // Current setup: 8 | // - div #croquis_nbext : outermost container 9 | // - div #{{canvas_id}}-btns : buttons for debugging 10 | // - div #{{canvas_id}} .cr_main : the whole area including axes 11 | // - div .cr_ctrl_panel : "Control panel" at the top. 12 | // - span .cr_ctrl_btns 13 | // - button .cr_home_btn : home (reset) button 14 | // - button .cr_zoom_in_btn : zoom in 15 | // - button .cr_zoom_out_btn : zoom out 16 | // - input #{{canvas_id}}-zoom : "drag mouse to zoom" 17 | // - input #{{canvas_id}}-pan : "drag mouse to pan" 18 | // - div .cr_main1 19 | // - div .cr_y_axis : y axis 20 | // - div cr_canvas_plus_x_axis 21 | // - div .cr_canvas : the main canvas --> this.canvas 22 | // - div .cr_progressbar : "please wait" message 23 | // - div .cr_inner : regular (non-highlight) tiles go here 24 | // - div .cr_foreground : highlight tiles go here 25 | // - div .cr_grid : coordinate grids 26 | // - div .cr_select_area : shows selected area when dragging 27 | // - div .cr_x_axis : x axis 28 | // - div .cr_tooltip : tooltip (activates when mouse stops) 29 | // - div .cr_legend : legends and items selection 30 | // - div .cr_searchbox : search box 31 | // - div .cr_search_ctrl 32 | // - input .cr_XXX : search control buttons 33 | // - button .cr_more : opens the pop-up box 34 | // - ul .cr_btn_popup : pop-up box 35 | // - li 36 | // - a .cr_XXX : "buttons" for selection updates 37 | // - div .cr_search_stat : "Showing 200 of 3,456 results" 38 | // - ul .cr_search_result : labels matching the search pattern 39 | // // Currently commented out: 40 | // // - div .cr_info : info about what's under the mouse cursor 41 | // - div id={{canvs_id}}-log : debug logs --> this.log_area 42 | // 43 | // See also display.py for HTML structure. 44 | export class Ctxt { 45 | constructor(env, canvas_id) { 46 | this.env = env; 47 | this.canvas_id = canvas_id; 48 | this.canvas_main = document.querySelector('#' + canvas_id); 49 | 50 | // Hmm looks like we can't do this with vanilla JS ... 51 | // TODO: Check if there's some hook inside Jupyter? 52 | env.$('#' + canvas_id).on('remove', () => { this.cleanup_handler(); }); 53 | 54 | // TileHandler constructor also builds the inner HTML structure. 55 | let parent_elem = document.querySelector(`#${canvas_id} .cr_main1`); 56 | parent_elem.innerHTML = ''; 57 | this.tile_handler = new TileHandler(this, parent_elem); 58 | 59 | this.log_area = env.$('#' + canvas_id + '-log'); 60 | if (this.log_area) { 61 | // this.log_area[0].style.backgroundColor = "#dfe"; 62 | this.dbglog('Cell ID = ', canvas_id); 63 | } 64 | 65 | // Pedantic: must be after `this.tile_handler` is initialized, because 66 | // after this this.resize_handler() may be called, which uses 67 | // `tile_handler`. 68 | env.ctxt_map[canvas_id] = this; 69 | 70 | // Get the current window size and send it as part of the init 71 | // request. 72 | env.css_loaded.then(() => { 73 | this.tile_handler.tile_set.reset_canvas(); 74 | this.tile_handler.search_handler(null); 75 | 76 | // Prepare the progress indicator to fire if BE takes too long. 77 | setTimeout(() => { 78 | let bar = this.get_canvas().querySelector('.cr_progressbar'); 79 | if (bar) bar.style.visibility = 'visible'; 80 | }, PROGRESSBAR_TIMEOUT); 81 | 82 | // console.log( 83 | // 'bounding rect = ', this.canvas[0].getBoundingClientRect()); 84 | }) 85 | .catch(error => { this.fail('Failed to load CSS: ' + error); }); 86 | } 87 | 88 | // Helper function to report failure. 89 | fail(err) { 90 | console.log('Failure: ', err) 91 | this.canvas_main.textContent = 92 | 'Error processing request ' + 93 | '(see developer console for details): ' + err; 94 | } 95 | 96 | // Called when the window size *may* have changed: see the discussion on 97 | // window_setInterval() inside croquis_loader.js. 98 | resize_handler() { 99 | this.tile_handler.tile_set.resize_canvas(); 100 | } 101 | 102 | // Cleanup handler: tell the server that this canvas is gone. 103 | cleanup_handler() { 104 | this.send('cell_fini'); 105 | delete this.env.ctxt_map[this.canvas_id]; 106 | } 107 | 108 | // Helper function to send a message. 109 | send(msg, more_data) { 110 | let data = more_data || {}; 111 | data.msg = msg; 112 | data.canvas_id = this.canvas_id; 113 | // console.log(`${get_timestamp_str()} sending msg: `, msg, more_data); 114 | this.env.send_msg(data); 115 | } 116 | 117 | // Handler for BE message. 118 | msg_handler(msg_dict, attachments) { 119 | // if (attachments.length) { 120 | // this.dbglog('BE message with attachment: ', msg_dict, 121 | // 'length = ', attachments.map(x => x.byteLength)); 122 | // } 123 | // else { 124 | // this.dbglog('BE message received: ', msg_dict); 125 | // } 126 | 127 | if (msg_dict.msg == 'canvas_config') { 128 | this.tile_handler.register_canvas_config(msg_dict); 129 | } 130 | else if (msg_dict.msg == 'axis_ticks') { 131 | this.tile_handler.axis_handler.update(msg_dict); 132 | } 133 | else if (msg_dict.msg == 'tile') { 134 | let tile = new Tile(msg_dict, attachments); 135 | let seqs = msg_dict.seqs.split(':').map(x => parseInt(x)); 136 | // console.log(`Received tile: ${tile.key}`); 137 | this.tile_handler.register_tile(tile, seqs); 138 | } 139 | else if (msg_dict.msg == 'pt') { 140 | this.tile_handler.nearest_pts.insert(msg_dict); 141 | this.tile_handler.recompute_highlight(); 142 | } 143 | else if (msg_dict.msg == 'labels') { 144 | this.tile_handler.update_search_result(msg_dict); 145 | } 146 | else { 147 | console.log('Unknown message', msg_dict) 148 | } 149 | } 150 | 151 | // Helper function for debug logging. 152 | dbglog(...args) { 153 | if (this.log_area) { 154 | const s = args.map( 155 | e => ((typeof(e) == 'object') ? JSON.stringify(e) : e) 156 | ).join(' '); 157 | this.log_area.append(document.createTextNode(s), "
"); 158 | } 159 | } 160 | 161 | // Internal helper function, because now `canvas` is dynamically generated. 162 | // TODO: Doesn't belong here, refactor! 163 | get_canvas() { 164 | return document.querySelector(`#${this.canvas_id} .cr_canvas`); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/js/label.js: -------------------------------------------------------------------------------- 1 | // A label shown in the legend (search result) section. 2 | 3 | import { ITEM_ID_SENTINEL } from './util.js'; 4 | 5 | const LABEL_SVG_WIDTH = 40; // px 6 | const LABEL_SVG_HEIGHT = 8; // px 7 | const LABEL_MARKER_SIZE_MAX = 8; // px 8 | const LABEL_LINE_WIDTH_MAX = 5; // px 9 | 10 | export class Label { 11 | constructor(item_id, selected, label, style, cb) { 12 | this.item_id = item_id; 13 | this.selected = selected; 14 | this.label = label; 15 | this.style = style; 16 | 17 | if (item_id == ITEM_ID_SENTINEL) { 18 | // This must be `null`, because its value is used as an argument to 19 | // search_result_area.insertBefore() inside TileHandler. 20 | this.elem = null; 21 | return; 22 | } 23 | 24 | this.elem = document.createElement('li'); 25 | 26 | this.checkbox = document.createElement('input'); 27 | this.checkbox.type = 'checkbox'; 28 | this.checkbox.checked = this.selected; 29 | this.elem.appendChild(this.checkbox); 30 | 31 | this.elem.appendChild(this.create_line_svg()); 32 | 33 | this.elem.appendChild(document.createTextNode(this.label)); 34 | 35 | this.highlighted = false; 36 | 37 | let cb2 = (ev) => cb(this, ev); 38 | this.elem.addEventListener('mouseenter', cb2); 39 | this.elem.addEventListener('mousemove', cb2); 40 | this.elem.addEventListener('mouseleave', cb2); 41 | } 42 | 43 | // `selected` is boolean. 44 | update_selected(selected) { 45 | if (this.item_id == ITEM_ID_SENTINEL) return; 46 | this.selected = selected; 47 | this.checkbox.checked = selected; 48 | if (selected == false) this.update_highlight(false); 49 | } 50 | 51 | // `highlighted` is boolean. 52 | update_highlight(highlighted) { 53 | if (this.item_id == ITEM_ID_SENTINEL) return; 54 | 55 | if (highlighted && !this.highlighted) { 56 | this.highlighted = true; 57 | this.elem.classList.add('highlighted'); 58 | } 59 | else if (!highlighted && this.highlighted) { 60 | this.highlighted = false; 61 | this.elem.classList.remove('highlighted'); 62 | } 63 | } 64 | 65 | clear_highlight() { 66 | if (this.item_id == ITEM_ID_SENTINEL) return; 67 | } 68 | 69 | // Create a simple line/marker image to show in the legend. 70 | create_line_svg() { 71 | const [w, h] = [LABEL_SVG_WIDTH, LABEL_SVG_HEIGHT]; 72 | const [w2, h2] = [w / 2, h / 2]; 73 | 74 | let [color, marker_size, line_width] = this.style.split(':'); 75 | marker_size = Math.min(marker_size, LABEL_MARKER_SIZE_MAX); 76 | line_width = Math.min(line_width, LABEL_LINE_WIDTH_MAX); 77 | 78 | let svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); 79 | svg.setAttribute('viewBox', `0 0 ${w} ${h}`); 80 | svg.setAttribute('width', `${w}px`); 81 | svg.setAttribute('height', `${h}px`); 82 | 83 | let path = 84 | document.createElementNS("http://www.w3.org/2000/svg", 'path'); 85 | path.setAttribute('stroke', '#' + color); 86 | path.setAttribute('stroke-width', line_width); 87 | path.setAttribute('d', `M 0,${h2} L ${w},${h2}`); 88 | svg.appendChild(path); 89 | 90 | let cir = 91 | document.createElementNS("http://www.w3.org/2000/svg", 'circle'); 92 | cir.setAttribute('cx', w2); 93 | cir.setAttribute('cy', h2); 94 | cir.setAttribute('r', marker_size / 2); 95 | cir.setAttribute('fill', '#' + color); 96 | svg.appendChild(cir); 97 | 98 | return svg; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/js/main.js: -------------------------------------------------------------------------------- 1 | // The main entry point of the croquis module. 2 | 3 | export function load_ipython_extension() {} 4 | export { Ctxt } from './ctxt.js'; 5 | -------------------------------------------------------------------------------- /src/js/tile.js: -------------------------------------------------------------------------------- 1 | // The tile itself. 2 | 3 | import { TILE_SIZE } from './util.js'; 4 | 5 | // A tile key does not contain `sm_version` because we want to use the latest 6 | // available version even if the "correct" (latest requested) version is not 7 | // available yet. 8 | export function tile_key(config_id, zoom_level, row, col, item_id = null) { 9 | if (item_id == null) 10 | return `${config_id}:${zoom_level}:${row}:${col}`; 11 | else 12 | return `${config_id}:${zoom_level}:${row}:${col}:${item_id}`; 13 | } 14 | 15 | export class Tile { 16 | constructor(msg_dict, attachments) { 17 | const is_hover = 'item_id' in msg_dict; 18 | 19 | this.sm_version = msg_dict.sm_version; // Selection map version. 20 | this.config_id = msg_dict.config_id; 21 | this.zoom_level = msg_dict.zoom_level; 22 | this.row = msg_dict.row; 23 | this.col = msg_dict.col; 24 | if (is_hover) { 25 | this.item_id = msg_dict.item_id; 26 | this.label = msg_dict.label; 27 | this.style = msg_dict.style; 28 | } 29 | else { 30 | this.item_id = this.label = this.style = null; 31 | } 32 | 33 | this.key = tile_key(this.config_id, this.zoom_level, 34 | this.row, this.col, this.item_id); 35 | 36 | const png_data = attachments[0]; 37 | this.elem = new Image(TILE_SIZE, TILE_SIZE); 38 | this.elem.classList.add('cr_tile'); 39 | this.elem.setAttribute('draggable', false); 40 | this.elem.src = URL.createObjectURL( 41 | new Blob([png_data], {type: 'image/png'})); 42 | 43 | // Hovermap data is available only if this is *not* a hover (i.e., 44 | // highlight) image. 45 | if (!is_hover) 46 | this.hovermap = attachments[1]; // Type DataView. 47 | } 48 | 49 | is_hover() { 50 | return this.item_id != null; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/js/util.js: -------------------------------------------------------------------------------- 1 | // Miscellaneous constants and utility functions. 2 | 3 | export const PROGRESSBAR_TIMEOUT = 500; // ms 4 | export const TILE_SIZE = 256; 5 | export const ZOOM_FACTOR = 1.5; // Must match constants.h. 6 | export const ITEM_ID_SENTINEL = Number.MAX_SAFE_INTEGER; // Used by Label. 7 | export const INFLIGHT_REQ_EXPIRE_MSEC = 5000; 8 | 9 | // TODO: Any way to generate a better assert message? 10 | export function assert(x, msg) { 11 | if (!x) { 12 | throw (msg) ? msg : 'Should not happen!'; 13 | } 14 | } 15 | 16 | export function sqr(x) { return x * x; } 17 | 18 | // Current time as string HH:MM:SS.mmm, for debug logging. 19 | export function get_timestamp_str() { 20 | let T = new Date(); 21 | let fmt = (x, sz) => ('000' + x).slice(-sz); // Seriously, no printf?? 22 | return fmt(T.getHours(), 2) + ':' + fmt(T.getMinutes(), 2) + ':' + 23 | fmt(T.getSeconds(), 2) + '.' + fmt(T.getMilliseconds(), 3); 24 | } 25 | 26 | // Helper function to hide/unhide stuff by changing "display" attribute. 27 | // (We assume that CSS did not set "display: none": in that case unhide() will 28 | // not work.) 29 | export function hide(elem) { elem.style.display = 'none'; } 30 | export function unhide(elem) { elem.style.display = null; } 31 | 32 | // Utility class for managing a popup box that closes itself when the user 33 | // clicks somewhere else. 34 | // cf. https://stackoverflow.com/a/3028037 35 | export class PopupBox { 36 | constructor(target) { 37 | this.target = target; 38 | this.target.style.visibility = 'hidden'; 39 | this.listener = null; 40 | } 41 | 42 | show() { 43 | this.target.style.visibility = 'visible'; 44 | if (this.listener != null) 45 | document.removeEventListener('click', this.listener); 46 | 47 | this.listener = (ev) => { 48 | if (!this.target.contains(ev.target)) this.hide(); 49 | }; 50 | setTimeout(() => document.addEventListener('click', this.listener), 0); 51 | } 52 | 53 | hide() { 54 | this.target.style.visibility = 'hidden'; 55 | if (this.listener != null) { 56 | document.removeEventListener('click', this.listener); 57 | this.listener = null; 58 | } 59 | } 60 | } 61 | 62 | // Utility class for LRU cache. 63 | export class LRUCache { 64 | // `should_replace` is a callback function - see insert() below. 65 | constructor(maxsize, should_replace) { 66 | this.maxsize = maxsize; 67 | this.should_replace = should_replace; 68 | this.d = new Map(); 69 | } 70 | 71 | clear() { this.d.clear(); } 72 | delete(key) { return this.d.delete(key); } 73 | get(key) { return this.d.get(key); } 74 | has(key) { return this.d.has(key); } 75 | size() { return this.d.size; } 76 | 77 | // Insert key/value pair: if the same key already exists, we call 78 | // should_replace(old item, new item), and replaces the old item iff it 79 | // returns true. 80 | insert(key, value) { 81 | let old = this.get(key); 82 | if (old != undefined && !this.should_replace(this.get(key), value)) 83 | return; 84 | 85 | this.d.delete(key); 86 | while (this.d.size >= this.maxsize) 87 | this.d.delete(this.d.keys().next().value); 88 | this.d.set(key, value); 89 | } 90 | 91 | pop(key) { 92 | let v = this.d.get(key); 93 | if (v == undefined) return null; 94 | this.d.delete(key); 95 | return v; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/js/webpack.config.js: -------------------------------------------------------------------------------- 1 | // Webpack config file. 2 | 3 | module.exports = { 4 | // Let's just turn off the minimizer here, by always using the "development" 5 | // mode: not sure why, but with minimization enabled here, the resulting 6 | // source map does not work for browsers (Chrome or Firefox), making 7 | // debugging pretty much impossible. 8 | // 9 | // So, instead we always create `croquis_fe_dev.js` here, without 10 | // minimization. When we build the package, we call "terser" ourselves 11 | // after webpack, which creates `croquis_fe.js` (with the correct source 12 | // map) which is then included in the final package: see CMakeLists.txt for 13 | // details. 14 | mode: 'development', 15 | 16 | entry: './main.js', 17 | output: { 18 | filename: './croquis_fe_dev.js', 19 | libraryTarget: 'amd', 20 | clean: true, 21 | }, 22 | resolve: { 23 | // Disable default dependency resolution: we're not including *any* 24 | // external libraries into the bundle. 25 | modules: [], 26 | }, 27 | devtool: 'source-map', 28 | stats: 'errors-only', 29 | }; 30 | -------------------------------------------------------------------------------- /src/ui_tests/.gitignore: -------------------------------------------------------------------------------- 1 | .test_workdir 2 | -------------------------------------------------------------------------------- /src/ui_tests/run_all_tests.py: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "exec" "python3" "-u" "$0" "$@" 3 | # 4 | # Runs all browser tests under this directory. 5 | 6 | import argparse 7 | import logging 8 | import os 9 | import sys 10 | 11 | from playwright.sync_api import sync_playwright 12 | 13 | curdir = os.path.dirname(os.path.realpath(sys.argv[0])) 14 | sys.path.insert(0, '%s/..' % curdir) 15 | sys.path.insert(0, curdir) 16 | 17 | from test_util.jupyter_launcher import JupyterLauncher 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | arg_parser = argparse.ArgumentParser() 22 | 23 | arg_parser.add_argument( 24 | '--verbose', action='store_true') 25 | 26 | # Arguments for playwright. 27 | # TODO: Test on Mac! 28 | arg_parser.add_argument( 29 | '--browser', choices=['chromium', 'firefox', 'webkit'], default='chromium', 30 | help='Browser type supported by playwright.') 31 | arg_parser.add_argument('--headless', action='store_true') 32 | arg_parser.add_argument('--slow_mo', type=int) 33 | 34 | # More arguments. 35 | arg_parser.add_argument('--timeout', type=float, default=2.5) 36 | 37 | cmd_args = arg_parser.parse_args() 38 | 39 | with JupyterLauncher(cmd_args) as launcher, sync_playwright() as p: 40 | kwargs = {'headless': cmd_args.headless} 41 | if cmd_args.slow_mo is not None: 42 | kwargs['slow_mo'] = cmd_args.slow_mo 43 | 44 | # Open the first page URL (containing the auth token). 45 | browser = getattr(p, cmd_args.browser).launch(**kwargs) 46 | context = browser.new_context() 47 | context.set_default_timeout(cmd_args.timeout * 1000) 48 | context.new_page().goto(launcher.url) 49 | 50 | from tests import basic_test 51 | basic_test.run_tests(launcher, context) 52 | 53 | from tests import dimension_check_test 54 | dimension_check_test.run_tests(launcher, context) 55 | 56 | from tests import save_test 57 | save_test.run_tests(launcher, context) 58 | 59 | from tests import resize_test 60 | resize_test.run_tests(launcher, context) 61 | 62 | browser.close() 63 | -------------------------------------------------------------------------------- /src/ui_tests/test_util/jupyter_launcher.py: -------------------------------------------------------------------------------- 1 | # Launches Jupyter notebook as subprocess for testing. 2 | 3 | import glob 4 | import logging 5 | import os 6 | import re 7 | import signal 8 | import subprocess 9 | import sys 10 | import threading 11 | import time 12 | import traceback 13 | 14 | import nbformat # from Jupyter 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | curdir = os.path.dirname(os.path.realpath(sys.argv[0])) 19 | croquis_srcdir = '%s/..' % curdir 20 | test_dir = os.path.join(curdir, '.test_workdir') 21 | try: 22 | os.mkdir(test_dir) 23 | except FileExistsError: 24 | pass 25 | 26 | # OK, quick and dirty, but should work. 27 | os.system(f'cd "{test_dir}" ; rm -f *.ipynb') 28 | 29 | class JupyterLauncher(object): 30 | def __init__(self, cmd_args): 31 | self.cmd_args = cmd_args 32 | self.jupyter_proc = None 33 | 34 | def __enter__(self): 35 | self._run_with_timeout( 36 | 2.0, self._launch_jupyter, 37 | 'Failed to launch Jupyter notebook in time.') 38 | return self 39 | 40 | def __exit__(self, exc_type, exc_value, tb): 41 | try: 42 | # Apparently SIGTERM only kills the notebook (parent) process and 43 | # doesn't kill kernels, so we end up doing something stupid like 44 | # this. 45 | # 46 | # TODO: what about Windows? 47 | s = subprocess.check_output( 48 | ['pgrep', '-P', str(self.jupyter_proc.pid)]) 49 | pids = [int(pid) for pid in s.split()] 50 | 51 | self.jupyter_proc.send_signal(signal.SIGKILL) 52 | for pid in pids: 53 | print('Terminating notebook kernel process %d ...' % pid) 54 | os.kill(pid, signal.SIGTERM) 55 | except: 56 | sys.stderr.write( 57 | 'Failed to shut down notebook: ' 58 | 'processes may be lingering ...\n') 59 | traceback.print_exc() 60 | pass 61 | 62 | # Run a function with timeout: if it doesn't finish in time, abort the 63 | # process. 64 | def _run_with_timeout(self, timeout, fn, abort_msg): 65 | done = False 66 | def _timeout(): 67 | time.sleep(timeout) 68 | if done: return # The function executed successfully! 69 | 70 | logger.critical('%s', abort_msg) 71 | try: 72 | self.jupyter_proc.terminate() 73 | except: 74 | pass 75 | os._exit(1) 76 | 77 | thr = threading.Thread(target=_timeout) 78 | thr.daemon = True 79 | thr.start() 80 | 81 | fn() 82 | done = True 83 | 84 | # TODO: This is brittle and depends on the exact console log format of 85 | # Jupyter notebook, but I'm not sure if there's a better way ... 86 | def _launch_jupyter(self): 87 | cmd = ['jupyter-notebook', '--no-browser', '-y'] 88 | if not self.cmd_args.verbose: 89 | cmd += ['--log-level=CRITICAL'] 90 | 91 | self.jupyter_proc = subprocess.Popen( 92 | cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, 93 | cwd=test_dir, encoding='utf-8') 94 | while True: 95 | line = self.jupyter_proc.stderr.readline() 96 | if self.cmd_args.verbose: sys.stderr.write(line) 97 | m = re.search(r'http://127.0.0.1:([0-9]+)/\?token=[0-9a-f]+', line) 98 | if m: 99 | self.url = m.group(0) 100 | self.port = m.group(1) 101 | return 102 | 103 | # Create the notebook (.ipynb) file with the given content and return a URL 104 | # for opening it. 105 | # cf. https://stackoverflow.com/a/45672031 106 | def create_new_notebook(self, filename, cell_contents, 107 | separate_prefix=False): 108 | assert filename.endswith('.ipynb'), filename 109 | 110 | PREFIX = f''' 111 | ### PREFIX ### 112 | import sys 113 | 114 | sys.path.insert(0, "{croquis_srcdir}") 115 | sys.path.insert(0, "{curdir}") 116 | 117 | print('###' + ' PREFIX' + ' OK' + ' ###') 118 | ''' 119 | if separate_prefix: 120 | cell_contents = [PREFIX] + cell_contents 121 | else: 122 | cell_contents = cell_contents.copy() 123 | cell_contents[0] = PREFIX + cell_contents[0] 124 | 125 | nb = nbformat.v4.new_notebook() 126 | nb['cells'] = [nbformat.v4.new_code_cell(c) for c in cell_contents] 127 | 128 | full_filename = os.path.join(test_dir, filename) 129 | with open(full_filename, 'w') as f: 130 | nbformat.write(nb, f) 131 | 132 | url_prefix = self.url.split('?')[0] 133 | return f'{url_prefix}notebooks/{filename}' 134 | -------------------------------------------------------------------------------- /src/ui_tests/test_util/test_helper.py: -------------------------------------------------------------------------------- 1 | # Utility functions for testing. 2 | 3 | import time 4 | 5 | import playwright.async_api 6 | 7 | # Execute a Jupyter cell containing the given substring (`substr`). 8 | # Wait until an element with `result_selector` is available, and return that 9 | # element. 10 | def run_jupyter_cell(page, substr, result_selector, **kwargs): 11 | selector = f'pre[role="presentation"]:has-text("{substr}")' 12 | pre = page.wait_for_selector(selector) 13 | pre.click() 14 | cell = page.wait_for_selector(f'div.cell.selected:has({selector})') 15 | focus = cell.wait_for_selector('textarea') 16 | focus.scroll_into_view_if_needed() 17 | focus.press('Control+Enter') 18 | 19 | return cell, cell.wait_for_selector(result_selector, **kwargs) 20 | 21 | def save(page): 22 | page.press('text=File', 'Control+S') 23 | page.wait_for_timeout(500) # Wait 0.5 sec just be sure. 24 | 25 | def restart_kernel(page): 26 | print('Restarting the ipython kernel ...') 27 | 28 | # Restart sometimes takes forever (including the next cell execution), so we 29 | # need huge timeout. 30 | page.set_default_timeout(10000) # 10 sec. 31 | 32 | page.click('button.btn[title*="restart the kernel"]') 33 | page.wait_for_selector('text="Restart kernel?"') 34 | page.click('button:has-text("Restart")') # Confirm. 35 | page.wait_for_selector('.kernel_idle_icon[id=kernel_indicator_icon]') 36 | 37 | def get_center_coord(elem): 38 | box = elem.bounding_box() 39 | return box['x'] + box['width'] / 2, box['y'] + box['height'] / 2 40 | 41 | def get_center_coord1(page, selector): 42 | item = page.wait_for_selector(selector) 43 | if item is None: return None 44 | return get_center_coord(item) 45 | 46 | # Hover mouse cursor at the given coordinate, verify that the tooltip's content 47 | # matches the given condition, and return. 48 | # 49 | # TODO: Get timeout from command line argument? 50 | def verify_tooltip(page, cell, coord_callback, verify_callback, timeout=2500): 51 | x, y = -999, -999 52 | T = time.time() 53 | 54 | while time.time() < T + timeout * 0.001: 55 | x1, y1 = coord_callback() 56 | if abs(x - x1) + abs(y - y1) > 1.0: 57 | # import datetime; print(datetime.datetime.now()) 58 | # print(f'Moving cursor from {x} {y} to {x1} {y1}') 59 | x, y = x1, y1 60 | page.mouse.move(x, y) 61 | 62 | # Just wait 100 ms, because element coordinates may change. 63 | try: 64 | tooltip = cell.wait_for_selector( 65 | 'div.cr_tooltip', state='visible', timeout=100) 66 | except playwright.async_api.TimeoutError: 67 | continue 68 | 69 | content = tooltip.text_content() 70 | if verify_callback(content): return content 71 | 72 | # page.pause() 73 | raise TimeoutError 74 | 75 | # Check the plot is functioning by hovering at the center of the plot and 76 | # checking that we get the correct label. 77 | def check_center_label(page, cell, label, **kwargs): 78 | def coord_cb(): 79 | grid = cell.wait_for_selector('.cr_grid') 80 | grid.scroll_into_view_if_needed() 81 | return get_center_coord(grid) 82 | 83 | verify_tooltip(page, cell, coord_cb, lambda text: label in text, **kwargs) 84 | -------------------------------------------------------------------------------- /src/ui_tests/tests/basic_test.py: -------------------------------------------------------------------------------- 1 | # Basic functionality test. 2 | 3 | import re 4 | import time 5 | 6 | from test_util import test_helper 7 | 8 | CELL1 = ''' 9 | ### CELL1 ### 10 | 11 | import os; print('PID = ', os.getpid()) 12 | 13 | import croquis 14 | import numpy as np 15 | 16 | # Create a spiral. 17 | N = 1000 18 | R = np.exp(np.linspace(np.log(0.1), np.log(3), N)) 19 | Th = np.linspace(0, 5 * np.pi, N) 20 | X = (R * np.cos(Th)).reshape(N, 1) 21 | Y = (R * np.sin(Th)).reshape(N, 1) 22 | labels=['pt %d' % i for i in range(N)] 23 | 24 | fig = croquis.plot() 25 | 26 | # Add a point at the origin. 27 | fig.add([0], [0], marker_size=5, label='origin') 28 | 29 | # Add the spiral. 30 | fig.add(X, Y, marker_size=3, labels=labels) 31 | 32 | fig.show() 33 | ''' 34 | 35 | def test_zoom(launcher, context): 36 | page = context.new_page() 37 | url = launcher.create_new_notebook('basic_test.ipynb', [CELL1]) 38 | page.goto(url) 39 | 40 | # Scroll to the bottom. 41 | cell, x_axis = test_helper.run_jupyter_cell(page, 'CELL1', 'div.cr_x_axis') 42 | x_axis.scroll_into_view_if_needed() 43 | 44 | # Check screen coordinate of x=-1, x=1, y=-1, y=1. 45 | xleft, _ = test_helper.get_center_coord1( 46 | page, r'div.cr_x_axis div.cr_label >> text=/^-1(\.0)?$/') 47 | xright, _ = test_helper.get_center_coord1( 48 | page, r'div.cr_x_axis div.cr_label >> text=/^1(\.0)?$/') 49 | _, ytop = test_helper.get_center_coord1( 50 | page, 'div.cr_y_axis div.cr_label >> text=/^1(\.0)?$/') 51 | _, ybottom = test_helper.get_center_coord1( 52 | page, 'div.cr_y_axis div.cr_label >> text=/^-1(\.0)?/') 53 | 54 | # print('coords = ', xleft, xright, ytop, ybottom) 55 | 56 | # Select a slightly smaller area (so that all the coordinates are between 57 | # -1 and 1). 58 | w = xright - xleft 59 | h = ybottom - ytop 60 | xleft += 0.1 * w 61 | xright -= 0.1 * w 62 | ytop += 0.1 * w 63 | ybottom -= 0.1 * w 64 | 65 | # page.pause() 66 | 67 | # Drag mouse across the area to zoom in. 68 | page.mouse.move(xleft, ytop) 69 | page.mouse.down() 70 | time.sleep(0.1) 71 | page.mouse.move(xright, ybottom) 72 | time.sleep(0.1) 73 | page.mouse.up() 74 | 75 | def coord_cb(): 76 | xzero, _ = test_helper.get_center_coord1( 77 | page, 'div.cr_x_axis div.cr_label >> text="0.0"') 78 | _, yzero = test_helper.get_center_coord1( 79 | page, 'div.cr_y_axis div.cr_label >> text="0.0"') 80 | 81 | # print('coords = ', xzero, yzero) 82 | return xzero, yzero 83 | 84 | # Now zoom at the origin, and verify that we have the tooltip. 85 | test_helper.verify_tooltip(page, cell, coord_cb, 86 | lambda text: re.search(r'origin(.|\n)*\(\s*0,\s+0\)', text)) 87 | 88 | # print('test_zoom successful!') 89 | # page.pause() 90 | 91 | page.close() 92 | 93 | def run_tests(launcher, context): 94 | print('Running basic_test.py ...') 95 | test_zoom(launcher, context) 96 | -------------------------------------------------------------------------------- /src/ui_tests/tests/dimension_check_test.py: -------------------------------------------------------------------------------- 1 | # Test DimensionChecker. 2 | 3 | from test_util import test_helper 4 | 5 | CELL1 = ''' 6 | ### CELL1 ### 7 | 8 | import croquis 9 | import numpy as np 10 | 11 | # Test coordinate broadcast logic (similar to numpy). 12 | fig = croquis.plot() 13 | X = np.linspace(-5, 5, 10) # shape = (10,) 14 | Y = np.arange(30).reshape(3, 10) 15 | fig.add(X, Y) 16 | fig.add(X, -Y, colors=np.zeros((3, 3))) 17 | 18 | # Add a center dot. 19 | fig.add([0], [0], marker_size=30, label='center') 20 | 21 | fig.show() 22 | ''' 23 | 24 | CELL2 = ''' 25 | ### CELL2 ### 26 | 27 | # Test dimension mismatch errors. 28 | def expect_error(*args, **kwargs): 29 | fig = croquis.plot() 30 | try: 31 | fig.add(*args, **kwargs) 32 | except ValueError: 33 | print('ValueError raised successfully!') 34 | return 35 | 36 | assert None, 'ERROR: no exception raised!' 37 | 38 | expect_error(X, Y=np.arange(27).reshape(3, 9)) 39 | expect_error(X, Y=np.arange(9)) 40 | expect_error(X[:9], Y) 41 | expect_error(X, Y, colors=[0.1, 0.2]) 42 | expect_error(X, Y, colors=np.zeros((3, 2))) 43 | expect_error(X, Y, colors=np.zeros((4, 3))) 44 | print('###' + ' TEST OK ' + '###') 45 | ''' 46 | 47 | def test_dimension_check(launcher, context): 48 | page = context.new_page() 49 | url = launcher.create_new_notebook('dimension_test.ipynb', [CELL1, CELL2]) 50 | page.goto(url) 51 | 52 | # Test broadcast. 53 | cell1, axis1 = test_helper.run_jupyter_cell(page, 'CELL1', 'div.cr_x_axis') 54 | test_helper.check_center_label(page, cell1, 'center') 55 | 56 | # Test mismatch errors. 57 | cell1, axis1 = test_helper.run_jupyter_cell( 58 | page, 'CELL2', 'pre:has-text("### TEST OK ###")') 59 | 60 | def run_tests(launcher, context): 61 | print('Running dimension_check_test.py ...') 62 | test_dimension_check(launcher, context) 63 | -------------------------------------------------------------------------------- /src/ui_tests/tests/resize_test.py: -------------------------------------------------------------------------------- 1 | # Test window resize. 2 | 3 | from test_util import test_helper 4 | 5 | CELL1 = ''' 6 | ### CELL1 ### 7 | 8 | import croquis 9 | 10 | # Create three dots. 11 | fig = croquis.plot() 12 | fig.add([-1], [0], marker_size=10, label='left') 13 | fig.add([0], [0], marker_size=10, label='center') 14 | fig.add([1], [0], marker_size=10, label='right') 15 | fig.show() 16 | ''' 17 | 18 | def test_resize(launcher, context): 19 | page = context.new_page() 20 | url = launcher.create_new_notebook('resize_test.ipynb', [CELL1]) 21 | page.goto(url) 22 | 23 | cell1, axis1 = test_helper.run_jupyter_cell(page, 'CELL1', 'div.cr_x_axis') 24 | test_helper.check_center_label(page, cell1, 'center') 25 | 26 | w0, h0 = page.viewport_size['width'], page.viewport_size['height'] 27 | print(f'Original window width, height = {w0}, {h0}') 28 | 29 | # Try reducing window size. 30 | w, h = int(w0 * 0.7), int(h0 * 0.7) 31 | page.set_viewport_size({'width': w, 'height': h}) 32 | print(f'New window width, height = {w}, {h}') 33 | test_helper.check_center_label(page, cell1, 'center') 34 | 35 | # Try increasing window size. 36 | w, h = int(w0 * 1.2), int(h0 * 1.2) 37 | page.set_viewport_size({'width': w, 'height': h}) 38 | print(f'New window width, height = {w}, {h}') 39 | test_helper.check_center_label(page, cell1, 'center') 40 | 41 | def run_tests(launcher, context): 42 | print('Running resize_test.py ...') 43 | test_resize(launcher, context) 44 | -------------------------------------------------------------------------------- /src/ui_tests/tests/save_test.py: -------------------------------------------------------------------------------- 1 | # Test that the cells behave sane across kernel restart and save/load. 2 | 3 | from test_util import test_helper 4 | 5 | CELL1 = ''' 6 | ### CELL1 ### 7 | 8 | import croquis 9 | 10 | # Create three dots. 11 | fig = croquis.plot() 12 | fig.add([-1], [0], marker_size=50, label='left') 13 | fig.add([0], [0], marker_size=50, label='center1') 14 | fig.add([1], [0], marker_size=50, label='right') 15 | fig.show() 16 | ''' 17 | 18 | CELL2 = ''' 19 | ### CELL2 ### 20 | 21 | import croquis 22 | 23 | # Create a different set of three dots. 24 | fig = croquis.plot() 25 | fig.add([-1], [-1], marker_size=50, label='down_left') 26 | fig.add([0], [0], marker_size=50, label='center2') 27 | fig.add([1], [1], marker_size=50, label='up_right') 28 | fig.show() 29 | ''' 30 | 31 | CELL3 = ''' 32 | ### CELL3 ### 33 | 34 | import croquis 35 | 36 | # Create the third set of dots! 37 | fig = croquis.plot() 38 | fig.add([-1], [1], marker_size=50, label='up_left') 39 | fig.add([0], [0], marker_size=50, label='center3') 40 | fig.add([1], [-1], marker_size=50, label='down_right') 41 | fig.show() 42 | ''' 43 | 44 | def test_kernel_restart(launcher, context): 45 | page = context.new_page() 46 | url = launcher.create_new_notebook( 47 | 'kernel_restart_test.ipynb', [CELL1, CELL2, CELL3], 48 | separate_prefix=True) 49 | page.goto(url) 50 | test_helper.run_jupyter_cell(page, 'PREFIX', 'pre:has-text("PREFIX OK")') 51 | 52 | test_helper.save(page) 53 | 54 | cell1, axis1 = test_helper.run_jupyter_cell(page, 'CELL1', 'div.cr_x_axis') 55 | cell2, axis2 = test_helper.run_jupyter_cell(page, 'CELL2', 'div.cr_x_axis') 56 | 57 | test_helper.check_center_label(page, cell1, 'center1') 58 | test_helper.check_center_label(page, cell2, 'center2') 59 | 60 | test_helper.restart_kernel(page) 61 | test_helper.run_jupyter_cell(page, 'PREFIX', 'pre:has-text("PREFIX OK")') 62 | 63 | # Try Cell 3 now. 64 | cell3, axis3 = test_helper.run_jupyter_cell(page, 'CELL3', 'div.cr_x_axis') 65 | test_helper.check_center_label(page, cell3, 'center3', timeout=10000) 66 | 67 | page.close() 68 | 69 | def test_save_load(launcher, context): 70 | page = context.new_page() 71 | url = launcher.create_new_notebook('save_test.ipynb', [CELL1, CELL2, CELL3]) 72 | page.goto(url) 73 | 74 | cell1, axis1 = test_helper.run_jupyter_cell(page, 'CELL1', 'div.cr_x_axis') 75 | cell2, axis2 = test_helper.run_jupyter_cell(page, 'CELL2', 'div.cr_x_axis') 76 | 77 | test_helper.check_center_label(page, cell1, 'center1') 78 | test_helper.check_center_label(page, cell2, 'center2') 79 | 80 | test_helper.save(page) 81 | 82 | # Now open another window. 83 | page2 = context.new_page() 84 | page2.goto(url) 85 | 86 | # Try Cell 3 now. 87 | cell3, axis3 = test_helper.run_jupyter_cell(page2, 'CELL3', 'div.cr_x_axis') 88 | test_helper.check_center_label(page2, cell3, 'center3') 89 | 90 | # Check that we can re-run cell 2. 91 | cell2, axis2 = test_helper.run_jupyter_cell(page2, 'CELL2', 'div.cr_x_axis') 92 | test_helper.check_center_label(page2, cell2, 'center2') 93 | 94 | page.close() 95 | page2.close() 96 | 97 | def run_tests(launcher, context): 98 | print('Running save_test.py ...') 99 | test_kernel_restart(launcher, context) 100 | test_save_load(launcher, context) 101 | --------------------------------------------------------------------------------