├── .gitattributes ├── .gitignore ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── README.md ├── conda-recipe └── meta.yaml ├── dependencies.txt ├── docs ├── Makefile ├── deploy-docs.sh ├── requirements.txt └── source │ ├── _static │ ├── docs-badge.svg │ ├── neuprint-explorer-cypher-button.png │ ├── theme_overrides.css │ └── token-screenshot.png │ ├── admin.rst │ ├── api.rst │ ├── changelog.rst │ ├── client.rst │ ├── conf.py │ ├── development.rst │ ├── faq.rst │ ├── index.rst │ ├── mitocriteria.rst │ ├── neuroncriteria.rst │ ├── notebooks │ ├── QueryTutorial.ipynb │ └── SimulationTutorial.ipynb │ ├── queries.rst │ ├── quickstart.rst │ ├── related.rst │ ├── simulation.rst │ ├── skeleton.rst │ ├── synapsecriteria.rst │ ├── tutorials.rst │ ├── utils.rst │ └── wrangle.rst ├── environment.yml ├── examples └── skeleton-with-synapses.ipynb ├── neuprint ├── __init__.py ├── _version.py ├── admin.py ├── client.py ├── plotting.py ├── queries │ ├── __init__.py │ ├── connectivity.py │ ├── general.py │ ├── mito.py │ ├── mitocriteria.py │ ├── neuroncriteria.py │ ├── neurons.py │ ├── recon.py │ ├── rois.py │ ├── synapsecriteria.py │ └── synapses.py ├── simulation.py ├── skeleton.py ├── tests │ ├── __init__.py │ ├── test_arrow_endpoint.py │ ├── test_client.py │ ├── test_neuroncriteria.py │ ├── test_queries.py │ ├── test_skeleton.py │ └── test_utils.py ├── utils.py └── wrangle.py ├── pixi.lock ├── pixi.toml ├── setup.cfg ├── setup.py ├── update-deps.sh ├── upload-to-pypi.sh └── versioneer.py /.gitattributes: -------------------------------------------------------------------------------- 1 | neuprint/_version.py export-subst 2 | # GitHub syntax highlighting 3 | pixi.lock linguist-language=YAML linguist-generated=true 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # pyenv python configuration file 62 | .python-version 63 | 64 | .project 65 | .pydevproject 66 | .ipynb_checkpoints 67 | .settings/ 68 | .vscode/ 69 | 70 | .DS_store 71 | _autosummary/ 72 | 73 | # pixi environments 74 | .pixi 75 | *.egg-info 76 | 77 | **/.claude/settings.local.json 78 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: "Berg" 5 | given-names: "Stuart" 6 | orcid: "https://orcid.org/0000-0002-0766-0488" 7 | - family-names: "Schlegel" 8 | given-names: "Philipp" 9 | orcid: "https://orcid.org/0000-0002-5633-1314" 10 | title: "neuprint-python" 11 | version: 0.4.25 12 | doi: 10.5281/zenodo.7592899 13 | date-released: 2017-12-18 14 | url: "https://connectome-neuprint.github.io/neuprint-python" 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 HHMI. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of HHMI nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include versioneer.py 2 | include neuprint/_version.py 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![docs-badge](docs/source/_static/docs-badge.svg)][docs] 2 | 3 | neuprint-python 4 | =============== 5 | 6 | Python client utilties for interacting with the [neuPrint][neuprint] connectome analysis service. 7 | 8 | [neuprint]: https://neuprint.janelia.org 9 | 10 | ## Install 11 | 12 | If you're using pixi, use this: 13 | ```shell 14 | pixi init -c flyem-forge -c conda-forge 15 | pixi add python=3.9 'neuprint-python>=0.5.1' 'pyarrow>=20' 'numpy>=2' 'pandas>=2' 16 | ``` 17 | 18 | If you're using conda, use this command: 19 | 20 | ```shell 21 | conda install -c flyem-forge neuprint-python 22 | ``` 23 | 24 | Otherwise, use pip: 25 | 26 | ```shell 27 | pip install neuprint-python 28 | ``` 29 | 30 | ## Getting started 31 | 32 | See the [Quickstart section][quickstart] in the [documentation][docs] 33 | 34 | [docs]: http://connectome-neuprint.github.io/neuprint-python/docs/ 35 | [quickstart]: http://connectome-neuprint.github.io/neuprint-python/docs/quickstart.html 36 | 37 | -------------------------------------------------------------------------------- /conda-recipe/meta.yaml: -------------------------------------------------------------------------------- 1 | 2 | {% set data = load_setup_py_data() %} 3 | 4 | package: 5 | name: neuprint-python 6 | 7 | version: {{ data['version'] }} 8 | 9 | source: 10 | path: .. 11 | 12 | build: 13 | script: python setup.py install --single-version-externally-managed --record=record.txt 14 | noarch: python 15 | script_env: 16 | - NEUPRINT_APPLICATION_CREDENTIALS 17 | 18 | requirements: 19 | build: 20 | - python >=3.9 21 | - setuptools 22 | run: 23 | - python >=3.9 24 | # dependencies are defined in setup.py 25 | {% for dep in data['install_requires'] %} 26 | - {{ dep.lower() }} 27 | {% endfor %} 28 | {# raw is for ignoring templating with cookiecutter, leaving it for use with conda-build #} 29 | 30 | test: 31 | imports: 32 | - neuprint 33 | requires: 34 | - pytest 35 | commands: 36 | - pytest --pyargs neuprint.tests 37 | 38 | about: 39 | home: https://github.com/stuarteberg/neuprint-python 40 | summary: Python client utilties for interacting with the neuPrint connectome analysis service 41 | license: BSD-3-Clause 42 | license_file: LICENSE 43 | -------------------------------------------------------------------------------- /dependencies.txt: -------------------------------------------------------------------------------- 1 | # This file is named 'dependencies.txt' instead of 'requirements.txt' 2 | # to prevent binder from detecting it. 3 | # (We want binder to use environment.yml) 4 | requests>=2.22 5 | pandas 6 | tqdm 7 | ujson 8 | asciitree 9 | scipy 10 | networkx 11 | packaging 12 | pyarrow 13 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /docs/deploy-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"; 6 | REPO_DIR="$(dirname ${SCRIPT_DIR})" 7 | 8 | echo "Building docs in your local repo" 9 | cd ${REPO_DIR}/docs 10 | GIT_DESC=$(git describe) 11 | make html 12 | 13 | TMP_REPO=$(mktemp -d) 14 | echo "Cloning to ${TMP_REPO}/neuprint-python" 15 | cd ${TMP_REPO} 16 | git clone ssh://git@github.com/connectome-neuprint/neuprint-python 17 | cd neuprint-python 18 | 19 | echo "Committing built docs" 20 | git switch -c gh-pages origin/gh-pages 21 | rm -r docs 22 | cp -R ${REPO_DIR}/docs/build/html docs 23 | git add . 24 | git commit -m "Updated docs for ${GIT_DESC}" . 25 | 26 | echo "Pushing to github" 27 | git push origin gh-pages 28 | 29 | echo "DONE deploying docs" 30 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | ###### Requirements without Version Specifiers ###### 2 | nbsphinx 3 | numpydoc 4 | sphinx_bootstrap_theme 5 | sphinx 6 | sphinx_rtd_theme 7 | ipython 8 | jupyter 9 | ipywidgets 10 | bokeh 11 | holoviews 12 | hvplot 13 | selenium 14 | phantomjs 15 | -------------------------------------------------------------------------------- /docs/source/_static/docs-badge.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | docs 16 | docs 17 | here! 18 | here! 19 | 20 | 21 | -------------------------------------------------------------------------------- /docs/source/_static/neuprint-explorer-cypher-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/connectome-neuprint/neuprint-python/1d25b67b5178e56ce52f0cbdeb126fcd08b4888a/docs/source/_static/neuprint-explorer-cypher-button.png -------------------------------------------------------------------------------- /docs/source/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | /* override table width restrictions */ 2 | @media screen and (min-width: 767px) { 3 | 4 | .wy-table-responsive table td { 5 | /* !important prevents the common CSS stylesheets from overriding 6 | this as on RTD they are loaded after this stylesheet */ 7 | white-space: normal !important; 8 | } 9 | 10 | .wy-table-responsive { 11 | overflow: visible !important; 12 | } 13 | 14 | /* 15 | .wy-nav-content { 16 | max-width: none; 17 | } 18 | */ 19 | } 20 | -------------------------------------------------------------------------------- /docs/source/_static/token-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/connectome-neuprint/neuprint-python/1d25b67b5178e56ce52f0cbdeb126fcd08b4888a/docs/source/_static/token-screenshot.png -------------------------------------------------------------------------------- /docs/source/admin.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.admin 2 | 3 | .. _admin: 4 | 5 | 6 | Admin Tools 7 | =========== 8 | 9 | .. automodule:: neuprint.admin 10 | 11 | .. autoclass:: Transaction 12 | :members: 13 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API 4 | === 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | client 10 | queries 11 | simulation 12 | skeleton 13 | wrangle 14 | utils 15 | admin 16 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.5.1 / 2025-02-02 5 | ------------------ 6 | - ``fetch_neurons()``: Added ``omit_rois`` option, which speeds up the function if you don't need ROI information. 7 | - For admins: Fixed an issue that could cause needless exceptions to be raised when cleaning up from a failed transaction. 8 | 9 | 0.5 / 2024-12-11 10 | ---------------- 11 | - Now compatible with numpy 2.x 12 | - Fixed various warnings that occur with pandas 2.x 13 | - Minimum supported Python version is now explicitly listed as 3.9 14 | - Add missing ``client`` arguments in various places instead of using the default. (PR #58 and related commits) 15 | This is crucial if multiple clients have been constructed. 16 | - ``fetch_mean_synapses()``: Added ``by_roi`` option to allow the user to fetch whole-neuron mean synapses 17 | - ``fetch_shorted_paths()`` allows you to omit filtering entirely using ``NC()`` 18 | - ``fetch_neurons()``: If no ``NeuronCriteria`` is provided, fetch all ``:Neuron``s by default 19 | - Added ``available_datasets`` to utils.py (PR #60) 20 | - Internally generated Cypher now uses backticks for variables/properties that require them. (PR #42 and related commits) 21 | - Bug fix in ``connection_table_to_matrix()`` (PR #47) 22 | - Bug fix in ``fetch_common_connectivity()`` (PR #63) 23 | - Several other bug fixes 24 | - For developers: Added basic ``pixi`` configuration 25 | 26 | 0.4.26 / 2023-06-08 27 | ------------------- 28 | - ``NeuronCriteria`` now supports many new properties for the MANC v1.0 dataset. 29 | - Neuron property columns are determined from cached metadata rather than a full scan of the database. 30 | - If more than one ``Client`` has been constructed, none of them become automatically become the default client. 31 | In that case, you must explicitly pass a ``client`` argument to each query function you call. This avoids a 32 | common pitfall when dealing with multiple neuprint datasets (and therefore multiple Clients). 33 | - ``SynapseCriteria`` now uses a default confidence threshold based on the dataset metadata (instead of using 0.0 by default) 34 | - ``Client`` constructor avoids contacting the database unless it needs to. (Duplicate clients are now cheaper to construct.) 35 | - Minor enhancements to skeleton utilities, including a couple new analysis functions. 36 | - Added CITATION.cff 37 | 38 | 0.4.25 / 2022-09-15 39 | ------------------- 40 | 41 | - In live-updated neuprint databases, it is possible that an edge's ``weight`` can become out-of-sync with its ``roiInfo`` totals. 42 | That inconsistency triggered an assertion in ``fetch_adjacencies()``, but now it will emit a warning instead. 43 | 44 | 0.4.24 / 2022-07-14 45 | ------------------- 46 | 47 | - Implemented a workaround to avoid a pandas bug in certain cases involving empty dataframes. 48 | 49 | 0.4.23 / 2022-06-14 50 | ------------------- 51 | 52 | - In ``fetch_adjacencies()`` (and ``fetch_simple_connections()``), we now ensure that no 0-weight "connections" are returned. 53 | 54 | .. note:: 55 | 56 | In recent neuprint databases, some ``:ConnectsTo`` relationships may have a ``weight`` of ``0``. 57 | In such cases, the relationship will have a non-zero ``weightHR`` (high-recall weight), but all of the relevant 58 | synapses are low-confidence, hence the "default" ``weight`` of ``0``. 59 | We now exclude such connections from our results. 60 | 61 | 0.4.22 / 2022-06-14 62 | ------------------- 63 | 64 | - Fixed a crash that could occur if you supplied more than three regular expressions for ``type`` or ``instance``. 65 | - Fixed a problem involving 'hidden' ROIs in the hemibrain v1.0. 66 | 67 | 0.4.21 / 2022-05-14 68 | ------------------- 69 | 70 | - Now ``heal_skeleton()`` is slightly faster in the case where no healing was necessary. 71 | 72 | 0.4.20 / 2022-05-13 73 | ------------------- 74 | 75 | - By default, ``NeuronCriteria`` will now guess whether the ``type`` and ``instance`` contain 76 | a regular expression or not, so you don't need to explicitly pass ``regex=True``. 77 | Override the guess by specifying ``regex=True`` or ``regex=False``. 78 | 79 | 0.4.19 / 2022-05-12 80 | ------------------- 81 | 82 | - Added ``fetch_mean_synapses()`` 83 | - Added ``attach_synapses_to_skeleton()`` 84 | 85 | 0.4.18 / 2022-04-06 86 | ------------------- 87 | 88 | - Fixed broken package distribution. 89 | 90 | 0.4.17 / 2022-04-06 91 | ------------------- 92 | 93 | - **[CHANGE IN RETURNED RESULTS]** ``fetch_synapse_connections()`` now applies ROI filtering criteria to only the post-synaptic points, 94 | for consistency with ``fetch_adjacencies()``. (See note in the docs.) 95 | This means that the number of synapses returned by ``fetch_synapse_connections()`` is now slightly different than it was in previous 96 | versions of ``neuprint-python``. 97 | - In ``fetch_neurons()``, better handling of old neuprint datasets which lack some fields (e.g. ``upstream``, ``downstream``, ``mito``). 98 | 99 | 0.4.16 / 2021-11-30 100 | ------------------- 101 | - ``NeuronCriteria`` has new fields to support upcoming datasets: ``somaSide``, ``class_``, ``statusLabel``, ``hemilineage``, ``exitNerve``. 102 | - ``NeuronCriteria`` now permits you to search for neurons that contain (or lack) a particular property via a special value ``NotNull`` (or ``IsNull``). 103 | - ``fetch_neurons()`` now returns all neuron properties. 104 | - ``fetch_neurons()`` now returns special rows for NotPrimary connection counts. 105 | - The per-ROI connection counts table returned by ``fetch_neurons()`` now includes rows for connections which fall outside of all primary ROIs. 106 | These are indicated by the special ROI name ``NotPrimary``. 107 | - ``fetch_synapse_connections()`` uses a more fine-grained batching strategy, splitting the query across more requests to avoid timeouts. 108 | - Fixed a bug in ``fetch_shortest_paths()`` which caused it to generate invalid cypher if the ``intermediate_criteria`` 109 | used a list of bodyIds (or statuses, or rois, etc.) with more than three items. 110 | - ``fetch_output_completeness`` now accepts a list of statuses to use, rather than assuming only ``"Traced"`` neurons are complete. 111 | - Added utility function ``skeleton_segments()``. 112 | 113 | 114 | 0.4.15 / 2021-06-16 115 | ------------------- 116 | - ``NeuronCriteria`` now accepts a boolean argument for ``soma``, indicating the presence or absence of a soma on the body. 117 | - Added ``fetch_connection_mitochondria()`` for finding the nearest mitochondria on both sides of a tbar/psd pair. (#24) 118 | - Integrated with Zenodo for DOI generation. 119 | 120 | 121 | 0.4.14 / 2021-03-27 122 | ------------------- 123 | - Updated to changes in the neuPrint mitochondria data model. 124 | Older versions of ``neuprint-python`` cannot query for mitochondria any more. 125 | - ``fetch_neurons()``: Added new columns to the ``roi_counts_df`` result, for ``upstream, downstream, mito`` 126 | - ``fetch_skeletons()``: Now supports ``with_distances`` option 127 | - ``NeuronCriteria`` permits lists of strings for type/instance regular expressions. 128 | (Previously, lists were only permitted when ``regex=False``.) 129 | - Fixed a performance problem in ``fetch_synapse_connections()`` 130 | - More FAQ entries 131 | 132 | 133 | 0.4.13 / 2020-12-23 134 | ------------------- 135 | 136 | - ``SynapseCriteria``: Changed the default value of ``primary_only`` to ``True``, 137 | since it may been counter-intuitive to obtain duplicate results by default. 138 | - ``NeuronCriteria``: Added ``cellBodyFiber`` parameter. (Philipp Shlegel #13) 139 | - Added mitochondria queries 140 | 141 | 142 | 0.4.12 / 2020-11-21 143 | ------------------- 144 | 145 | - Better handling when adjacency queries return empty results 146 | - Simulation: Minor change to subprocess communication implementation 147 | - Skeleton DataFrames use economical dtypes 148 | - Minor bug fixes and performance enhancements 149 | - fetch_synapse_connections(): Fix pandas error in assertion 150 | 151 | 152 | 0.4.11 / 2020-06-30 153 | ------------------- 154 | 155 | - Fixed ``ngspice`` install instructions. 156 | 157 | 158 | 0.4.10 / 2020-06-30 159 | ------------------- 160 | 161 | - Moved skeleton-related functions into their own module, and added a few more skeleton utilty functions 162 | - Simulation: Support Windows 163 | - ``heal_skeleton():`` Allow caller to specify a maximum distance for repaired skeleton segments (#12) 164 | 165 | 166 | 0.4.9 / 2020-04-29 167 | ------------------ 168 | 169 | - Added simulation functions and tutorial 170 | -------------------------------------------------------------------------------- /docs/source/client.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.client 2 | 3 | .. _client: 4 | 5 | 6 | Client 7 | ====== 8 | 9 | .. automodule:: neuprint.client 10 | 11 | .. autosummary:: 12 | 13 | default_client 14 | set_default_client 15 | clear_default_client 16 | list_all_clients 17 | setup_debug_logging 18 | disable_debug_logging 19 | 20 | 21 | :py:class:`Client` methods correspond directly to built-in 22 | `neuprintHTTP API endpoints `_. 23 | 24 | 25 | .. autosummary:: 26 | 27 | Client 28 | Client.fetch_custom 29 | Client.fetch_available 30 | Client.fetch_help 31 | Client.fetch_server_info 32 | Client.fetch_version 33 | Client.fetch_database 34 | Client.fetch_datasets 35 | Client.fetch_instances 36 | Client.fetch_db_version 37 | Client.fetch_profile 38 | Client.fetch_token 39 | Client.fetch_daily_type 40 | Client.fetch_roi_completeness 41 | Client.fetch_roi_connectivity 42 | Client.fetch_roi_mesh 43 | Client.fetch_skeleton 44 | Client.fetch_raw_keyvalue 45 | Client.post_raw_keyvalue 46 | 47 | .. autoclass:: neuprint.client.Client 48 | :members: 49 | 50 | .. autofunction:: default_client 51 | .. autofunction:: set_default_client 52 | .. autofunction:: clear_default_client 53 | .. autofunction:: list_all_clients 54 | .. autofunction:: setup_debug_logging 55 | .. autofunction:: disable_debug_logging 56 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | sys.path.insert(0, os.path.abspath('.')) 18 | sys.path.insert(0, os.path.abspath('../..')) 19 | 20 | import inspect 21 | import neuprint 22 | from os.path import relpath, dirname 23 | 24 | import versioneer 25 | import numpydoc 26 | 27 | # -- Project information ----------------------------------------------------- 28 | 29 | project = 'neuprint-python' 30 | copyright = '2019, FlyEM' 31 | author = 'FlyEM' 32 | 33 | # The short X.Y version 34 | os.chdir(os.path.dirname(__file__) + '/../..') 35 | version = versioneer.get_version() 36 | os.chdir(os.path.dirname(__file__)) 37 | 38 | latest_tag = version.split('+')[0] 39 | 40 | # The full version, including alpha/beta/rc tags 41 | release = version 42 | 43 | # -- General configuration --------------------------------------------------- 44 | 45 | # If your documentation needs a minimal Sphinx version, state it here. 46 | # 47 | # needs_sphinx = '1.0' 48 | 49 | # Add any Sphinx extension module names here, as strings. They can be 50 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 51 | # ones. 52 | extensions = [ 53 | 'sphinx.ext.autodoc', 54 | #'sphinx.ext.viewcode', # Link to sphinx-generated source code pages. 55 | 'sphinx.ext.linkcode', # Link to source code on github (see linkcode_resolve(), below.) 56 | 'sphinx.ext.githubpages', 57 | 'sphinx.ext.autosummary', 58 | 'sphinx.ext.napoleon', 59 | 'IPython.sphinxext.ipython_console_highlighting', 60 | 'IPython.sphinxext.ipython_directive', 61 | 'nbsphinx' 62 | ] 63 | 64 | nbsphinx_execute = 'always' 65 | os.environ['RUNNING_IN_SPHINX'] = '1' 66 | 67 | nbsphinx_prolog = f""" 68 | 69 | .. 70 | (The following |br| definition is the only way 71 | I can force numpydoc to display explicit newlines...) 72 | 73 | .. |br| raw:: html 74 | 75 |
76 | 77 | .. note:: 78 | 79 | This page corresponds to a Jupyter notebook you can 80 | `try out yourself`_. |br| 81 | (The original version is `here`_.) 82 | 83 | .. _try out yourself: https://mybinder.org/v2/gh/connectome-neuprint/neuprint-python/{latest_tag}?filepath=docs%2Fsource%2F{{{{ env.doc2path(env.docname, base=None) }}}} 84 | 85 | .. _here: https://github.com/connectome-neuprint/neuprint-python/tree/{latest_tag}/docs/source/{{{{ env.doc2path(env.docname, base=None) }}}} 86 | 87 | .. image:: https://mybinder.org/badge_logo.svg 88 | :target: https://mybinder.org/v2/gh/connectome-neuprint/neuprint-python/{latest_tag}?filepath=docs%2Fsource%2F{{{{ env.doc2path(env.docname, base=None) }}}} 89 | 90 | ---- 91 | """ 92 | 93 | # generate autosummary pages 94 | autosummary_generate = True 95 | autoclass_content = 'both' 96 | 97 | # Don't alphabetically sort functions 98 | autodoc_member_order = 'groupwise' 99 | 100 | # Combine class docstrings and class __init__ docstrings 101 | # (e.g. see NeuronCriteria) 102 | autoapi_python_class_content = 'both' 103 | 104 | # Add any paths that contain templates here, relative to this directory. 105 | templates_path = ['_templates'] 106 | 107 | # The suffix(es) of source filenames. 108 | # You can specify multiple suffix as a list of string: 109 | # 110 | # source_suffix = ['.rst', '.md'] 111 | source_suffix = '.rst' 112 | 113 | # The master toctree document. 114 | master_doc = 'index' 115 | 116 | # The language for content autogenerated by Sphinx. Refer to documentation 117 | # for a list of supported languages. 118 | # 119 | # This is also used if you do content translation via gettext catalogs. 120 | # Usually you set "language" from the command line for these cases. 121 | language = 'en' 122 | 123 | # List of patterns, relative to source directory, that match files and 124 | # directories to ignore when looking for source files. 125 | # This pattern also affects html_static_path and html_extra_path. 126 | exclude_patterns = ['build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints'] 127 | 128 | # The name of the Pygments (syntax highlighting) style to use. 129 | pygments_style = None 130 | 131 | 132 | # -- Options for HTML output ------------------------------------------------- 133 | 134 | # The theme to use for HTML and HTML Help pages. See the documentation for 135 | # a list of builtin themes. 136 | # 137 | #html_theme = 'nature' 138 | 139 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 140 | if not on_rtd: 141 | import sphinx_rtd_theme 142 | html_theme = 'sphinx_rtd_theme' 143 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 144 | 145 | 146 | # Theme options are theme-specific and customize the look and feel of a theme 147 | # further. For a list of options available for each theme, see the 148 | # documentation. 149 | # 150 | html_theme_options = { 151 | #'source_link_position': "footer", 152 | #'navbar_sidebarrel': False, 153 | #'navbar_links': [ 154 | # ("API", "src/api"), 155 | # ], 156 | 157 | } 158 | 159 | # Add any paths that contain custom static files (such as style sheets) here, 160 | # relative to this directory. They are copied after the builtin static files, 161 | # so a file named "default.css" will overwrite the builtin "default.css". 162 | html_static_path = ['_static'] 163 | html_css_files = ['theme_overrides.css'] 164 | 165 | 166 | # Custom sidebar templates, must be a dictionary that maps document names 167 | # to template names. 168 | # 169 | # The default sidebars (for documents that don't match any pattern) are 170 | # defined by theme itself. Builtin themes are using these templates by 171 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 172 | # 'searchbox.html']``. 173 | # 174 | # html_sidebars = {} 175 | 176 | 177 | # -- Options for HTMLHelp output --------------------------------------------- 178 | 179 | # Output file base name for HTML help builder. 180 | htmlhelp_basename = 'neuprintdoc' 181 | 182 | 183 | # -- Options for LaTeX output ------------------------------------------------ 184 | 185 | latex_elements = { 186 | # The paper size ('letterpaper' or 'a4paper'). 187 | # 188 | # 'papersize': 'letterpaper', 189 | 190 | # The font size ('10pt', '11pt' or '12pt'). 191 | # 192 | # 'pointsize': '10pt', 193 | 194 | # Additional stuff for the LaTeX preamble. 195 | # 196 | # 'preamble': '', 197 | 198 | # Latex figure (float) alignment 199 | # 200 | # 'figure_align': 'htbp', 201 | } 202 | 203 | # Grouping the document tree into LaTeX files. List of tuples 204 | # (source start file, target name, title, 205 | # author, documentclass [howto, manual, or own class]). 206 | latex_documents = [ 207 | (master_doc, 'neuprint-python.tex', 'neuprint-python Documentation', 208 | 'Philipp Schlegel', 'manual'), 209 | ] 210 | 211 | 212 | # -- Options for manual page output ------------------------------------------ 213 | 214 | # One entry per manual page. List of tuples 215 | # (source start file, name, description, authors, manual section). 216 | man_pages = [ 217 | (master_doc, 'neuprint-python', 'neuprint-python Documentation', 218 | [author], 1) 219 | ] 220 | 221 | 222 | # -- Options for Texinfo output ---------------------------------------------- 223 | 224 | # Grouping the document tree into Texinfo files. List of tuples 225 | # (source start file, target name, title, author, 226 | # dir menu entry, description, category) 227 | texinfo_documents = [ 228 | (master_doc, 'neuprint-python', 'neuprint-python Documentation', 229 | author, 'neuprint-python', 'One line description of project.', 230 | 'Miscellaneous'), 231 | ] 232 | 233 | 234 | # -- Options for Epub output ------------------------------------------------- 235 | 236 | # Bibliographic Dublin Core info. 237 | epub_title = project 238 | 239 | # The unique identifier of the text. This can be a ISBN number 240 | # or the project homepage. 241 | # 242 | # epub_identifier = '' 243 | 244 | # A unique identification for the text. 245 | # 246 | # epub_uid = '' 247 | 248 | # A list of files that should not be packed into the epub file. 249 | epub_exclude_files = ['search.html'] 250 | 251 | 252 | # -- Extension configuration ------------------------------------------------- 253 | 254 | # Function courtesy of NumPy to return URLs containing line numbers 255 | # (with edits to handle wrapped functions properly) 256 | def linkcode_resolve(domain, info): 257 | """ 258 | Determine the URL corresponding to Python object 259 | """ 260 | if domain != 'py': 261 | return None 262 | 263 | modname = info['module'] 264 | fullname = info['fullname'] 265 | 266 | submod = sys.modules.get(modname) 267 | if submod is None: 268 | return None 269 | 270 | obj = submod 271 | for part in fullname.split('.'): 272 | try: 273 | obj = getattr(obj, part) 274 | except: 275 | return None 276 | 277 | obj = inspect.unwrap(obj) 278 | 279 | try: 280 | fn = inspect.getsourcefile(obj) 281 | except: 282 | fn = None 283 | if not fn: 284 | return None 285 | 286 | try: 287 | _source, lineno = inspect.findsource(obj) 288 | except: 289 | lineno = None 290 | 291 | if lineno: 292 | linespec = "#L%d" % (lineno + 1) 293 | else: 294 | linespec = "" 295 | 296 | fn = relpath(fn, start=dirname(neuprint.__file__)) 297 | 298 | if '.g' in neuprint.__version__: 299 | return ("https://github.com/connectome-neuprint/neuprint-python/blob/" 300 | "master/neuprint/%s%s" % (fn, linespec)) 301 | else: 302 | return ("https://github.com/connectome-neuprint/neuprint-python/blob/" 303 | "%s/neuprint/%s%s" % (neuprint.__version__, fn, linespec)) -------------------------------------------------------------------------------- /docs/source/development.rst: -------------------------------------------------------------------------------- 1 | .. _development: 2 | 3 | Development Notes 4 | ================= 5 | 6 | Notes for maintaining ``neuprint-python``. 7 | 8 | Prerequisites 9 | ------------- 10 | 11 | Make sure you have both ``flyem-forge`` and ``conda-forge`` listed as channels in your ``.condarc`` file. 12 | (If you don't know where your ``.condarc`` file is, check ``conda config --show-sources``.) 13 | 14 | .. code-block:: yaml 15 | 16 | # .condarc 17 | channels: 18 | - flyem-forge 19 | - conda-forge 20 | - nodefaults # A magic channel that forbids any downloads from the anaconda default channels. 21 | 22 | Install ``conda-build`` if you don't have it yet: 23 | 24 | .. code-block:: bash 25 | 26 | conda install -n base conda-build anaconda-client twine setuptools 27 | 28 | 29 | Before you can upload packages to anaconda.org, you'll need to be a member of the ``flyem-forge`` organization. 30 | Then you'll need to run ``anaconda login``. 31 | 32 | Before you can upload packages to PyPI, you'll need to be added as a "collaborator" of the 33 | ``neuprint-python`` project on PyPI. Then you'll need to log in and obtain a token with 34 | an appropriate scope for ``neuprint-python`` and add it to your ``~/.pypirc`` file: 35 | 36 | .. code-block:: 37 | 38 | [distutils] 39 | index-servers = 40 | neuprint-python 41 | my-other-project 42 | 43 | [neuprint-python] 44 | repository = https://upload.pypi.org/legacy/ 45 | username = __token__ 46 | password = 47 | 48 | [my-other-project] 49 | repository = https://upload.pypi.org/legacy/ 50 | username = __token__ 51 | password = 52 | 53 | 54 | Packaging and Release 55 | --------------------- 56 | 57 | ``neuprint-python`` is packaged for both ``conda`` (on the `flyem-forge channel `_) 58 | and ``pip`` (on `PyPI `_). 59 | 60 | The package version is automatically inferred from the git tag. 61 | To prepare a release, follow these steps: 62 | 63 | .. code-block:: bash 64 | 65 | cd neuprint-python 66 | 67 | # Update the change log! 68 | code docs/source/changelog.rst 69 | git commit -m "Updated changelog" docs/source/changelog.rst 70 | 71 | # Do the tests still pass? 72 | pytest . 73 | 74 | # Do the docs still build? 75 | ( 76 | export PYTHONPATH=$(pwd) 77 | cd docs 78 | make html 79 | open build/html/index.html 80 | ) 81 | 82 | # Tag the git repo with the new version 83 | NEW_TAG=0.3.1 84 | git tag -a ${NEW_TAG} -m ${NEW_TAG} 85 | git push --tags origin 86 | 87 | # Build and upload the conda package 88 | conda build conda-recipe 89 | anaconda upload -u flyem-forge $(conda info --base)/conda-bld/noarch/neuprint-python-${NEW_TAG}-py_0.tar.bz2 90 | 91 | # Build and upload the PyPI package 92 | ./upload-to-pypi.sh 93 | 94 | # Deploy the docs 95 | ./docs/deploy-docs.sh 96 | 97 | 98 | Dependencies 99 | ------------ 100 | 101 | If you need to add dependencies to ``neuprint-python``, edit ``dependencies.txt`` (which is used by the conda recipe). 102 | You should also update ``environment.yml`` so that our binder container will acquire the new dependencies 103 | when users try out the interactive `tutorial`_. After publishing a new conda package with the updated dependencies, 104 | follow these steps **on a Linux machine**: 105 | 106 | .. code-block:: bash 107 | 108 | #!/bin/bash 109 | # update-deps.sh 110 | 111 | set -e 112 | 113 | # Create an environment with the binder dependencies 114 | TUTORIAL_DEPS="ipywidgets bokeh holoviews hvplot" 115 | SIMULATION_DEPS="ngspice umap-learn scikit-learn matplotlib" 116 | BINDER_DEPS="neuprint-python jupyterlab ${TUTORIAL_DEPS} ${SIMULATION_DEPS}" 117 | conda create -y -n neuprint-python -c flyem-forge -c conda-forge ${BINDER_DEPS} 118 | 119 | # Export to environment.yml, but relax the neuprint-python version requirement 120 | conda env export -n neuprint-python > environment.yml 121 | sed --in-place 's/neuprint-python=.*/neuprint-python/g' environment.yml 122 | 123 | git commit -m "Updated environment.yml for binder" environment.yml 124 | git push origin master 125 | 126 | 127 | .. _tutorial: notebooks/QueryTutorial.ipynb 128 | 129 | Documentation 130 | ------------- 131 | 132 | The docs are built with Sphinx. See ``docs/requirements.txt`` for the docs dependencies. 133 | To build the docs locally: 134 | 135 | .. code-block:: bash 136 | 137 | cd neuprint-python/docs 138 | make html 139 | open build/html/index.html 140 | 141 | We publish the docs via `github pages `_. 142 | Use the script ``docs/deploy-docs.sh`` to build and publish the docs to GitHub in the `gh-pages` branch. 143 | (At some point in the future, we may automate this via a CI system.) 144 | 145 | .. code-block:: bash 146 | 147 | ./docs/deploy-docs.sh 148 | 149 | 150 | Interactive Tutorial 151 | -------------------- 152 | 153 | The documentation contains a `tutorial`_ which can be launched interactively via binder. 154 | To update the tutorial contents, simply edit the ``.ipynb`` file and re-build the docs. 155 | 156 | If the binder setup is broken, make sure the dependencies are configured properly as described above. 157 | 158 | It takes a few minutes to initialize the binder container for the first time after a new release. 159 | Consider sparing your users from that by clicking the binder button yourself after each release. 160 | 161 | Tests 162 | ----- 163 | 164 | The tests require ``pytest``, and they rely on the public ``hemibrain:v1.2.1`` dataset on ``neuprint.janelia.org``, 165 | which means you must define ``NEUPRINT_APPLICATION_CREDENTIALS`` in your environment before running them. 166 | 167 | To run the tests: 168 | 169 | .. code-block:: bash 170 | 171 | cd neuprint-python 172 | PYTHONPATH=. pytest neuprint/tests 173 | -------------------------------------------------------------------------------- /docs/source/faq.rst: -------------------------------------------------------------------------------- 1 | .. _faq: 2 | 3 | FAQ 4 | === 5 | 6 | Why use this API? Why not just use plain Cypher? 7 | ------------------------------------------------ 8 | 9 | Cypher is a powerful language for querying the neuprint database, 10 | and there will always be some needs that can only be satisfed with 11 | a custom-tailored Cypher query. 12 | 13 | However, there are some advantages that come from using the higher-level 14 | API provided in ``neuprint-python``: 15 | 16 | * To use Cypher, you need an understanding of the neuprint data model. 17 | It's not too complex, but for many users, basic neuron attributes and 18 | connection information is enough. 19 | * Some queries are difficult to specify. For example, efficiently filtering neurons 20 | by ``inputRoi`` or ``outputRoi`` is not trivial. But ``NeuronCriteria`` handles that for you. 21 | * The ``neuprint-python`` API uses reasonable default parameters, 22 | which aren't always obvious in raw Cypher queries. 23 | * ``neuprint-python`` saves you from certain nuisance tasks, like converting ``roiInfo`` 24 | from JSON data into a DataFrame for easy analysis. 25 | * When a query might return a large amount of data, it's often critical to break the query 26 | into batches, to avoid timeouts from the server. For functions in which that is likely to occur, 27 | ``neuprint-python`` implements batching for you. 28 | 29 | Nonetheless, if you need to run a query that isn't conveniently 30 | supported by the high-level API in this library, 31 | or you simply prefer to write your own Cypher, 32 | then feel free to use :py:meth:`.Client.fetch_custom()`. 33 | 34 | 35 | What Cypher queries are being used by this code internally? 36 | ----------------------------------------------------------- 37 | 38 | Enable debug logging to see the cypher queries that are being sent to the neuPrint server. 39 | See :py:func:`.setup_debug_logging()` for details. 40 | 41 | 42 | Where are the release notes for the *data*? 43 | ------------------------------------------- 44 | 45 | Please see the `neuprint dataset release notes and errata `_. 46 | 47 | 48 | Where can I find general information about the FlyEM Hemibrain dataset? 49 | ----------------------------------------------------------------------- 50 | 51 | See the `hemibrain description page `_. 52 | 53 | 54 | I just want the complete connection table for the FlyEM Hemibrain. Can I download that separately? 55 | -------------------------------------------------------------------------------------------------- 56 | 57 | Yes, the complete connection table for all ``Traced`` neurons is available for download. 58 | To find the latest version, see the `hemibrain description page `_, 59 | and find the link to the "compact connection matrix summary". 60 | `Here's a link `_ 61 | to the table we released for version v1.2 (the most recent release at the time of this writing). 62 | 63 | 64 | How can I download the exact Hemibrain ROI shapes? 65 | -------------------------------------------------- 66 | 67 | A volume containing the exact primary ROI region labels for the hemibrain in hdf5 format can be `found here`_. 68 | Please see the enclosed README for details on how to read and interpret the volume. 69 | 70 | .. note:: 71 | 72 | The volume tarball is only 10MB to download, but loading the full uncompressed volume requires 2 GB of RAM. 73 | 74 | .. _found here: https://storage.cloud.google.com/hemibrain/v1.1/hemibrain-v1.1-primary-roi-segmentation.tar.gz 75 | 76 | 77 | Can this library be used with ``multiprocessing``? 78 | -------------------------------------------------- 79 | 80 | Yes. ``neuprint-python``'s mechanism for selecting the "default" client will automatically 81 | copy the default client once per thread/process if necessary. Thus, as long you're not 82 | explicitly passing a ``client`` to any ``neuprint`` queries, your code can be run in 83 | a ``threading`` or ``multiprocessing`` context without special care. 84 | But if you are *not* using the default client, then it's your responsibility to create 85 | a separate client for each thread/process in your program. 86 | (``Client`` objects cannot be shared across threads or processes.) 87 | 88 | .. note:: 89 | 90 | Running many queries in parallel can place a heavy load the neuprint server. 91 | Please be considerate to other users, and limit the number of parallel queries you make. 92 | 93 | 94 | Where can I find help? 95 | ---------------------- 96 | 97 | - Please report issues and feature requests for ``neuprint-python`` on 98 | `github `_. 99 | 100 | - General questions about neuPrint or the hemibrain dataset can be asked on the neuPrint 101 | `Google Groups forum `_. 102 | 103 | - For information about the Cypher query language, see the 104 | `neo4j docs `_. 105 | 106 | - The best way to become acquainted with neuPrint's capabilities and data 107 | model is to experiment with a public neuprint database via the neuprint 108 | web UI. Try exploring the `Janelia FlyEM Hemibrain neuprint database `_. 109 | To see the Cypher query that was used for each result on the site, 110 | click the information icon (shown below). 111 | 112 | .. image:: _static/neuprint-explorer-cypher-button.png 113 | :scale: 25 % 114 | :alt: Neuprint Explorer Cypher Info Button 115 | 116 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | neuprint-python 2 | =============== 3 | 4 | .. _intro: 5 | 6 | 7 | Introduction to neuPrint+ and ``neuprint-python`` 8 | ------------------------------------------------- 9 | 10 | The `neuPrint+ project `_ defines 11 | a graph database structure and suite of tools for storing and analyzing 12 | inter- and intra-cellular interactions. It supports various data analyses, 13 | especially those related to connectomic datasets. 14 | 15 | The best way to become acquainted with neuPrint's capabilities and data 16 | model is to experiment with a public neuprint database via the neuprint 17 | web UI. Try exploring the `Janelia FlyEM Hemibrain neuprint database `_. 18 | 19 | 20 | Once you're familiar with the basics, you're ready to start writing 21 | Python scripts to query the database programmatically with 22 | ``neuprint-python``. 23 | 24 | .. _install: 25 | 26 | Install neuprint-python 27 | ----------------------- 28 | 29 | If you're using `conda `_, use this command: 30 | 31 | 32 | .. code-block:: bash 33 | 34 | conda install -c flyem-forge neuprint-python 35 | 36 | 37 | Otherwise, use ``pip``: 38 | 39 | 40 | .. code-block:: bash 41 | 42 | pip install neuprint-python 43 | 44 | For developers, the ``neuprint-python`` `source code can be found here `_. 45 | 46 | Contents 47 | -------- 48 | 49 | .. toctree:: 50 | :maxdepth: 2 51 | 52 | quickstart 53 | tutorials 54 | api 55 | development 56 | related 57 | faq 58 | changelog -------------------------------------------------------------------------------- /docs/source/mitocriteria.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.queries 2 | 3 | .. _mitocriteria: 4 | 5 | 6 | MitoCriteria 7 | =============== 8 | 9 | .. autoclass:: MitoCriteria 10 | -------------------------------------------------------------------------------- /docs/source/neuroncriteria.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.queries 2 | 3 | .. _neuroncriteria: 4 | 5 | 6 | NeuronCriteria 7 | =============== 8 | 9 | .. autoclass:: NeuronCriteria 10 | 11 | .. autodata:: SegmentCriteria -------------------------------------------------------------------------------- /docs/source/queries.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.queries 2 | 3 | 4 | .. 5 | (The following |br| definition is the only way 6 | I can force numpydoc to display explicit newlines...) 7 | 8 | .. |br| raw:: html 9 | 10 |
11 | 12 | 13 | .. _queries: 14 | 15 | ======= 16 | Queries 17 | ======= 18 | 19 | Convenience functions for common queries. 20 | 21 | If you are familiar with the neuPrint data model and the 22 | `cypher `_ 23 | query language, you can write your own queries using 24 | :py:func:`fetch_custom `. 25 | But the functions in this file offer a convenient API for common queries. 26 | 27 | Server Built-in Queries 28 | ======================= 29 | 30 | See the :ref:`Client` class reference for the neuprint server's built-in 31 | (non-cypher) queries, such as **skeletons**, **ROI meshes**, **ROI connectivity**, 32 | and server metadata. 33 | 34 | General 35 | ======= 36 | 37 | .. autosummary:: 38 | 39 | fetch_custom 40 | fetch_meta 41 | 42 | ROIs 43 | ==== 44 | 45 | .. autosummary:: 46 | 47 | fetch_all_rois 48 | fetch_primary_rois 49 | fetch_roi_hierarchy 50 | 51 | .. seealso:: 52 | 53 | - :py:meth:`.Client.fetch_roi_completeness()` 54 | - :py:meth:`.Client.fetch_roi_connectivity()` 55 | 56 | Neurons 57 | ======= 58 | 59 | .. autosummary:: 60 | 61 | NeuronCriteria 62 | fetch_neurons 63 | fetch_custom_neurons 64 | 65 | Connectivity 66 | ============ 67 | 68 | .. autosummary:: 69 | 70 | fetch_simple_connections 71 | fetch_adjacencies 72 | fetch_traced_adjacencies 73 | fetch_common_connectivity 74 | fetch_shortest_paths 75 | 76 | Synapses 77 | ======== 78 | 79 | .. autosummary:: 80 | 81 | SynapseCriteria 82 | fetch_synapses 83 | fetch_mean_synapses 84 | fetch_synapse_connections 85 | 86 | Mitochondria 87 | ============ 88 | 89 | .. autosummary:: 90 | 91 | MitoCriteria 92 | fetch_mitochondria 93 | fetch_synapses_and_closest_mitochondria 94 | fetch_connection_mitochondria 95 | 96 | Reconstruction Tools 97 | ==================== 98 | 99 | .. autosummary:: 100 | 101 | fetch_output_completeness 102 | fetch_downstream_orphan_tasks 103 | 104 | 105 | Reference 106 | ========= 107 | 108 | .. I can't figure out how to make automodule display these in the 'bysource' order, so I'm specifying the order explicitly. 109 | 110 | General 111 | ------- 112 | 113 | .. autofunction:: fetch_custom 114 | .. autofunction:: fetch_meta 115 | 116 | ROIs 117 | ---- 118 | 119 | .. autofunction:: fetch_all_rois 120 | .. autofunction:: fetch_primary_rois 121 | .. autofunction:: fetch_roi_hierarchy 122 | 123 | Neurons 124 | ------- 125 | 126 | .. autofunction:: fetch_neurons 127 | .. autofunction:: fetch_custom_neurons 128 | 129 | Connectivity 130 | ------------ 131 | 132 | .. autofunction:: fetch_simple_connections 133 | .. autofunction:: fetch_adjacencies 134 | .. autofunction:: fetch_traced_adjacencies 135 | .. autofunction:: fetch_common_connectivity 136 | .. autofunction:: fetch_shortest_paths 137 | 138 | Synapses 139 | -------- 140 | 141 | .. autofunction:: fetch_synapses 142 | .. autofunction:: fetch_mean_synapses 143 | .. autofunction:: fetch_synapse_connections 144 | 145 | Mitochondria 146 | ------------ 147 | 148 | .. autofunction:: fetch_mitochondria 149 | .. autofunction:: fetch_synapses_and_closest_mitochondria 150 | .. autofunction:: fetch_connection_mitochondria 151 | 152 | Reconstruction Tools 153 | -------------------- 154 | 155 | .. autofunction:: fetch_output_completeness 156 | .. autofunction:: fetch_downstream_orphan_tasks 157 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.client 2 | 3 | .. _quickstart: 4 | 5 | Quickstart 6 | ========== 7 | 8 | Install neuprint-python 9 | ----------------------- 10 | 11 | If you're using `conda `_, use this command: 12 | 13 | 14 | .. code-block:: bash 15 | 16 | conda install -c flyem-forge neuprint-python 17 | 18 | 19 | Otherwise, use ``pip``: 20 | 21 | 22 | .. code-block:: bash 23 | 24 | pip install neuprint-python 25 | 26 | Client and Authorization Token 27 | ------------------------------ 28 | 29 | All communication with the ``neuPrintHTTP`` server is done via a :py:class:`Client` object. 30 | 31 | To create a :py:class:`Client`, you must provide three things: 32 | 33 | - The neuprint server address (e.g. ``neuprint.janelia.org``) 34 | - Which dataset you'll be fetching from (e.g. ``hemibrain:v1.2.1``) 35 | - Your personal authentication token 36 | 37 | To obtain your authorization token, follow these steps: 38 | 39 | 1. Navigate your web browser to the neuprint server address. 40 | 2. Log in. 41 | 3. Using the account menu in the upper right-hand corner, select "Account" as shown in the screenshot below. 42 | 4. Copy the entire auth token. 43 | 44 | 45 | .. image:: _static/token-screenshot.png 46 | :scale: 50 % 47 | :alt: Auth Token menu screenshot 48 | 49 | Create the Client 50 | ----------------- 51 | 52 | .. code-block:: python 53 | 54 | from neuprint import Client 55 | 56 | c = Client('neuprint.janelia.org', dataset='hemibrain:v1.2.1', token='YOUR-TOKEN-HERE') 57 | c.fetch_version() 58 | 59 | Alternatively, you can set your token in the following environment variable, in which case the ``token`` parameter can be omitted: 60 | 61 | 62 | .. code-block:: shell 63 | 64 | $ export NEUPRINT_APPLICATION_CREDENTIALS= 65 | 66 | 67 | Execute a query 68 | --------------- 69 | 70 | Use your :py:class:`Client` to request data from neuprint. 71 | 72 | The :py:meth:`Client.fetch_custom()` method will run an arbitrary cypher query against the database. 73 | For information about the neuprint data model, see the `neuprint explorer web help. `_ 74 | 75 | Also, ``neuprint-python`` comes with convenience functions to implement common queries. See :ref:`queries`. 76 | 77 | .. code-block:: ipython 78 | 79 | 80 | In [1]: ## This query will return all neurons in the ROI ‘AB’ 81 | ...: ## that have greater than 10 pre-synaptic sites. 82 | ...: ## Results are ordered by total synaptic sites (pre+post). 83 | ...: q = """\ 84 | ...: MATCH (n :Neuron {`AB(R)`: true}) 85 | ...: WHERE n.pre > 10 86 | ...: RETURN n.bodyId AS bodyId, n.type as type, n.instance AS instance, n.pre AS numpre, n.post AS numpost 87 | ...: ORDER BY n.pre + n.post DESC 88 | ...: """ 89 | 90 | In [2]: results = c.fetch_custom(q) 91 | 92 | In [3]: print(f"Found {len(results)} results") 93 | Found 177 results 94 | 95 | In [4]: results.head() 96 | Out[4]: 97 | bodyId type instance numpre numpost 98 | 0 5813027016 FB4Y FB4Y(EB/NO1)_R 1720 6508 99 | 1 1008378448 FB4Y FB4Y(EB/NO1)_R 1791 6301 100 | 2 1513363614 LCNOpm LCNOpm(LAL-NO3pm)_R 858 6501 101 | 3 5813057274 FB4Y FB4Y(EB/NO1)_L 2001 5089 102 | 4 1131827390 FB4M FB4M(PPM3-FB3/4-NO-DAN)_R 2614 4431 103 | 104 | Next Steps 105 | ---------- 106 | 107 | Try the `interactive tutorial`_ for a tour of basic features in ``neuprint-python``. 108 | 109 | .. _interactive tutorial: notebooks/QueryTutorial.ipynb 110 | -------------------------------------------------------------------------------- /docs/source/related.rst: -------------------------------------------------------------------------------- 1 | .. _related: 2 | 3 | Related Projects 4 | ================ 5 | 6 | 7 | NeuPrint on github 8 | ------------------ 9 | 10 | The `NeuPrint `_ organization on github is home to several 11 | repositories, including `neuprint-python `_. 12 | 13 | - `CBLAST `_ clusters neurons by cell type according to their common connectivity. 14 | - `neuVid `_ is a collection of scripts to generate animations of neuron meshes using Blender. 15 | - `react-skeleton `_ is a ``React`` component to display 16 | neurons skeletons in 3D using `SharkViewer `_. 17 | 18 | 19 | Clio 20 | ---- 21 | 22 | The `Clio `_ project provides tools for creating your own annotations on 23 | public EM datasets, and sharing those annotations with others if you choose to. 24 | There's also a fun image search tool for finding interesting features in the raw EM data. 25 | Additionally, use Clio to make lightweight corrections to the segmentation for your own viewing, 26 | and submit those changes to be included in the public version of the segmentation, too. 27 | 28 | 29 | DVID 30 | ---- 31 | 32 | `DVID `_ provides a versioned storage service for image volumes. 33 | It serves as the primary storage engine for FlyEM's reconstruction efforts. 34 | 35 | 36 | neuprintr 37 | --------- 38 | 39 | `neuprintr `_ provides R users with access to neuprint. 40 | 41 | 42 | NAVis 43 | ----- 44 | 45 | `NAVis `_ supports manipulation 46 | and visualization of neuron morphologies. A tutorial on using NAVis with 47 | ``neuprint-python`` can be found `here `_. 48 | 49 | -------------------------------------------------------------------------------- /docs/source/simulation.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.simulation 2 | 3 | .. _simulation: 4 | 5 | Simulation 6 | ========== 7 | 8 | .. automodule:: neuprint.simulation 9 | 10 | NeuronModel 11 | ----------- 12 | 13 | .. autosummary:: 14 | 15 | NeuronModel 16 | NeuronModel.simulate 17 | NeuronModel.estimate_intra_neuron_delay 18 | 19 | TimingResult 20 | ------------ 21 | 22 | .. autosummary:: 23 | 24 | TimingResult 25 | TimingResult.compute_region_delay_matrix 26 | TimingResult.plot_response_from_region 27 | TimingResult.plot_neuron_domains 28 | TimingResult.estimate_neuron_domains 29 | 30 | .. autoclass:: neuprint.simulation.NeuronModel 31 | :members: 32 | 33 | .. autoclass:: neuprint.simulation.TimingResult 34 | :members: 35 | -------------------------------------------------------------------------------- /docs/source/skeleton.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.skeleton 2 | 3 | 4 | .. 5 | (The following |br| definition is the only way 6 | I can force numpydoc to display explicit newlines...) 7 | 8 | .. |br| raw:: html 9 | 10 |
11 | 12 | 13 | .. _skeleton: 14 | 15 | 16 | Skeletons 17 | ========= 18 | 19 | .. automodule:: neuprint.skeleton 20 | 21 | .. autosummary:: 22 | 23 | fetch_skeleton 24 | heal_skeleton 25 | reorient_skeleton 26 | skeleton_swc_to_df 27 | skeleton_df_to_swc 28 | skeleton_df_to_nx 29 | skeleton_segments 30 | upsample_skeleton 31 | attach_synapses_to_skeleton 32 | 33 | Reference 34 | --------- 35 | 36 | .. autofunction:: fetch_skeleton 37 | .. autofunction:: heal_skeleton 38 | .. autofunction:: reorient_skeleton 39 | .. autofunction:: skeleton_swc_to_df 40 | .. autofunction:: skeleton_df_to_swc 41 | .. autofunction:: skeleton_df_to_nx 42 | .. autofunction:: skeleton_segments 43 | .. autofunction:: upsample_skeleton 44 | .. autofunction:: attach_synapses_to_skeleton 45 | 46 | -------------------------------------------------------------------------------- /docs/source/synapsecriteria.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.queries 2 | 3 | .. _synapsecriteria: 4 | 5 | 6 | SynapseCriteria 7 | =============== 8 | 9 | .. autoclass:: SynapseCriteria 10 | -------------------------------------------------------------------------------- /docs/source/tutorials.rst: -------------------------------------------------------------------------------- 1 | .. _tutorials: 2 | 3 | Tutorials 4 | ========= 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | notebooks/QueryTutorial 10 | notebooks/SimulationTutorial 11 | -------------------------------------------------------------------------------- /docs/source/utils.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.utils 2 | 3 | 4 | .. 5 | (The following |br| definition is the only way 6 | I can force numpydoc to display explicit newlines...) 7 | 8 | .. |br| raw:: html 9 | 10 |
11 | 12 | 13 | .. _utils: 14 | 15 | 16 | Utilities 17 | ========= 18 | 19 | .. automodule:: neuprint.utils 20 | 21 | .. autosummary:: 22 | 23 | merge_neuron_properties 24 | connection_table_to_matrix 25 | 26 | Reference 27 | --------- 28 | 29 | .. autofunction:: merge_neuron_properties 30 | .. autofunction:: connection_table_to_matrix 31 | 32 | -------------------------------------------------------------------------------- /docs/source/wrangle.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: neuprint.wrangle 2 | 3 | 4 | .. 5 | (The following |br| definition is the only way 6 | I can force numpydoc to display explicit newlines...) 7 | 8 | .. |br| raw:: html 9 | 10 |
11 | 12 | 13 | .. _wrangle: 14 | 15 | 16 | Data Wrangling 17 | ============== 18 | 19 | .. automodule:: neuprint.wrangle 20 | 21 | .. autosummary:: 22 | 23 | syndist_matrix 24 | bilateral_syndist 25 | assign_sides_in_groups 26 | 27 | Reference 28 | --------- 29 | 30 | .. autofunction:: syndist_matrix 31 | .. autofunction:: bilateral_syndist 32 | .. autofunction:: assign_sides_in_groups 33 | 34 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: neuprint-python 2 | channels: 3 | - flyem-forge 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - _libgcc_mutex=0.1=conda_forge 8 | - _openmp_mutex=4.5=1_gnu 9 | - alsa-lib=1.2.3=h516909a_0 10 | - anyio=3.4.0=py39hf3d152e_0 11 | - argon2-cffi=21.1.0=py39h3811e60_2 12 | - asciitree=0.3.3=py_2 13 | - async_generator=1.10=py_0 14 | - attrs=21.2.0=pyhd8ed1ab_0 15 | - babel=2.9.1=pyh44b312d_0 16 | - backcall=0.2.0=pyh9f0ad1d_0 17 | - backports=1.0=py_2 18 | - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 19 | - bleach=4.1.0=pyhd8ed1ab_0 20 | - bokeh=2.4.2=py39hf3d152e_0 21 | - brotli=1.0.9=h7f98852_6 22 | - brotli-bin=1.0.9=h7f98852_6 23 | - brotlipy=0.7.0=py39h3811e60_1003 24 | - ca-certificates=2021.10.8=ha878542_0 25 | - certifi=2021.10.8=py39hf3d152e_1 26 | - cffi=1.15.0=py39h4bc2ebd_0 27 | - charset-normalizer=2.0.9=pyhd8ed1ab_0 28 | - colorama=0.4.4=pyh9f0ad1d_0 29 | - colorcet=3.0.0=pyhd8ed1ab_0 30 | - cryptography=36.0.0=py39h95dcef6_0 31 | - cycler=0.11.0=pyhd8ed1ab_0 32 | - dbus=1.13.6=h48d8840_2 33 | - debugpy=1.5.1=py39he80948d_0 34 | - decorator=5.1.0=pyhd8ed1ab_0 35 | - defusedxml=0.7.1=pyhd8ed1ab_0 36 | - entrypoints=0.3=pyhd8ed1ab_1003 37 | - expat=2.4.1=h9c3ff4c_0 38 | - fontconfig=2.13.1=hba837de_1005 39 | - fonttools=4.28.3=py39h3811e60_0 40 | - freetype=2.10.4=h0708190_1 41 | - gettext=0.19.8.1=h73d1719_1008 42 | - glib=2.70.2=h780b84a_0 43 | - glib-tools=2.70.2=h780b84a_0 44 | - gst-plugins-base=1.18.5=hf529b03_2 45 | - gstreamer=1.18.5=h9f60fe5_2 46 | - holoviews=1.14.6=pyhd8ed1ab_0 47 | - hvplot=0.7.3=pyh6c4a22f_0 48 | - icu=68.2=h9c3ff4c_0 49 | - idna=3.1=pyhd3deb0d_0 50 | - importlib-metadata=4.8.2=py39hf3d152e_0 51 | - importlib_resources=5.4.0=pyhd8ed1ab_0 52 | - ipykernel=6.6.0=py39hef51801_0 53 | - ipython=7.30.1=py39hf3d152e_0 54 | - ipython_genutils=0.2.0=py_1 55 | - ipywidgets=7.6.5=pyhd8ed1ab_0 56 | - jbig=2.1=h7f98852_2003 57 | - jedi=0.18.1=py39hf3d152e_0 58 | - jinja2=3.0.3=pyhd8ed1ab_0 59 | - joblib=1.1.0=pyhd8ed1ab_0 60 | - jpeg=9d=h36c2ea0_0 61 | - json5=0.9.5=pyh9f0ad1d_0 62 | - jsonschema=4.2.1=pyhd8ed1ab_1 63 | - jupyter_client=7.1.0=pyhd8ed1ab_0 64 | - jupyter_core=4.9.1=py39hf3d152e_1 65 | - jupyter_server=1.13.0=pyhd8ed1ab_0 66 | - jupyterlab=3.2.4=pyhd8ed1ab_0 67 | - jupyterlab_pygments=0.1.2=pyh9f0ad1d_0 68 | - jupyterlab_server=2.8.2=pyhd8ed1ab_0 69 | - jupyterlab_widgets=1.0.2=pyhd8ed1ab_0 70 | - kiwisolver=1.3.2=py39h1a9c180_1 71 | - krb5=1.19.2=hcc1bbae_3 72 | - lcms2=2.12=hddcbb42_0 73 | - ld_impl_linux-64=2.36.1=hea4e1c9_2 74 | - lerc=3.0=h9c3ff4c_0 75 | - libblas=3.9.0=12_linux64_openblas 76 | - libbrotlicommon=1.0.9=h7f98852_6 77 | - libbrotlidec=1.0.9=h7f98852_6 78 | - libbrotlienc=1.0.9=h7f98852_6 79 | - libcblas=3.9.0=12_linux64_openblas 80 | - libclang=11.1.0=default_ha53f305_1 81 | - libdeflate=1.8=h7f98852_0 82 | - libedit=3.1.20191231=he28a2e2_2 83 | - libevent=2.1.10=h9b69904_4 84 | - libffi=3.4.2=h7f98852_5 85 | - libgcc-ng=11.2.0=h1d223b6_11 86 | - libgfortran-ng=11.2.0=h69a702a_11 87 | - libgfortran5=11.2.0=h5c6108e_11 88 | - libglib=2.70.2=h174f98d_0 89 | - libgomp=11.2.0=h1d223b6_11 90 | - libiconv=1.16=h516909a_0 91 | - liblapack=3.9.0=12_linux64_openblas 92 | - libllvm11=11.1.0=hf817b99_2 93 | - libogg=1.3.4=h7f98852_1 94 | - libopenblas=0.3.18=pthreads_h8fe5266_0 95 | - libopus=1.3.1=h7f98852_1 96 | - libpng=1.6.37=h21135ba_2 97 | - libpq=13.5=hd57d9b9_1 98 | - libsodium=1.0.18=h36c2ea0_1 99 | - libstdcxx-ng=11.2.0=he4da1e4_11 100 | - libtiff=4.3.0=h6f004c6_2 101 | - libuuid=2.32.1=h7f98852_1000 102 | - libvorbis=1.3.7=h9c3ff4c_0 103 | - libwebp-base=1.2.1=h7f98852_0 104 | - libxcb=1.13=h7f98852_1004 105 | - libxkbcommon=1.0.3=he3ba5ed_0 106 | - libxml2=2.9.12=h72842e0_0 107 | - libzlib=1.2.11=h36c2ea0_1013 108 | - llvmlite=0.37.0=py39h1bbdace_1 109 | - lz4-c=1.9.3=h9c3ff4c_1 110 | - markdown=3.3.6=pyhd8ed1ab_0 111 | - markupsafe=2.0.1=py39h3811e60_1 112 | - matplotlib=3.5.0=py39hf3d152e_0 113 | - matplotlib-base=3.5.0=py39h2fa2bec_0 114 | - matplotlib-inline=0.1.3=pyhd8ed1ab_0 115 | - mistune=0.8.4=py39h3811e60_1005 116 | - munkres=1.0.12=pyh8f17d0a_0 117 | - mysql-common=8.0.27=ha770c72_1 118 | - mysql-libs=8.0.27=hfa10184_1 119 | - nbclassic=0.3.4=pyhd8ed1ab_0 120 | - nbclient=0.5.9=pyhd8ed1ab_0 121 | - nbconvert=6.3.0=py39hf3d152e_1 122 | - nbformat=5.1.3=pyhd8ed1ab_0 123 | - ncurses=6.2=h58526e2_4 124 | - nest-asyncio=1.5.4=pyhd8ed1ab_0 125 | - networkx=2.6.3=pyhd8ed1ab_1 126 | - neuprint-python 127 | - ngspice=32=6 128 | - ngspice-exe=32=hcee41ef_6 129 | - ngspice-lib=32=hcee41ef_6 130 | - notebook=6.4.6=pyha770c72_0 131 | - nspr=4.32=h9c3ff4c_1 132 | - nss=3.73=hb5efdd6_0 133 | - numba=0.54.1=py39h56b8d98_0 134 | - numpy=1.20.3=py39hdbf815f_1 135 | - olefile=0.46=pyh9f0ad1d_1 136 | - openjpeg=2.4.0=hb52868f_1 137 | - openssl=1.1.1l=h7f98852_0 138 | - packaging=21.3=pyhd8ed1ab_0 139 | - pandas=1.3.4=py39hde0f152_1 140 | - pandoc=2.16.2=h7f98852_0 141 | - pandocfilters=1.5.0=pyhd8ed1ab_0 142 | - panel=0.12.4=pyhd8ed1ab_0 143 | - param=1.12.0=pyh6c4a22f_0 144 | - parso=0.8.3=pyhd8ed1ab_0 145 | - pcre=8.45=h9c3ff4c_0 146 | - pexpect=4.8.0=pyh9f0ad1d_2 147 | - pickleshare=0.7.5=py_1003 148 | - pillow=8.4.0=py39ha612740_0 149 | - pip=21.3.1=pyhd8ed1ab_0 150 | - prometheus_client=0.12.0=pyhd8ed1ab_0 151 | - prompt-toolkit=3.0.23=pyha770c72_0 152 | - pthread-stubs=0.4=h36c2ea0_1001 153 | - ptyprocess=0.7.0=pyhd3deb0d_0 154 | - pycparser=2.21=pyhd8ed1ab_0 155 | - pyct=0.4.6=py_0 156 | - pyct-core=0.4.6=py_0 157 | - pygments=2.10.0=pyhd8ed1ab_0 158 | - pynndescent=0.5.5=pyh6c4a22f_0 159 | - pyopenssl=21.0.0=pyhd8ed1ab_0 160 | - pyparsing=3.0.6=pyhd8ed1ab_0 161 | - pyqt=5.12.3=py39hf3d152e_8 162 | - pyqt-impl=5.12.3=py39hde8b62d_8 163 | - pyqt5-sip=4.19.18=py39he80948d_8 164 | - pyqtchart=5.12=py39h0fcd23e_8 165 | - pyqtwebengine=5.12.1=py39h0fcd23e_8 166 | - pyrsistent=0.18.0=py39h3811e60_0 167 | - pysocks=1.7.1=py39hf3d152e_4 168 | - python=3.9.7=hb7a2778_3_cpython 169 | - python-dateutil=2.8.2=pyhd8ed1ab_0 170 | - python_abi=3.9=2_cp39 171 | - pytz=2021.3=pyhd8ed1ab_0 172 | - pyviz_comms=2.1.0=pyhd8ed1ab_0 173 | - pyyaml=6.0=py39h3811e60_3 174 | - pyzmq=22.3.0=py39h37b5a0c_1 175 | - qt=5.12.9=hda022c4_4 176 | - readline=8.1=h46c0cb4_0 177 | - requests=2.26.0=pyhd8ed1ab_1 178 | - scikit-learn=1.0.1=py39h4dfa638_2 179 | - scipy=1.7.3=py39hee8e79c_0 180 | - send2trash=1.8.0=pyhd8ed1ab_0 181 | - setuptools=59.4.0=py39hf3d152e_0 182 | - six=1.16.0=pyh6c4a22f_0 183 | - sniffio=1.2.0=py39hf3d152e_2 184 | - sqlite=3.37.0=h9cd32fc_0 185 | - tbb=2021.4.0=h4bd325d_1 186 | - terminado=0.12.1=py39hf3d152e_1 187 | - testpath=0.5.0=pyhd8ed1ab_0 188 | - threadpoolctl=3.0.0=pyh8a188c0_0 189 | - tk=8.6.11=h27826a3_1 190 | - tornado=6.1=py39h3811e60_2 191 | - tqdm=4.62.3=pyhd8ed1ab_0 192 | - traitlets=5.1.1=pyhd8ed1ab_0 193 | - typing_extensions=4.0.1=pyha770c72_0 194 | - tzdata=2021e=he74cb21_0 195 | - ujson=4.2.0=py39he80948d_1 196 | - umap-learn=0.5.2=py39hf3d152e_0 197 | - urllib3=1.26.7=pyhd8ed1ab_0 198 | - wcwidth=0.2.5=pyh9f0ad1d_2 199 | - webencodings=0.5.1=py_1 200 | - websocket-client=1.2.3=pyhd8ed1ab_0 201 | - wheel=0.37.0=pyhd8ed1ab_1 202 | - widgetsnbextension=3.5.2=py39hf3d152e_1 203 | - xorg-kbproto=1.0.7=h7f98852_1002 204 | - xorg-libice=1.0.10=h7f98852_0 205 | - xorg-libsm=1.2.3=hd9c2040_1000 206 | - xorg-libx11=1.6.12=h36c2ea0_0 207 | - xorg-libxau=1.0.9=h7f98852_0 208 | - xorg-libxaw=1.0.14=h7f98852_0 209 | - xorg-libxdmcp=1.1.3=h7f98852_0 210 | - xorg-libxext=1.3.4=h516909a_0 211 | - xorg-libxmu=1.1.3=h516909a_0 212 | - xorg-libxpm=3.5.13=h516909a_0 213 | - xorg-libxt=1.1.5=h516909a_1003 214 | - xorg-xextproto=7.3.0=h7f98852_1002 215 | - xorg-xproto=7.0.31=h7f98852_1007 216 | - xz=5.2.5=h516909a_1 217 | - yaml=0.2.5=h516909a_0 218 | - zeromq=4.3.4=h9c3ff4c_1 219 | - zipp=3.6.0=pyhd8ed1ab_0 220 | - zlib=1.2.11=h36c2ea0_1013 221 | - zstd=1.5.0=ha95c52a_0 222 | prefix: /groups/flyem/proj/cluster/miniforge/envs/neuprint-python 223 | -------------------------------------------------------------------------------- /neuprint/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | 4 | from .client import Client, default_client, set_default_client, clear_default_client, list_all_clients 5 | from .queries import ( fetch_custom, fetch_meta, fetch_all_rois, fetch_primary_rois, fetch_roi_hierarchy, 6 | fetch_neurons, fetch_custom_neurons, fetch_simple_connections, fetch_adjacencies, 7 | fetch_traced_adjacencies, fetch_common_connectivity, fetch_shortest_paths, 8 | fetch_mitochondria, fetch_synapses_and_closest_mitochondria, fetch_connection_mitochondria, 9 | fetch_synapses, fetch_mean_synapses, fetch_synapse_connections, fetch_output_completeness, 10 | fetch_downstream_orphan_tasks, 11 | NeuronCriteria, SegmentCriteria, SynapseCriteria, MitoCriteria ) 12 | from .utils import merge_neuron_properties, connection_table_to_matrix, IsNull, NotNull 13 | from .simulation import ( NeuronModel, TimingResult, Ra_LOW, Ra_MED, Ra_HIGH, Rm_LOW, Rm_MED, Rm_HIGH ) 14 | from .skeleton import ( fetch_skeleton, skeleton_df_to_nx, skeleton_swc_to_df, skeleton_df_to_swc, heal_skeleton, 15 | reorient_skeleton, calc_segment_distances, skeleton_segments, upsample_skeleton, 16 | attach_synapses_to_skeleton) 17 | from .wrangle import syndist_matrix, bilateral_syndist, assign_sides_in_groups 18 | 19 | from . import _version 20 | __version__ = _version.get_versions()['version'] 21 | 22 | # On Mac, requests uses a system library which is not fork-safe, 23 | # so using multiprocessing results in segfaults such as the following: 24 | # 25 | # File ".../lib/python3.7/urllib/request.py", line 2588 in proxy_bypass_macosx_sysconf 26 | # File ".../lib/python3.7/urllib/request.py", line 2612 in proxy_bypass 27 | # File ".../lib/python3.7/site-packages/requests/utils.py", line 745 in should_bypass_proxies 28 | # File ".../lib/python3.7/site-packages/requests/utils.py", line 761 in get_environ_proxies 29 | # File ".../lib/python3.7/site-packages/requests/sessions.py", line 700 in merge_environment_settings 30 | # File ".../lib/python3.7/site-packages/requests/sessions.py", line 524 in request 31 | # File ".../lib/python3.7/site-packages/requests/sessions.py", line 546 in get 32 | # ... 33 | 34 | # The workaround is to set a special environment variable 35 | # to avoid the particular system function in question. 36 | # Details here: 37 | # https://bugs.python.org/issue30385 38 | if platform.system() == "Darwin": 39 | os.environ["no_proxy"] = "*" 40 | -------------------------------------------------------------------------------- /neuprint/admin.py: -------------------------------------------------------------------------------- 1 | """ 2 | Administration utilities for managing a neuprint server. 3 | Using these tools requires admin privileges on the neuPrintHttp server. 4 | """ 5 | import re 6 | from requests import HTTPError 7 | from .client import inject_client 8 | 9 | 10 | class Transaction: 11 | """ 12 | For admins only. 13 | Used to batch a set of operations into a single database transaction. 14 | 15 | This class is implemented as a context manager. 16 | 17 | Example: 18 | 19 | .. code-block:: python 20 | 21 | with Transaction('hemibrain') as t: 22 | t.query("MATCH (n :Neuron {bodyId: 1047426385}) SET m.type=TuBu4)") 23 | """ 24 | @inject_client 25 | def __init__(self, dataset, *, client=None): 26 | """ 27 | Transaction constructor. 28 | 29 | Args: 30 | dataset: 31 | Name of the dataset to use. Required. 32 | 33 | client: 34 | Client object to use. 35 | """ 36 | # This requirement isn't technically necessary, 37 | # but hopefully it avoids some confusing mistakes. 38 | if client.dataset and client.dataset != dataset: 39 | msg = ("The dataset you provided does not match the client's dataset.\n" 40 | "To avoid confusion, provide a client whose dataset matches the transaction dataset.") 41 | raise RuntimeError(msg) 42 | 43 | assert dataset, \ 44 | "Transactions require an an explicit dataset." 45 | self.dataset = dataset 46 | self.client = client 47 | self.transaction_id = None 48 | self.killed = False 49 | 50 | def query(self, cypher, format='pandas'): 51 | """ 52 | Make a custom cypher query within the context 53 | of this transaction (allows writes). 54 | """ 55 | if self.transaction_id is None: 56 | raise RuntimeError("no transaction was created") 57 | 58 | url = f"{self.client.server}/api/raw/cypher/transaction/{self.transaction_id}/cypher" 59 | return self.client._fetch_cypher(url, cypher, self.dataset, format) 60 | 61 | def kill(self): 62 | """ 63 | Kills (rolls back) transaction. 64 | """ 65 | if self.transaction_id is None: 66 | raise RuntimeError("no transaction was created") 67 | 68 | url = f"{self.client.server}/api/raw/cypher/transaction/{self.transaction_id}/kill" 69 | self.client._fetch_json(url, ispost=True) 70 | self.killed = True 71 | self.transaction_id = None 72 | 73 | def __enter__(self): 74 | self._start() 75 | return self 76 | 77 | def __exit__(self, exc_type, exc_value, traceback): 78 | if self.killed: 79 | return 80 | 81 | if exc_type is None: 82 | self._commit() 83 | return 84 | 85 | if self.transaction_id is None: 86 | return 87 | 88 | try: 89 | self.kill() 90 | except HTTPError as ex: 91 | # We intentionally ignore 'unrecognized transaction id' and 'has been terminated' 92 | # because these imply that the transaction has already failed or has been killed. 93 | ignore = ( 94 | ex.response.status_code == 400 and 95 | re.match(r'(unrecognized transaction id)|(has been terminated)', 96 | ex.response.content.decode('utf-8').lower()) 97 | ) 98 | if not ignore: 99 | raise ex from exc_value 100 | 101 | def _start(self): 102 | try: 103 | url = f"{self.client.server}/api/raw/cypher/transaction" 104 | result = self.client._fetch_json(url, json={"dataset": self.dataset}, ispost=True) 105 | self.transaction_id = result["transaction_id"] 106 | except HTTPError as ex: 107 | if ex.response.status_code == 401: 108 | raise RuntimeError( 109 | "Transaction request was denied. " 110 | "Do you have admin privileges on the neuprintHttp server " 111 | f"({self.client.server})?" 112 | ) from ex 113 | raise 114 | 115 | def _commit(self): 116 | if self.transaction_id is None: 117 | raise RuntimeError("no transaction was created") 118 | url = f"{self.client.server}/api/raw/cypher/transaction/{self.transaction_id}/commit" 119 | self.client._fetch_json(url, ispost=True) 120 | self.transaction_id = None 121 | -------------------------------------------------------------------------------- /neuprint/plotting.py: -------------------------------------------------------------------------------- 1 | """ 2 | Miscellaneous plotting functions. 3 | 4 | 5 | Note: 6 | These functions require additional dependencies, 7 | which aren't listed by default dependencies of neuprint-python. 8 | (See docs for each function.) 9 | """ 10 | import numpy as np 11 | import pandas as pd 12 | 13 | from .client import inject_client 14 | from .skeleton import skeleton_df_to_nx 15 | 16 | def plot_soma_projections(neurons_df, color_by='cellBodyFiber'): 17 | """ 18 | Plot the soma locations as XY, XZ, and ZY 2D projections, 19 | colored by the given column. 20 | 21 | Requires ``bokeh``. 22 | 23 | Returns a layout which can be displayed 24 | with ``bokeh.plotting.show()``. 25 | 26 | Example: 27 | 28 | .. code-block: python 29 | 30 | from neuprint import fetch_neurons, NeuronCriteria as NC 31 | from bokeh.plotting import output_notebook 32 | output_notebook() 33 | 34 | criteria = NC(status='Traced', cropped=False) 35 | neurons_df, _roi_counts_df = fetch_neurons(criteria) 36 | p = plot_soma_projections(neurons_df, 'cellBodyFiber') 37 | show(p) 38 | 39 | """ 40 | import bokeh 41 | from bokeh.plotting import figure 42 | from bokeh.layouts import gridplot 43 | 44 | neurons_df = neurons_df[['somaLocation', color_by]].copy() 45 | 46 | extract_soma_coords(neurons_df) 47 | assign_colors(neurons_df, color_by) 48 | 49 | neurons_with_soma_df = neurons_df.query('not somaLocation.isnull()') 50 | def soma_projection(axis1, axis2, flip1, flip2): 51 | x = neurons_with_soma_df[f'soma_{axis1}'].values 52 | y = neurons_with_soma_df[f'soma_{axis2}'].values 53 | p = figure(title=f'{axis1}{axis2}') 54 | p.scatter(x, y, color=neurons_with_soma_df['color']) 55 | p.x_range.flipped = flip1 56 | p.y_range.flipped = flip2 57 | p.toolbar.logo = None 58 | return p 59 | 60 | p_xy = soma_projection('x', 'y', False, True) 61 | p_xz = soma_projection('x', 'z', False, True) 62 | p_zy = soma_projection('z', 'y', True, True) 63 | 64 | # This will produce one big plot with a shared toolbar 65 | layout = gridplot([[p_xy, p_xz], [None, p_zy]]) 66 | 67 | # Discard the help buttons and bokeh logo 68 | tbar = layout.children[0].toolbar 69 | tbar.logo = None 70 | tbar.tools = [t for t in tbar.tools if not isinstance(t, bokeh.models.tools.HelpTool)] 71 | 72 | return layout 73 | 74 | 75 | def plot_soma_3d(neurons_df, color_by='cellBodyFiber', point_size=1.0): 76 | """ 77 | Plot the soma locations in 3D, colored randomly according 78 | to the column given in ``color_by``. 79 | 80 | Requires ``ipyvolume``. 81 | If using Jupyterlab, install it like this: 82 | 83 | .. code-block: bash 84 | 85 | conda install -c conda-forge ipyvolume 86 | jupyter labextension install ipyvolume 87 | 88 | Example: 89 | 90 | .. code-block: python 91 | 92 | from neuprint import fetch_neurons, NeuronCriteria as NC 93 | 94 | criteria = NC(status='Traced', cropped=False) 95 | neurons_df, _roi_counts_df = fetch_neurons(criteria) 96 | plot_soma_3d(neurons_df, 'cellBodyFiber') 97 | """ 98 | import ipyvolume.pylab as ipv 99 | neurons_df = neurons_df[['somaLocation', color_by]].copy() 100 | 101 | extract_soma_coords(neurons_df) 102 | assign_colors(neurons_df, color_by) 103 | 104 | neurons_with_soma_df = neurons_df.query('not somaLocation.isnull()') 105 | assert neurons_with_soma_df.eval('color.isnull()').sum() == 0 106 | 107 | soma_x = neurons_with_soma_df['soma_x'].values 108 | soma_y = neurons_with_soma_df['soma_y'].values 109 | soma_z = neurons_with_soma_df['soma_z'].values 110 | 111 | def color_to_vals(color_string): 112 | # Convert bokeh color string into float tuples, 113 | # e.g. '#00ff00' -> (0.0, 1.0, 0.0) 114 | s = color_string 115 | return (int(s[1:3], 16) / 255, 116 | int(s[3:5], 16) / 255, 117 | int(s[5:7], 16) / 255 ) 118 | 119 | color_vals = neurons_with_soma_df['color'].apply(color_to_vals).tolist() 120 | 121 | # DVID coordinate system assumes (0,0,0) is in the upper-left. 122 | # For consistency with DVID and neuroglancer conventions, 123 | # we invert the Y and X coordinates. 124 | ipv.figure() 125 | ipv.scatter(soma_x, -soma_y, -soma_z, color=color_vals, marker="circle_2d", size=point_size) 126 | ipv.show() 127 | 128 | 129 | @inject_client 130 | def plot_skeleton_3d(skeleton, color='blue', *, client=None): 131 | """ 132 | Plot the given skeleton in 3D. 133 | 134 | Args: 135 | skeleton: 136 | Either a bodyId or a pre-fetched pandas DataFrame 137 | 138 | color: 139 | See ``ipyvolume`` docs. 140 | Examples: ``'blue'``, ``'#0000ff'`` 141 | If the skeleton is fragmented, you can give a list 142 | of colors and each fragment will be shown in a 143 | different color. 144 | 145 | Requires ``ipyvolume``. 146 | If using Jupyterlab, install it like this: 147 | 148 | .. code-block: bash 149 | 150 | conda install -c conda-forge ipyvolume 151 | jupyter labextension install ipyvolume 152 | """ 153 | import ipyvolume.pylab as ipv 154 | 155 | if np.issubdtype(type(skeleton), np.integer): 156 | skeleton = client.fetch_skeleton(skeleton, format='pandas') 157 | 158 | assert isinstance(skeleton, pd.DataFrame) 159 | g = skeleton_df_to_nx(skeleton) 160 | 161 | def skel_path(root): 162 | """ 163 | We want to plot each skeleton fragment as a single continuous line, 164 | but that means we have to backtrack: parent -> leaf -> parent 165 | to avoid jumping from one branch to another. 166 | This means that the line will be drawn on top of itself, 167 | and we'll have 2x as many line segments in the plot, 168 | but that's not a big deal. 169 | """ 170 | def accumulate_points(n): 171 | p = (g.nodes[n]['x'], g.nodes[n]['y'], g.nodes[n]['z']) 172 | points.append(p) 173 | 174 | children = [*g.successors(n)] 175 | if not children: 176 | return 177 | for c in children: 178 | accumulate_points(c) 179 | points.append(p) 180 | 181 | points = [] 182 | accumulate_points(root) 183 | return np.asarray(points) 184 | 185 | # Skeleton may contain multiple fragments, 186 | # so compute the path for each one. 187 | def skel_paths(df): 188 | paths = [] 189 | for root in df.query('link == -1')['rowId']: 190 | paths.append(skel_path(root)) 191 | return paths 192 | 193 | paths = skel_paths(skeleton) 194 | if isinstance(color, str): 195 | colors = len(paths)*[color] 196 | else: 197 | colors = (1+len(paths)//len(color))*color 198 | 199 | ipv.figure() 200 | for points, color in zip(paths, colors): 201 | ipv.plot(*points.transpose(), color) 202 | ipv.show() 203 | 204 | 205 | def extract_soma_coords(neurons_df): 206 | """ 207 | Expand the ``somaLocation`` column into three separate 208 | columns for ``soma_x``, ``soma_y``, and ``soma_z``. 209 | 210 | If ``somaLocation is None``, then the soma coords will be ``NaN``. 211 | 212 | Works in-place. 213 | """ 214 | neurons_df['soma_x'] = neurons_df['soma_y'] = neurons_df['soma_z'] = np.nan 215 | 216 | somaloc = neurons_df.query('not somaLocation.isnull()')['somaLocation'] 217 | somaloc_array = np.asarray(somaloc.tolist()) 218 | 219 | neurons_df.loc[somaloc.index, 'soma_x'] = somaloc_array[:, 0] 220 | neurons_df.loc[somaloc.index, 'soma_y'] = somaloc_array[:, 1] 221 | neurons_df.loc[somaloc.index, 'soma_z'] = somaloc_array[:, 2] 222 | 223 | 224 | def assign_colors(neurons_df, color_by='cellBodyFiber'): 225 | """ 226 | Use a random colortable to assign a color to each row, 227 | according to the column given in ``color_by``. 228 | 229 | NaN values are always black. 230 | 231 | Works in-place. 232 | """ 233 | from bokeh.palettes import Turbo256 234 | colors = list(Turbo256) 235 | colors[0] = '#000000' 236 | color_categories = np.sort(neurons_df[color_by].fillna('').unique()) 237 | assert color_categories[0] == '' 238 | 239 | np.random.seed(0) 240 | np.random.shuffle(color_categories[1:]) 241 | assert color_categories[0] == '' 242 | 243 | while len(colors) < len(color_categories): 244 | colors.extend(colors[1:]) 245 | 246 | color_mapping = dict(zip(color_categories, colors)) 247 | neurons_df['color'] = neurons_df[color_by].fillna('').map(color_mapping) 248 | 249 | -------------------------------------------------------------------------------- /neuprint/queries/__init__.py: -------------------------------------------------------------------------------- 1 | from .neuroncriteria import NeuronCriteria, SegmentCriteria 2 | from .mitocriteria import MitoCriteria 3 | from .synapsecriteria import SynapseCriteria 4 | from .general import fetch_custom, fetch_meta 5 | from .rois import fetch_all_rois, fetch_primary_rois,fetch_roi_hierarchy 6 | from .neurons import fetch_neurons, fetch_custom_neurons 7 | from .connectivity import (fetch_simple_connections, fetch_adjacencies, fetch_traced_adjacencies, 8 | fetch_common_connectivity, fetch_shortest_paths) 9 | from .synapses import fetch_synapses, fetch_mean_synapses, fetch_synapse_connections 10 | from .mito import fetch_mitochondria, fetch_synapses_and_closest_mitochondria, fetch_connection_mitochondria 11 | from .recon import fetch_output_completeness, fetch_downstream_orphan_tasks 12 | 13 | # Change the __module__ of each function to make it look like it was defined in this file. 14 | # This hackery is to make the Sphinx autosummary and autodoc play nicely together. 15 | for k, v in dict(locals()).items(): 16 | if k.startswith('fetch'): 17 | v.__module__ = 'neuprint.queries' 18 | del k, v 19 | -------------------------------------------------------------------------------- /neuprint/queries/general.py: -------------------------------------------------------------------------------- 1 | from ..client import inject_client 2 | 3 | 4 | @inject_client 5 | def fetch_custom(cypher, dataset="", format='pandas', *, client=None): 6 | ''' 7 | Make a custom cypher query. 8 | 9 | Alternative form of :py:meth:`.Client.fetch_custom()`, as a free function. 10 | That is, ``fetch_custom(..., client=c)`` is equivalent to ``c.fetch_custom(...)``. 11 | 12 | If ``client=None``, the default ``Client`` is used 13 | (assuming you have created at least one ``Client``.) 14 | 15 | Args: 16 | cypher: 17 | A cypher query string 18 | 19 | dataset: 20 | *Deprecated. Please provide your dataset as a Client constructor argument.* 21 | 22 | Which neuprint dataset to query against. 23 | If None provided, the client's default dataset is used. 24 | 25 | format: 26 | Either 'pandas' or 'json'. 27 | Whether to load the results into a pandas DataFrame, 28 | or return the server's raw json response as a Python dict. 29 | 30 | client: 31 | If not provided, the global default :py:class:`.Client` will be used. 32 | 33 | Returns: 34 | Either json or DataFrame, depending on ``format``. 35 | 36 | .. code-block:: ipython 37 | 38 | In [4]: from neuprint import fetch_custom 39 | ...: 40 | ...: q = """\\ 41 | ...: MATCH (n:Neuron) 42 | ...: WHERE n.bodyId = 5813027016 43 | ...: RETURN n.type, n.instance 44 | ...: """ 45 | ...: fetch_custom(q) 46 | Out[4]: 47 | n.type n.instance 48 | 0 FB4Y FB4Y(EB/NO1)_R 49 | ''' 50 | return client.fetch_custom(cypher, dataset, format) 51 | 52 | 53 | @inject_client 54 | def fetch_meta(*, client=None): 55 | """ 56 | Fetch the dataset metadata. 57 | Parses json fields as needed. 58 | 59 | Returns: 60 | dict 61 | 62 | Example 63 | 64 | .. code-block:: ipython 65 | 66 | In [1]: from neuprint import fetch_meta 67 | 68 | In [2]: meta = fetch_meta() 69 | 70 | In [3]: list(meta.keys()) 71 | Out[3]: 72 | ['dataset', 73 | 'info', 74 | 'lastDatabaseEdit', 75 | 'latestMutationId', 76 | 'logo', 77 | 'meshHost', 78 | 'neuroglancerInfo', 79 | 'neuroglancerMeta', 80 | 'postHPThreshold', 81 | 'postHighAccuracyThreshold', 82 | 'preHPThreshold', 83 | 'primaryRois', 84 | 'roiHierarchy', 85 | 'roiInfo', 86 | 'statusDefinitions', 87 | 'superLevelRois', 88 | 'tag', 89 | 'totalPostCount', 90 | 'totalPreCount', 91 | 'uuid'] 92 | """ 93 | q = """\ 94 | MATCH (m:Meta) 95 | WITH m as m, 96 | apoc.convert.fromJsonMap(m.roiInfo) as roiInfo, 97 | apoc.convert.fromJsonMap(m.roiHierarchy) as roiHierarchy, 98 | apoc.convert.fromJsonMap(m.neuroglancerInfo) as neuroglancerInfo, 99 | apoc.convert.fromJsonList(m.neuroglancerMeta) as neuroglancerMeta, 100 | apoc.convert.fromJsonMap(m.statusDefinitions) as statusDefinitions 101 | RETURN m as meta, roiInfo, roiHierarchy, neuroglancerInfo, neuroglancerMeta, statusDefinitions 102 | """ 103 | df = client.fetch_custom(q) 104 | meta = df['meta'].iloc[0] 105 | meta.update(df.drop(columns='meta').iloc[0].to_dict()) 106 | return meta 107 | -------------------------------------------------------------------------------- /neuprint/queries/mito.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import copy 3 | from textwrap import dedent 4 | 5 | import numpy as np 6 | import pandas as pd 7 | 8 | from ..client import inject_client 9 | from ..utils import tqdm, iter_batches 10 | from .neuroncriteria import neuroncriteria_args 11 | from .synapsecriteria import SynapseCriteria 12 | from .mitocriteria import MitoCriteria 13 | from .synapses import fetch_synapse_connections 14 | 15 | 16 | @inject_client 17 | @neuroncriteria_args('neuron_criteria') 18 | def fetch_mitochondria(neuron_criteria, mito_criteria=None, batch_size=10, *, client=None): 19 | """ 20 | Fetch mitochondria from a neuron or selection of neurons. 21 | 22 | Args: 23 | 24 | neuron_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 25 | Determines which bodies from which to fetch mitochondria. 26 | 27 | Note: 28 | Any ROI criteria specified in this argument does not affect 29 | which mitochondria are returned, only which bodies are inspected. 30 | 31 | mito_criteria (MitoCriteria): 32 | Optional. Allows you to filter mitochondria by roi, mitoType, size. 33 | See :py:class:`.MitoCriteria` for details. 34 | 35 | If the criteria specifies ``primary_only=True`` only primary ROIs will be returned in the results. 36 | If a mitochondrion does not intersect any primary ROI, it will be listed with an roi of ``None``. 37 | (Since 'primary' ROIs do not overlap, each mitochondrion will be listed only once.) 38 | Otherwise, all ROI names will be included in the results. 39 | In that case, some mitochondria will be listed multiple times -- once per intersecting ROI. 40 | If a mitochondria does not intersect any ROI, it will be listed with an roi of ``None``. 41 | 42 | batch_size: 43 | To improve performance and avoid timeouts, the mitochondria for multiple bodies 44 | will be fetched in batches, where each batch corresponds to N bodies. 45 | 46 | client: 47 | If not provided, the global default :py:class:`.Client` will be used. 48 | 49 | Returns: 50 | 51 | DataFrame in which each row represent a single synapse. 52 | If ``primary_only=False`` was specified in ``mito_criteria``, some mitochondria 53 | may be listed more than once, if they reside in more than one overlapping ROI. 54 | 55 | Example: 56 | 57 | .. code-block:: ipython 58 | 59 | In [1]: from neuprint import NeuronCriteria as NC, SynapseCriteria as SC, fetch_synapses 60 | ...: 61 | ...: # Consider only neurons which innervate EB 62 | ...: nc = NC(type='ExR.*', rois=['EB']) 63 | ...: 64 | ...: # But return only large mitos from those neurons that reside in the FB or LAL(R) 65 | ...: mc = MC(rois=['FB', 'LAL(R)'], size=100_000) 66 | ...: fetch_mitochondria(nc, mc) 67 | Out[1]: 68 | bodyId mitoType roi x y z size r0 r1 r2 69 | 0 1136865339 dark LAL(R) 15094 30538 23610 259240 101.586632 31.482559 0.981689 70 | 1 1136865339 dark LAL(R) 14526 30020 23464 297784 67.174950 36.328964 0.901079 71 | 2 1136865339 dark LAL(R) 15196 30386 23336 133168 54.907104 25.761894 0.912385 72 | 3 1136865339 dark LAL(R) 14962 30126 23184 169776 66.780258 27.168915 0.942389 73 | 4 1136865339 dark LAL(R) 15004 30252 23164 148528 69.316467 24.082989 0.951892 74 | ... ... ... ... ... ... ... ... ... ... ... 75 | 2807 1259386264 dark FB 18926 24632 21046 159184 99.404472 21.919170 0.984487 76 | 2808 1259386264 dark FB 22162 24474 22486 127968 94.380531 20.547171 0.985971 77 | 2809 1259386264 medium FB 19322 24198 21952 1110888 116.050323 66.010017 0.954467 78 | 2810 1259386264 dark FB 19272 23632 21728 428168 87.865768 40.370171 0.944690 79 | 2811 1259386264 dark FB 19208 23442 21602 141928 53.694149 29.956501 0.919831 80 | 81 | [2812 rows x 10 columns] 82 | """ 83 | mito_criteria = copy.copy(mito_criteria) or MitoCriteria() 84 | mito_criteria.matchvar = 'm' 85 | neuron_criteria.matchvar = 'n' 86 | 87 | q = f""" 88 | {neuron_criteria.global_with(prefix=8)} 89 | MATCH (n:{neuron_criteria.label}) 90 | {neuron_criteria.all_conditions(prefix=8)} 91 | RETURN n.bodyId as bodyId 92 | """ 93 | bodies = client.fetch_custom(q)['bodyId'].values 94 | 95 | batch_dfs = [] 96 | for batch_bodies in tqdm(iter_batches(bodies, batch_size)): 97 | batch_criteria = copy.copy(neuron_criteria) 98 | batch_criteria.bodyId = batch_bodies 99 | batch_df = _fetch_mitos(batch_criteria, mito_criteria, client) 100 | if len(batch_df) > 0: 101 | batch_dfs.append( batch_df ) 102 | 103 | if batch_dfs: 104 | return pd.concat( batch_dfs, ignore_index=True ) 105 | 106 | # Return empty results, but with correct dtypes 107 | dtypes = { 108 | 'bodyId': np.dtype('int64'), 109 | 'mitoType': np.dtype('O'), 110 | 'roi': np.dtype('O'), 111 | 'x': np.dtype('int32'), 112 | 'y': np.dtype('int32'), 113 | 'z': np.dtype('int32'), 114 | 'size': np.dtype('int32'), 115 | 'r0': np.dtype('float32'), 116 | 'r1': np.dtype('float32'), 117 | 'r2': np.dtype('float32'), 118 | } 119 | 120 | return pd.DataFrame([], columns=dtypes.keys()).astype(dtypes) 121 | 122 | 123 | def _fetch_mitos(neuron_criteria, mito_criteria, client): 124 | if mito_criteria.primary_only: 125 | return_rois = {*client.primary_rois} 126 | else: 127 | return_rois = {*client.all_rois} 128 | 129 | # If the user specified rois to filter mitos by, but hasn't specified rois 130 | # in the NeuronCriteria, add them to the NeuronCriteria to speed up the query. 131 | if mito_criteria.rois and not neuron_criteria.rois: 132 | neuron_criteria.rois = {*mito_criteria.rois} 133 | neuron_criteria.roi_req = 'any' 134 | 135 | # Fetch results 136 | cypher = dedent(f"""\ 137 | {neuron_criteria.global_with(prefix=8)} 138 | MATCH (n:{neuron_criteria.label}) 139 | {neuron_criteria.all_conditions('n', prefix=8)} 140 | 141 | MATCH (n)-[:Contains]->(:ElementSet)-[:Contains]->(m:Element {{type: "mitochondrion"}}) 142 | 143 | {mito_criteria.condition('n', 'm', prefix=8)} 144 | 145 | RETURN n.bodyId as bodyId, 146 | m.mitoType as mitoType, 147 | m.size as size, 148 | m.location.x as x, 149 | m.location.y as y, 150 | m.location.z as z, 151 | m.r0 as r0, 152 | m.r1 as r1, 153 | m.r2 as r2, 154 | apoc.map.removeKeys(m, ['location', 'type', 'mitoType', 'size', 'r0', 'r1', 'r2']) as mito_info 155 | """) 156 | data = client.fetch_custom(cypher, format='json')['data'] 157 | 158 | # Assemble DataFrame 159 | mito_table = [] 160 | for body, mitoType, size, x, y, z, r0, r1, r2, mito_info in data: 161 | # Exclude non-primary ROIs if necessary 162 | mito_rois = return_rois & {*mito_info.keys()} 163 | # Fixme: Filter for the user's ROIs (drop duplicates) 164 | for roi in mito_rois: 165 | mito_table.append((body, mitoType, roi, x, y, z, size, r0, r1, r2)) 166 | 167 | if not mito_rois: 168 | mito_table.append((body, mitoType, None, x, y, z, size, r0, r1, r2)) 169 | 170 | cols = ['bodyId', 'mitoType', 'roi', 'x', 'y', 'z', 'size', 'r0', 'r1', 'r2'] 171 | mito_df = pd.DataFrame(mito_table, columns=cols) 172 | 173 | # Save RAM with smaller dtypes and interned strings 174 | mito_df['mitoType'] = mito_df['mitoType'].apply(lambda s: sys.intern(s) if s else s) 175 | mito_df['roi'] = mito_df['roi'].apply(lambda s: sys.intern(s) if s else s) 176 | mito_df['x'] = mito_df['x'].astype(np.int32) 177 | mito_df['y'] = mito_df['y'].astype(np.int32) 178 | mito_df['z'] = mito_df['z'].astype(np.int32) 179 | mito_df['size'] = mito_df['size'].astype(np.int32) 180 | mito_df['r0'] = mito_df['r0'].astype(np.float32) 181 | mito_df['r1'] = mito_df['r1'].astype(np.float32) 182 | mito_df['r2'] = mito_df['r2'].astype(np.float32) 183 | return mito_df 184 | 185 | 186 | @inject_client 187 | @neuroncriteria_args('neuron_criteria') 188 | def fetch_synapses_and_closest_mitochondria(neuron_criteria, synapse_criteria=None, *, batch_size=10, client=None): 189 | """ 190 | Fetch a set of synapses from a selection of neurons and also return 191 | their nearest mitocondria (by path-length within the neuron segment). 192 | 193 | Note: 194 | Some synapses have no nearby mitochondria, possibly due to 195 | fragmented segmentation around the synapse point. 196 | Such synapses ARE NOT RETURNED by this function. They're omitted. 197 | 198 | Args: 199 | 200 | neuron_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 201 | Determines which bodies to fetch synapses for. 202 | 203 | Note: 204 | Any ROI criteria specified in this argument does not affect 205 | which synapses are returned, only which bodies are inspected. 206 | 207 | synapse_criteria (SynapseCriteria): 208 | Optional. Allows you to filter synapses by roi, type, confidence. 209 | See :py:class:`.SynapseCriteria` for details. 210 | 211 | If the criteria specifies ``primary_only=True`` only primary ROIs will be returned in the results. 212 | If a synapse does not intersect any primary ROI, it will be listed with an roi of ``None``. 213 | (Since 'primary' ROIs do not overlap, each synapse will be listed only once.) 214 | Otherwise, all ROI names will be included in the results. 215 | In that case, some synapses will be listed multiple times -- once per intersecting ROI. 216 | If a synapse does not intersect any ROI, it will be listed with an roi of ``None``. 217 | 218 | batch_size: 219 | To improve performance and avoid timeouts, the synapses for multiple bodies 220 | will be fetched in batches, where each batch corresponds to N bodies. 221 | This argument sets the batch size N. 222 | 223 | client: 224 | If not provided, the global default :py:class:`.Client` will be used. 225 | 226 | Returns: 227 | 228 | DataFrame in which each row represent a single synapse, 229 | along with information about its closest mitochondrion. 230 | Unless ``primary_only`` was specified, some synapses may be listed more than once, 231 | if they reside in more than one overlapping ROI. 232 | 233 | The synapse coordinates will be returned in columns ``x,y,z``, 234 | and the mitochondria centroids will be stored in columns ``mx,my,mz``. 235 | 236 | The ``distance`` column indicates the distance from the synapse coordinate to the 237 | nearest edge of the mitochondria (not the centroid), as traveled along the neuron 238 | dendrite (not euclidean distance). The distance is given in voxel units (e.g. 8nm), 239 | not nanometers. See release notes concerning the estimated error of these measurements. 240 | 241 | Example: 242 | 243 | .. code-block:: ipython 244 | 245 | In [1]: from neuprint import fetch_synapses_and_closest_mitochondria, NeuronCriteria as NC, SynapseCriteria as SC 246 | ...: fetch_synapses_and_closest_mitochondria(NC(type='ExR2'), SC(type='pre')) 247 | Out[1]: 248 | bodyId type roi x y z confidence mitoType distance size mx my mz r0 r1 r2 249 | 0 1136865339 pre EB 25485 22873 19546 0.902 medium 214.053040 410544 25544 23096 19564 105.918625 35.547806 0.969330 250 | 1 1136865339 pre EB 25985 25652 23472 0.930 dark 19.313709 90048 26008 25646 23490 81.459419 21.493509 0.988575 251 | 2 1136865339 pre LAL(R) 14938 29149 22604 0.826 dark 856.091736 495208 14874 29686 22096 64.086639 46.906826 0.789570 252 | 3 1136865339 pre EB 24387 23583 20681 0.945 dark 78.424950 234760 24424 23536 20752 80.774353 29.854616 0.957713 253 | 4 1136865339 pre BU(R) 16909 25233 17658 0.994 dark 230.588562 215160 16862 25418 17824 42.314690 36.891937 0.628753 254 | ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... 255 | 4508 787762461 pre BU(R) 16955 26697 17300 0.643 dark 105.765854 176952 16818 26642 17200 91.884338 22.708199 0.975422 256 | 4509 787762461 pre LAL(R) 15008 28293 25995 0.747 dark 112.967644 446800 15044 28166 26198 176.721512 27.971079 0.992517 257 | 4510 787762461 pre EB 23468 24073 20882 0.757 dark 248.562714 92536 23400 23852 20760 39.696674 27.490204 0.860198 258 | 4511 787762461 pre BU(R) 18033 25846 20393 0.829 dark 38.627419 247640 18028 25846 20328 73.585144 29.661413 0.929788 259 | 4512 787762461 pre EB 22958 24565 20340 0.671 dark 218.104736 120880 23148 24580 20486 39.752777 32.047478 0.821770 260 | 261 | [4513 rows x 16 columns] 262 | """ 263 | neuron_criteria.matchvar = 'n' 264 | q = f""" 265 | {neuron_criteria.global_with(prefix=8)} 266 | MATCH (n:{neuron_criteria.label}) 267 | {neuron_criteria.all_conditions(prefix=8)} 268 | RETURN n.bodyId as bodyId 269 | """ 270 | bodies = client.fetch_custom(q)['bodyId'].values 271 | 272 | batch_dfs = [] 273 | for batch_bodies in tqdm(iter_batches(bodies, batch_size)): 274 | batch_criteria = copy.copy(neuron_criteria) 275 | batch_criteria.bodyId = batch_bodies 276 | batch_df = _fetch_synapses_and_closest_mitochondria(batch_criteria, synapse_criteria, client) 277 | if len(batch_df) > 0: 278 | batch_dfs.append( batch_df ) 279 | 280 | if batch_dfs: 281 | return pd.concat( batch_dfs, ignore_index=True ) 282 | 283 | # Return empty results, but with correct dtypes 284 | dtypes = { 285 | 'bodyId': np.dtype('int64'), 286 | 'type': pd.CategoricalDtype(categories=['pre', 'post'], ordered=False), 287 | 'roi': np.dtype('O'), 288 | 'x': np.dtype('int32'), 289 | 'y': np.dtype('int32'), 290 | 'z': np.dtype('int32'), 291 | 'confidence': np.dtype('float32'), 292 | 'mitoType': np.dtype('O'), 293 | 'distance': np.dtype('float32'), 294 | 'mx': np.dtype('int32'), 295 | 'my': np.dtype('int32'), 296 | 'mz': np.dtype('int32'), 297 | 'size': np.dtype('int32'), 298 | 'r0': np.dtype('float32'), 299 | 'r1': np.dtype('float32'), 300 | 'r2': np.dtype('float32'), 301 | } 302 | 303 | return pd.DataFrame([], columns=dtypes.keys()).astype(dtypes) 304 | 305 | 306 | def _fetch_synapses_and_closest_mitochondria(neuron_criteria, synapse_criteria, client): 307 | 308 | if synapse_criteria is None: 309 | synapse_criteria = SynapseCriteria(client=client) 310 | 311 | if synapse_criteria.primary_only: 312 | return_rois = {*client.primary_rois} 313 | else: 314 | return_rois = {*client.all_rois} 315 | 316 | # If the user specified rois to filter synapses by, but hasn't specified rois 317 | # in the NeuronCriteria, add them to the NeuronCriteria to speed up the query. 318 | if synapse_criteria.rois and not neuron_criteria.rois: 319 | neuron_criteria.rois = {*synapse_criteria.rois} 320 | neuron_criteria.roi_req = 'any' 321 | 322 | # Fetch results 323 | cypher = dedent(f"""\ 324 | {neuron_criteria.global_with(prefix=8)} 325 | MATCH (n:{neuron_criteria.label}) 326 | {neuron_criteria.all_conditions('n', prefix=8)} 327 | 328 | MATCH (n)-[:Contains]->(ss:SynapseSet)-[:Contains]->(s:Synapse)-[c:CloseTo]->(m:Element {{type: "mitochondrion"}}) 329 | 330 | {synapse_criteria.condition('n', 's', 'm', 'c', prefix=8)} 331 | // De-duplicate 's' because 'pre' synapses can appear in more than one SynapseSet 332 | WITH DISTINCT n, s, m, c 333 | 334 | RETURN n.bodyId as bodyId, 335 | s.type as type, 336 | s.confidence as confidence, 337 | s.location.x as x, 338 | s.location.y as y, 339 | s.location.z as z, 340 | apoc.map.removeKeys(s, ['location', 'confidence', 'type']) as syn_info, 341 | m.mitoType as mitoType, 342 | c.distance as distance, 343 | m.size as size, 344 | m.location.x as mx, 345 | m.location.y as my, 346 | m.location.z as mz, 347 | m.r0 as r0, 348 | m.r1 as r1, 349 | m.r2 as r2 350 | """) 351 | data = client.fetch_custom(cypher, format='json')['data'] 352 | 353 | # Assemble DataFrame 354 | syn_table = [] 355 | for body, syn_type, conf, x, y, z, syn_info, mitoType, distance, size, mx, my, mz, r0, r1, r2 in data: 356 | # Exclude non-primary ROIs if necessary 357 | syn_rois = return_rois & {*syn_info.keys()} 358 | # Fixme: Filter for the user's ROIs (drop duplicates) 359 | for roi in syn_rois: 360 | syn_table.append((body, syn_type, roi, x, y, z, conf, mitoType, distance, size, mx, my, mz, r0, r1, r2)) 361 | 362 | if not syn_rois: 363 | syn_table.append((body, syn_type, None, x, y, z, conf, mitoType, distance, size, mx, my, mz, r0, r1, r2)) 364 | 365 | cols = [ 366 | 'bodyId', 367 | 'type', 'roi', 'x', 'y', 'z', 'confidence', 368 | 'mitoType', 'distance', 'size', 'mx', 'my', 'mz', 'r0', 'r1', 'r2' 369 | ] 370 | syn_df = pd.DataFrame(syn_table, columns=cols) 371 | 372 | # Save RAM with smaller dtypes and interned strings 373 | syn_df['type'] = pd.Categorical(syn_df['type'], ['pre', 'post']) 374 | syn_df['roi'] = syn_df['roi'].apply(lambda s: sys.intern(s) if s else s) 375 | syn_df['x'] = syn_df['x'].astype(np.int32) 376 | syn_df['y'] = syn_df['y'].astype(np.int32) 377 | syn_df['z'] = syn_df['z'].astype(np.int32) 378 | syn_df['confidence'] = syn_df['confidence'].astype(np.float32) 379 | syn_df['mitoType'] = syn_df['mitoType'].apply(lambda s: sys.intern(s) if s else s) 380 | syn_df['distance'] = syn_df['distance'].astype(np.float32) 381 | syn_df['size'] = syn_df['size'].astype(np.int32) 382 | syn_df['mx'] = syn_df['mx'].astype(np.int32) 383 | syn_df['my'] = syn_df['my'].astype(np.int32) 384 | syn_df['mz'] = syn_df['mz'].astype(np.int32) 385 | syn_df['r0'] = syn_df['r0'].astype(np.float32) 386 | syn_df['r1'] = syn_df['r1'].astype(np.float32) 387 | syn_df['r2'] = syn_df['r2'].astype(np.float32) 388 | return syn_df 389 | 390 | 391 | @inject_client 392 | @neuroncriteria_args('source_criteria', 'target_criteria') 393 | def fetch_connection_mitochondria(source_criteria, target_criteria, synapse_criteria=None, min_total_weight=1, *, client=None): 394 | """ 395 | For a given set of source neurons and target neurons, find all 396 | synapse-level connections between the sources and targets, along 397 | with the nearest mitochondrion on the pre-synaptic side and the 398 | post-synaptic side. 399 | 400 | Returns a table similar to :py:func:`fetch_synapse_connections()`, but with 401 | extra ``_pre`` and ``_post`` columns to describe the nearest mitochondria 402 | to the pre/post synapse in the connection. 403 | If a given synapse has no nearby mitochondrion, the corresponding 404 | mito columns will be populated with ``NaN`` values. (This is typically 405 | much more likely to occur on the post-synaptic side than the pre-synaptic side.) 406 | 407 | Arguments are the same as :py:func:`fetch_synapse_connections()` 408 | 409 | Note: 410 | This function does not employ a custom cypher query to minimize the 411 | data fetched from the server. Instead, it makes multiple calls to the 412 | server and merges the results on the client. 413 | 414 | Args: 415 | source_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 416 | Criteria to by which to filter source (pre-synaptic) neurons. 417 | If omitted, all Neurons will be considered as possible sources. 418 | 419 | Note: 420 | Any ROI criteria specified in this argument does not affect 421 | which synapses are returned, only which bodies are inspected. 422 | 423 | target_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 424 | Criteria to by which to filter target (post-synaptic) neurons. 425 | If omitted, all Neurons will be considered as possible sources. 426 | 427 | Note: 428 | Any ROI criteria specified in this argument does not affect 429 | which synapses are returned, only which bodies are inspected. 430 | 431 | synapse_criteria (SynapseCriteria): 432 | Optional. Allows you to filter synapses by roi, type, confidence. 433 | The same criteria is used to filter both ``pre`` and ``post`` sides 434 | of the connection. 435 | By default, ``SynapseCriteria(primary_only=True)`` is used. 436 | 437 | If ``primary_only`` is specified in the criteria, then the resulting 438 | ``roi_pre`` and ``roi_post`` columns will contain a single 439 | string (or ``None``) in every row. 440 | 441 | Otherwise, the roi columns will contain a list of ROIs for every row. 442 | (Primary ROIs do not overlap, so every synapse resides in only one 443 | (or zero) primary ROI.) 444 | See :py:class:`.SynapseCriteria` for details. 445 | 446 | min_total_weight: 447 | If the total weight of the connection between two bodies is not at least 448 | this strong, don't include the synapses for that connection in the results. 449 | 450 | Note: 451 | This filters for total connection weight, regardless of the weight 452 | within any particular ROI. So, if your ``SynapseCriteria`` limits 453 | results to a particular ROI, but two bodies connect in multiple ROIs, 454 | then the number of synapses returned for the two bodies may appear to 455 | be less than ``min_total_weight``. That's because you filtered out 456 | the synapses in other ROIs. 457 | 458 | client: 459 | If not provided, the global default :py:class:`.Client` will be used. 460 | 461 | """ 462 | SC = SynapseCriteria 463 | 464 | # Fetch the synapses that connect sources and targets 465 | # (subject to min_total_weight) 466 | conn = fetch_synapse_connections(source_criteria, target_criteria, synapse_criteria, min_total_weight, batch_size=10) 467 | 468 | output_bodies = conn['bodyId_pre'].unique() 469 | output_mito = fetch_synapses_and_closest_mitochondria(output_bodies, SC(type='pre', client=client), batch_size=1) 470 | output_mito = output_mito[[*'xyz', 'mitoType', 'distance', 'size', 'mx', 'my', 'mz']] 471 | output_mito = output_mito.rename(columns={'size': 'mitoSize'}) 472 | 473 | input_bodies = conn['bodyId_post'].unique() 474 | input_mito = fetch_synapses_and_closest_mitochondria(input_bodies, SC(type='post', client=client), batch_size=1) 475 | input_mito = input_mito[[*'xyz', 'mitoType', 'distance', 'size', 'mx', 'my', 'mz']] 476 | input_mito = input_mito.rename(columns={'size': 'mitoSize'}) 477 | 478 | # This double-merge will add _pre and _post columns for the mito fields 479 | conn_with_mito = conn 480 | conn_with_mito = conn_with_mito.merge(output_mito, 481 | 'left', 482 | left_on=['x_pre', 'y_pre', 'z_pre'], 483 | right_on=['x', 'y', 'z']).drop(columns=[*'xyz']) 484 | 485 | conn_with_mito = conn_with_mito.merge(input_mito, 486 | 'left', 487 | left_on=['x_post', 'y_post', 'z_post'], 488 | right_on=['x', 'y', 'z'], 489 | suffixes=['_pre', '_post']).drop(columns=[*'xyz']) 490 | return conn_with_mito 491 | -------------------------------------------------------------------------------- /neuprint/queries/mitocriteria.py: -------------------------------------------------------------------------------- 1 | from textwrap import indent, dedent 2 | 3 | from ..utils import ensure_list_args, cypher_identifier 4 | from ..client import inject_client 5 | 6 | 7 | class MitoCriteria: 8 | """ 9 | Mitochondrion selection criteria. 10 | 11 | Specifies which fields to filter by when searching for mitochondria. 12 | This class does not send queries itself, but you use it to specify search 13 | criteria for various query functions. 14 | """ 15 | 16 | @inject_client 17 | @ensure_list_args(['rois']) 18 | def __init__(self, matchvar='m', *, rois=None, mitoType=None, size=0, primary_only=True, client=None): 19 | """ 20 | Except for ``matchvar``, all parameters must be passed as keyword arguments. 21 | 22 | Args: 23 | matchvar (str): 24 | An arbitrary cypher variable name to use when this 25 | ``MitoCriteria`` is used to construct cypher queries. 26 | 27 | rois (str or list): 28 | Optional. 29 | If provided, limit the results to mitochondria that reside within the given roi(s). 30 | 31 | mitoType: 32 | If provided, limit the results to mitochondria of the specified type. 33 | Either ``dark``, ``medium``, or ``light``. 34 | (Neuroglancer users: Note that in the hemibrain mito segmentation, ``medium=3``) 35 | 36 | size: 37 | Specifies a minimum size (in voxels) for mitochondria returned in the results. 38 | 39 | primary_only (boolean): 40 | If True, only include primary ROI names in the results. 41 | Disable this with caution. 42 | 43 | Note: 44 | This parameter does NOT filter by ROI. (See the ``rois`` argument for that.) 45 | It merely determines whether or not each mitochondrion should be associated with exactly 46 | one ROI in the query output, or with multiple ROIs (one for every non-primary 47 | ROI the mitochondrion intersects). 48 | 49 | If you set ``primary_only=False``, then the table will contain duplicate entries 50 | for each mito -- one per intersecting ROI. 51 | 52 | client: 53 | Used to validate ROI names. 54 | If not provided, the global default :py:class:`.Client` will be used. 55 | """ 56 | unknown_rois = {*rois} - {*client.all_rois} 57 | assert not unknown_rois, f"Unrecognized mito rois: {unknown_rois}" 58 | 59 | mitoType = mitoType or None 60 | assert mitoType in (None, 'dark', 'medium', 'light'), \ 61 | f"Invalid mitoType: {mitoType}." 62 | 63 | self.matchvar = matchvar 64 | self.rois = rois 65 | self.mitoType = mitoType 66 | self.size = size 67 | self.primary_only = primary_only 68 | 69 | def condition(self, *vars, prefix='', comments=True): 70 | """ 71 | Construct a cypher WITH..WHERE clause to filter for mito criteria. 72 | 73 | Any match variables you wish to "carry through" for subsequent clauses 74 | in your query must be named in the ``vars`` arguments. 75 | """ 76 | if not vars: 77 | vars = [self.matchvar] 78 | 79 | assert self.matchvar in vars, \ 80 | ("Please pass all match vars, including the one that " 81 | f"belongs to this criteria ('{self.matchvar}').") 82 | 83 | if isinstance(prefix, int): 84 | prefix = ' '*prefix 85 | 86 | type_expr = f'{self.matchvar}.type = "mitochondrion"' 87 | roi_expr = size_expr = mitoType_expr = "" 88 | 89 | if self.rois: 90 | roi_expr = '(' + ' OR '.join([f'{self.matchvar}.{cypher_identifier(roi)}' for roi in self.rois]) + ')' 91 | 92 | if self.size: 93 | size_expr = f'({self.matchvar}.size >= {self.size})' 94 | 95 | if self.mitoType: 96 | mitoType_expr = f"({self.matchvar}.mitoType = '{self.mitoType}')" 97 | 98 | exprs = [*filter(None, [type_expr, roi_expr, size_expr, mitoType_expr])] 99 | 100 | if not exprs: 101 | return "" 102 | 103 | cond = dedent(f"""\ 104 | WITH {', '.join(vars)} 105 | WHERE {' AND '.join(exprs)} 106 | """) 107 | 108 | if comments: 109 | cond = f"// -- Filter mito '{self.matchvar}' --\n" + cond 110 | 111 | cond = indent(cond, prefix)[len(prefix):] 112 | return cond 113 | 114 | 115 | def __eq__(self, other): 116 | return ( (self.matchvar == other.matchvar) 117 | and (self.rois == other.rois) 118 | and (self.mitoType == other.mitoType) 119 | and (self.size == other.size) 120 | and (self.primary_only == other.primary_only)) 121 | 122 | 123 | def __repr__(self): 124 | s = f"MitoCriteria('{self.matchvar}'" 125 | 126 | args = [] 127 | 128 | if self.rois: 129 | args.append("rois=[" + ", ".join(f"'{roi}'" for roi in self.rois) + "]") 130 | 131 | if self.mitoType: 132 | args.append(f"mitoType='{self.mitoType}'") 133 | 134 | if self.size: 135 | args.append(f"size={self.size}") 136 | 137 | if self.primary_only: 138 | args.append("primary_only=True") 139 | 140 | if args: 141 | s += ', ' + ', '.join(args) 142 | 143 | s += ")" 144 | return s 145 | -------------------------------------------------------------------------------- /neuprint/queries/neurons.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import ujson 3 | 4 | from textwrap import indent 5 | from ..client import inject_client 6 | from ..utils import compile_columns, cypher_identifier 7 | from .neuroncriteria import neuroncriteria_args 8 | 9 | # Core set of columns 10 | CORE_NEURON_COLS = ['bodyId', 'instance', 'type', 11 | 'pre', 'post', 'downstream', 'upstream', 'mito', 'size', 12 | 'status', 'cropped', 'statusLabel', 13 | 'cellBodyFiber', 14 | 'somaRadius', 'somaLocation', 15 | 'inputRois', 'outputRois', 'roiInfo'] 16 | 17 | 18 | @inject_client 19 | @neuroncriteria_args('criteria') 20 | def fetch_neurons(criteria=None, *, omit_rois=False,client=None): 21 | """ 22 | Return properties and per-ROI synapse counts for a set of neurons. 23 | 24 | Searches for a set of Neurons (or Segments) that match the given :py:class:`.NeuronCriteria`. 25 | Returns their properties, including the distibution of their synapses in all brain regions. 26 | 27 | This implements a superset of the features on the Neuprint Explorer `Find Neurons`_ page. 28 | 29 | Returns data in the the same format as :py:func:`fetch_custom_neurons()`, 30 | but doesn't require you to write cypher. 31 | 32 | .. _Find Neurons: https://neuprint.janelia.org/?dataset=hemibrain%3Av1.2.1&qt=findneurons&q=1 33 | 34 | Args: 35 | criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 36 | Only Neurons which satisfy all components of the given criteria are returned. 37 | If no criteria is specified then the default ``NeuronCriteria()`` is used. 38 | 39 | omit_rois (bool): 40 | If True, the ROI columns are omitted from the output. 41 | If you don't need ROI information, this can speed up the query. 42 | 43 | client: 44 | If not provided, the global default :py:class:`.Client` will be used. 45 | 46 | Returns: 47 | Two DataFrames: ``(neurons_df, roi_counts_df)`` unless ``omit_rois`` is True, 48 | in which case only ``neurons_df`` is returned. 49 | 50 | In ``neurons_df``, all available ``:Neuron`` columns are returned, with the following changes: 51 | 52 | - ROI boolean columns are removed 53 | - ``roiInfo`` is parsed as json data 54 | - coordinates (such as ``somaLocation``) are provided as a list ``[x, y, z]`` 55 | - New columns ``input_rois`` and ``output_rois`` contain lists of each neuron's ROIs. 56 | 57 | In ``roi_counts_df``, the ``roiInfo`` has been loadded into a table 58 | of per-neuron-per-ROI synapse counts, with separate columns 59 | for ``pre`` (outputs) and ``post`` (inputs). 60 | 61 | .. note:: 62 | 63 | In ``roi_counts_df``, the sum of the ``pre`` and ``post`` counts will be more than 64 | the total ``pre`` and ``post`` values returned in ``neuron_df``. 65 | That is, synapses are double-counted (or triple-counted, etc.) in ``roi_counts_df``. 66 | This is because ROIs form a hierarchical structure, so each synapse intersects 67 | more than one ROI. See :py:func:`.fetch_roi_hierarchy()` for more information. 68 | 69 | .. note:: 70 | 71 | Connections which fall outside of all primary ROIs are listed via special entries 72 | using ``NotPrimary`` in place of an ROI name. The term ``NotPrimary`` is 73 | introduced by this function. It isn't used internally by the neuprint database. 74 | 75 | See also: 76 | 77 | :py:func:`.fetch_custom_neurons()` produces similar output, 78 | but permits you to supply your own cypher query directly. 79 | 80 | 81 | Example: 82 | 83 | .. code-block:: ipython 84 | 85 | In [1]: from neuprint import fetch_neurons, NeuronCriteria as NC 86 | 87 | In [2]: neurons_df, roi_counts_df = fetch_neurons( 88 | ...: NC(inputRois=['SIP(R)', 'aL(R)'], 89 | ...: status='Traced', 90 | ...: type='MBON.*')) 91 | 92 | In [3]: neurons_df.iloc[:5, :11] 93 | Out[3]: 94 | bodyId instance type cellBodyFiber pre post size status cropped statusLabel somaRadius 95 | 0 300972942 MBON14(a3)_R MBON14 NaN 543 13634 1563154937 Traced False Roughly traced NaN 96 | 1 422725634 MBON06(B1>a)(AVM07)_L MBON06 NaN 1356 20978 3118269136 Traced False Roughly traced NaN 97 | 2 423382015 MBON23(a2sp)(PDL05)_R MBON23 SFS1 733 4466 857093893 Traced False Roughly traced 291.0 98 | 3 423774471 MBON19(a2p3p)(PDL05)_R MBON19 SFS1 299 1484 628019179 Traced False Roughly traced 286.0 99 | 4 424767514 MBON11(y1pedc>a/B)(ADM05)_R MBON11 mAOTU2 1643 27641 5249327644 Traced False Traced 694.5 100 | 101 | In [4]: neurons_df['inputRois'].head() 102 | Out[4]: 103 | 0 [MB(+ACA)(R), MB(R), None, SIP(R), SLP(R), SMP... 104 | 1 [CRE(-ROB,-RUB)(R), CRE(R), INP, MB(+ACA)(R), ... 105 | 2 [MB(+ACA)(R), MB(R), None, SIP(R), SLP(R), SMP... 106 | 3 [MB(+ACA)(R), MB(R), SIP(R), SMP(R), SNP(R), a... 107 | 4 [CRE(-ROB,-RUB)(R), CRE(L), CRE(R), INP, MB(+A... 108 | Name: inputRois, dtype: object 109 | 110 | In [5]: roi_counts_df.head() 111 | Out[5]: 112 | bodyId roi pre post 113 | 0 300972942 MB(R) 17 13295 114 | 1 300972942 aL(R) 17 13271 115 | 2 300972942 a3(R) 17 13224 116 | 3 300972942 MB(+ACA)(R) 17 13295 117 | 4 300972942 SNP(R) 526 336 118 | """ 119 | criteria.matchvar = 'n' 120 | 121 | # Unlike in fetch_custom_neurons() below, here we specify the 122 | # return properties individually to avoid a large JSON payload. 123 | # (Returning a map on every row is ~2x more costly than returning a table of rows/columns.) 124 | props = compile_columns(client, core_columns=CORE_NEURON_COLS) 125 | props = map(cypher_identifier, props) 126 | if omit_rois: 127 | props = [p for p in props if p != 'roiInfo'] 128 | 129 | return_exprs = ',\n'.join(f'n.{prop} as {prop}' for prop in props) 130 | return_exprs = indent(return_exprs, ' '*15)[15:] 131 | 132 | q = f"""\ 133 | {criteria.global_with(prefix=8)} 134 | MATCH (n :{criteria.label}) 135 | {criteria.all_conditions(prefix=8)} 136 | RETURN {return_exprs} 137 | ORDER BY n.bodyId 138 | """ 139 | neuron_df = client.fetch_custom(q) 140 | if omit_rois: 141 | return neuron_df 142 | 143 | neuron_df, roi_counts_df = _process_neuron_df(neuron_df, client) 144 | return neuron_df, roi_counts_df 145 | 146 | 147 | @inject_client 148 | def fetch_custom_neurons(q, *, client=None): 149 | """ 150 | Return properties and per-ROI synapse counts for a set of neurons, 151 | using your own cypher query. 152 | 153 | Use a custom query to fetch a neuron table, with nicer output 154 | than you would get from a call to :py:func:`.fetch_custom()`. 155 | 156 | Returns data in the the same format as :py:func:`.fetch_neurons()`. 157 | but allows you to provide your own cypher query logic 158 | (subject to certain requirements; see below). 159 | 160 | This function includes all Neuron fields in the results, 161 | and also sends back ROI counts as a separate table. 162 | 163 | Args: 164 | 165 | q: 166 | Custom query. Must match a neuron named ``n``, 167 | and must ``RETURN n``. 168 | 169 | .. code-block:: cypher 170 | 171 | ... 172 | MATCH (n :Neuron) 173 | ... 174 | RETURN n 175 | ... 176 | 177 | client: 178 | If not provided, the global default ``Client`` will be used. 179 | 180 | Returns: 181 | Two DataFrames. 182 | ``(neurons_df, roi_counts_df)`` 183 | 184 | In ``neurons_df``, all available columns ``:Neuron`` columns are returned, with the following changes: 185 | 186 | - ROI boolean columns are removed 187 | - ``roiInfo`` is parsed as json data 188 | - coordinates (such as ``somaLocation``) are provided as a list ``[x, y, z]`` 189 | - New columns ``inputRoi`` and ``outputRoi`` contain lists of each neuron's ROIs. 190 | 191 | In ``roi_counts_df``, the ``roiInfo`` has been loaded into a table 192 | of per-neuron-per-ROI synapse counts, with separate columns 193 | for ``pre`` (outputs) and ``post`` (inputs). 194 | 195 | Connections which fall outside of all primary ROIs are listed via special entries 196 | using ``NotPrimary`` in place of an ROI name. The term ``NotPrimary`` is 197 | introduced by this function. It isn't used internally by the neuprint database. 198 | """ 199 | results = client.fetch_custom(q) 200 | 201 | if len(results) == 0: 202 | NEURON_COLS = compile_columns(client, core_columns=CORE_NEURON_COLS) 203 | neuron_df = pd.DataFrame([], columns=NEURON_COLS, dtype=object) 204 | roi_counts_df = pd.DataFrame([], columns=['bodyId', 'roi', 'pre', 'post']) 205 | return neuron_df, roi_counts_df 206 | 207 | neuron_df = pd.DataFrame(results['n'].tolist()) 208 | 209 | neuron_df, roi_counts_df = _process_neuron_df(neuron_df, client) 210 | return neuron_df, roi_counts_df 211 | 212 | 213 | def _process_neuron_df(neuron_df, client, parse_locs=True): 214 | """ 215 | Given a DataFrame of neuron properties, parse the roiInfo into 216 | inputRois and outputRois, and a secondary DataFrame for per-ROI 217 | synapse counts. 218 | 219 | Returns: 220 | neuron_df, roi_counts_df 221 | 222 | Warning: destructively modifies the input DataFrame. 223 | """ 224 | # Drop roi columns 225 | columns = {*neuron_df.columns} - {*client.all_rois} 226 | neuron_df = neuron_df[[*columns]] 227 | 228 | # Specify column order: 229 | # Standard columns first, then any extra columns in the results (if any). 230 | neuron_cols = [*filter(lambda c: c in neuron_df.columns, CORE_NEURON_COLS)] 231 | extra_cols = {*neuron_df.columns} - {*neuron_cols} 232 | neuron_cols += [*extra_cols] 233 | neuron_df = neuron_df[[*neuron_cols]] 234 | 235 | # Make a list of rois for every neuron (both pre and post) 236 | neuron_df['roiInfo'] = neuron_df['roiInfo'].apply(ujson.loads) 237 | neuron_df['inputRois'] = neuron_df['roiInfo'].apply(lambda d: sorted([k for k,v in d.items() if v.get('post')])) 238 | neuron_df['outputRois'] = neuron_df['roiInfo'].apply(lambda d: sorted([k for k,v in d.items() if v.get('pre')])) 239 | 240 | # Find location columns 241 | if parse_locs: 242 | for c in neuron_df.columns: 243 | if neuron_df[c].dtype != 'object': 244 | continue 245 | # Skip columns which contain no dictionaries 246 | is_dict = [isinstance(x, dict) for x in neuron_df[c]] 247 | if not any(is_dict): 248 | continue 249 | neuron_df.loc[is_dict, c] = neuron_df.loc[is_dict, c].apply(lambda x: x.get('coordinates', x)) 250 | 251 | # Return roi info as a separate table. 252 | # (Note: Some columns aren't present in old neuprint databases.) 253 | countcols = ['pre', 'post', 'downstream', 'upstream', 'mito'] 254 | countcols = [c for c in countcols if c in neuron_df.columns] 255 | fullcols = ['bodyId', 'roi', *countcols] 256 | nonroi_cols = ['bodyId', *countcols] 257 | 258 | roi_counts = [ 259 | {'bodyId': bodyId, 'roi': roi, **counts} 260 | for bodyId, roiInfo in zip(neuron_df['bodyId'], neuron_df['roiInfo']) 261 | for roi, counts in roiInfo.items() 262 | ] 263 | roi_counts_df = pd.DataFrame(roi_counts, columns=fullcols) 264 | roi_counts_df = roi_counts_df.fillna(0).astype({c: int for c in countcols}) 265 | 266 | # The 'NotPrimary' entries aren't stored by neuprint explicitly. 267 | # We must compute them by subtracting the summed per-ROI counts 268 | # from the overall counts in the neuron table. 269 | roi_totals_df = roi_counts_df.query('roi in @client.primary_rois')[nonroi_cols].groupby('bodyId').sum() 270 | roi_totals_df = roi_totals_df.reindex(neuron_df['bodyId']) 271 | 272 | not_primary_df = neuron_df[nonroi_cols].set_index('bodyId').fillna(0) - roi_totals_df.fillna(0) 273 | not_primary_df = not_primary_df.astype(int) 274 | not_primary_df['roi'] = 'NotPrimary' 275 | not_primary_df = not_primary_df.reset_index()[fullcols] 276 | 277 | roi_counts_df = pd.concat((roi_counts_df, not_primary_df), ignore_index=True) 278 | roi_counts_df = roi_counts_df.sort_values(['bodyId', 'roi'], ignore_index=True) 279 | 280 | # Drop the rows with all-zero counts (introduced via the NotPrimary rows we added) 281 | roi_counts_df = roi_counts_df.loc[roi_counts_df[countcols].any(axis=1)].copy() 282 | 283 | return neuron_df, roi_counts_df 284 | -------------------------------------------------------------------------------- /neuprint/queries/recon.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Container 2 | 3 | import pandas as pd 4 | 5 | from ..client import inject_client 6 | from ..utils import trange 7 | from .neuroncriteria import NeuronCriteria, neuroncriteria_args 8 | from .general import fetch_custom 9 | from .connectivity import fetch_adjacencies 10 | 11 | 12 | @inject_client 13 | @neuroncriteria_args('criteria') 14 | def fetch_output_completeness(criteria, complete_statuses=['Traced'], batch_size=1000, *, client=None): 15 | """ 16 | Compute an estimate of "output completeness" for a set of neurons. 17 | Output completeness is defined as the fraction of post-synaptic 18 | connections which belong to 'complete' neurons, as defined by their status. 19 | 20 | Args: 21 | criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 22 | Defines the set of neurons for which output completeness should be computed. 23 | 24 | complete_statuses: 25 | A list of neuron statuses should be considered complete for the purposes of this function. 26 | 27 | Returns: 28 | DataFrame with columns ``['bodyId', 'completeness', 'traced_weight', 'untraced_weight', 'total_weight']`` 29 | 30 | For the purposes of these results, any statuses in the 31 | set of complete_statuses are considered 'traced'. 32 | """ 33 | assert isinstance(criteria, NeuronCriteria) 34 | criteria.matchvar = 'n' 35 | 36 | assert isinstance(complete_statuses, Container) 37 | complete_statuses = list(complete_statuses) 38 | 39 | if batch_size is None: 40 | return _fetch_output_completeness(criteria, client) 41 | 42 | q = f"""\ 43 | {criteria.global_with(prefix=8)} 44 | MATCH (n:{criteria.label}) 45 | {criteria.all_conditions(prefix=8)} 46 | RETURN n.bodyId as bodyId 47 | """ 48 | bodies = fetch_custom(q)['bodyId'] 49 | 50 | batch_results = [] 51 | for start in trange(0, len(bodies), batch_size): 52 | criteria.bodyId = bodies[start:start+batch_size] 53 | _df = _fetch_output_completeness(criteria, complete_statuses, client) 54 | if len(_df) > 0: 55 | batch_results.append( _df ) 56 | return pd.concat( batch_results, ignore_index=True ) 57 | 58 | 59 | def _fetch_output_completeness(criteria, complete_statuses, client=None): 60 | q = f"""\ 61 | {criteria.global_with(prefix=8)} 62 | MATCH (n:{criteria.label}) 63 | {criteria.all_conditions(prefix=8)} 64 | 65 | // Total connection weights 66 | MATCH (n)-[e:ConnectsTo]->(:Segment) 67 | WITH n, sum(e.weight) as total_weight 68 | 69 | // Traced connection weights 70 | MATCH (n)-[e2:ConnectsTo]->(m:Segment) 71 | WHERE 72 | m.status in {complete_statuses} 73 | OR m.statusLabel in {complete_statuses} 74 | 75 | RETURN n.bodyId as bodyId, 76 | total_weight, 77 | sum(e2.weight) as traced_weight 78 | """ 79 | completion_stats_df = client.fetch_custom(q) 80 | completion_stats_df['untraced_weight'] = completion_stats_df.eval('total_weight - traced_weight') 81 | completion_stats_df['completeness'] = completion_stats_df.eval('traced_weight / total_weight') 82 | 83 | return completion_stats_df[['bodyId', 'total_weight', 'traced_weight', 'untraced_weight', 'completeness']] 84 | 85 | 86 | @inject_client 87 | @neuroncriteria_args('criteria') 88 | def fetch_downstream_orphan_tasks(criteria, complete_statuses=['Traced'], *, client=None): 89 | """ 90 | Fetch the set of "downstream orphans" for a given set of neurons. 91 | 92 | Returns a single DataFrame, where the downstream orphans have 93 | been sorted by the weight of the connection, and their cumulative 94 | contributions to the overall output-completeness of the upstream 95 | neuron is also given. 96 | 97 | That is, if you started tracing orphans from this DataFrame in 98 | order, then the ``cum_completeness`` column indicates how complete 99 | the upstream body is after each orphan becomes traced. 100 | 101 | Args: 102 | criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`): 103 | Determines the set of "upstream" bodies for which 104 | downstream orphans should be identified. 105 | 106 | Returns: 107 | DataFrame, where ``bodyId_pre`` contains the upstream bodies you specified 108 | via ``criteria``, and ``bodyId_post`` contains the list of downstream orphans. 109 | 110 | Example: 111 | 112 | .. code-block:: ipython 113 | 114 | In [1]: orphan_tasks = fetch_downstream_orphan_tasks(NC(status='Traced', cropped=False, rois=['PB'])) 115 | 116 | In [1]: orphan_tasks.query('cum_completeness < 0.2').head(10) 117 | Out[1]: 118 | bodyId_pre bodyId_post orphan_weight status_post total_weight orig_traced_weight orig_untraced_weight orig_completeness cum_orphan_weight cum_completeness 119 | 6478 759685279 759676733 2 Assign 7757 1427 6330 0.183963 2 0.184221 120 | 8932 759685279 913193340 1 None 7757 1427 6330 0.183963 3 0.184350 121 | 8943 759685279 913529796 1 None 7757 1427 6330 0.183963 4 0.184479 122 | 8950 759685279 913534416 1 None 7757 1427 6330 0.183963 5 0.184607 123 | 12121 1002507170 1387701052 1 None 522 102 420 0.195402 1 0.197318 124 | 35764 759685279 790382544 1 None 7757 1427 6330 0.183963 6 0.184736 125 | 36052 759685279 851023555 1 Assign 7757 1427 6330 0.183963 7 0.184865 126 | 36355 759685279 974908767 2 None 7757 1427 6330 0.183963 9 0.185123 127 | 36673 759685279 1252526211 1 None 7757 1427 6330 0.183963 10 0.185252 128 | 44840 759685279 1129418900 1 None 7757 1427 6330 0.183963 11 0.185381 129 | 130 | """ 131 | # Find all downstream segments, along with the status of all upstream and downstream bodies. 132 | status_df, roi_conn_df = fetch_adjacencies(criteria, NeuronCriteria(label='Segment', client=client), properties=['status', 'statusLabel'], client=client) 133 | 134 | # That table is laid out per-ROI, but we don't care about ROI. Aggregate. 135 | conn_df = roi_conn_df.groupby(['bodyId_pre', 'bodyId_post'])['weight'].sum().reset_index() 136 | 137 | # Sort connections from strong to weak. 138 | conn_df.sort_values(['bodyId_pre', 'weight', 'bodyId_post'], ascending=[True, False, True], inplace=True) 139 | conn_df.reset_index(drop=True, inplace=True) 140 | 141 | # Append status column. 142 | conn_df = conn_df.merge(status_df, left_on='bodyId_post', right_on='bodyId').drop(columns={'bodyId'}) 143 | 144 | # Drop non-orphans. 145 | conn_df.query('status not in @complete_statuses and statusLabel not in @complete_statuses', inplace=True) 146 | conn_df.rename(columns={'status': 'status_post', 'weight': 'orphan_weight'}, inplace=True) 147 | 148 | # Calculate current output completeness 149 | completeness_df = fetch_output_completeness(criteria, complete_statuses, client=client) 150 | completeness_df = completeness_df.rename(columns={'completeness': 'orig_completeness', 151 | 'traced_weight': 'orig_traced_weight', 152 | 'untraced_weight': 'orig_untraced_weight'}) 153 | 154 | # Calculate the potential output completeness we would 155 | # achieve if these orphans became traced, one-by-one. 156 | conn_df = conn_df.merge(completeness_df, 'left', left_on='bodyId_pre', right_on='bodyId').drop(columns=['bodyId']) 157 | conn_df['cum_orphan_weight'] = conn_df.groupby('bodyId_pre')['orphan_weight'].cumsum() 158 | conn_df['cum_completeness'] = conn_df.eval('(orig_traced_weight + cum_orphan_weight) / total_weight') 159 | 160 | return conn_df 161 | -------------------------------------------------------------------------------- /neuprint/queries/rois.py: -------------------------------------------------------------------------------- 1 | from asciitree import LeftAligned 2 | 3 | from ..client import inject_client 4 | from .general import fetch_meta 5 | 6 | 7 | @inject_client 8 | def fetch_all_rois(*, client=None): 9 | """ 10 | List all ROIs in the dataset. 11 | """ 12 | meta = fetch_meta(client=client) 13 | return _all_rois_from_meta(meta) 14 | 15 | 16 | def _all_rois_from_meta(meta): 17 | rois = {*meta['roiInfo'].keys()} 18 | 19 | if meta['dataset'] == 'hemibrain': 20 | # These ROIs are special: 21 | # For historical reasons, they exist as tags, 22 | # but are not (always) listed in the Meta roiInfo. 23 | rois |= {'FB-column3', 'AL-DC3'} 24 | rois |= {f"AL-DC{i}(R)" for i in [1,2,3,4]} 25 | 26 | return sorted(rois) 27 | 28 | 29 | @inject_client 30 | def fetch_primary_rois(*, client=None): 31 | """ 32 | List 'primary' ROIs in the dataset. 33 | Primary ROIs do not overlap with each other. 34 | """ 35 | q = "MATCH (m:Meta) RETURN m.primaryRois as rois" 36 | rois = client.fetch_custom(q)['rois'].iloc[0] 37 | return sorted(rois) 38 | 39 | 40 | def fetch_roi_hierarchy(include_subprimary=True, mark_primary=True, format='dict', *, client=None): 41 | """ 42 | Fetch the ROI hierarchy nesting relationships. 43 | 44 | Most ROIs in neuprint are part of a hierarchy of nested regions. 45 | The structure of the hierarchy is stored in the dataset metadata, 46 | and can be retrieved with this function. 47 | 48 | Args: 49 | include_subprimary: 50 | If True, all hierarchy levels are included in the output. 51 | Otherwise, the hierarchy will only go as deep as necessary to 52 | cover all "primary" ROIs, but not any sub-primary ROIs that 53 | are contained within them. 54 | 55 | mark_primary: 56 | If True, append an asterisk (``*``) to the names of 57 | "primary" ROIs in the hierarchy. 58 | Primary ROIs do not overlap with each other. 59 | 60 | format: 61 | Either ``"dict"``, ``"text"``, or ``nx``. 62 | Specifies whether to return the hierarchy as a `dict`, or as 63 | a printable text-based tree, or as a ``networkx.DiGraph`` 64 | (requires ``networkx``). 65 | 66 | Returns: 67 | Either ``dict``, ``str``, or ``nx.DiGraph``, 68 | depending on your chosen ``format``. 69 | 70 | Example: 71 | 72 | .. code-block:: ipython 73 | 74 | In [1]: from neuprint.queries import fetch_roi_hierarchy 75 | ...: 76 | ...: # Print the first few nodes of the tree -- you get the idea 77 | ...: roi_tree_text = fetch_roi_hierarchy(False, True, 'text') 78 | ...: print(roi_tree_text[:180]) 79 | hemibrain 80 | +-- AL(L)* 81 | +-- AL(R)* 82 | +-- AOT(R) 83 | +-- CX 84 | | +-- AB(L)* 85 | | +-- AB(R)* 86 | | +-- EB* 87 | | +-- FB* 88 | | +-- NO* 89 | | +-- PB* 90 | +-- GC 91 | +-- GF(R) 92 | +-- GNG* 93 | +-- INP 94 | | 95 | """ 96 | assert format in ('dict', 'text', 'nx') 97 | meta = fetch_meta(client=client) 98 | hierarchy = meta['roiHierarchy'] 99 | primary_rois = {*meta['primaryRois']} 100 | 101 | def insert(h, d): 102 | name = h['name'] 103 | is_primary = (name in primary_rois) 104 | if mark_primary and is_primary: 105 | name += "*" 106 | 107 | d[name] = {} 108 | 109 | if 'children' not in h: 110 | return 111 | 112 | if is_primary and not include_subprimary: 113 | return 114 | 115 | for c in sorted(h['children'], key=lambda c: c['name']): 116 | insert(c, d[name]) 117 | 118 | d = {} 119 | insert(hierarchy, d) 120 | 121 | if format == 'dict': 122 | return d 123 | 124 | if format == "text": 125 | return LeftAligned()(d) 126 | 127 | if format == 'nx': 128 | import networkx as nx 129 | g = nx.DiGraph() 130 | 131 | def add_nodes(parent, d): 132 | for k in d.keys(): 133 | g.add_edge(parent, k) 134 | add_nodes(k, d[k]) 135 | add_nodes('hemibrain', d['hemibrain']) 136 | return g 137 | -------------------------------------------------------------------------------- /neuprint/queries/synapsecriteria.py: -------------------------------------------------------------------------------- 1 | from textwrap import indent, dedent 2 | 3 | from ..utils import ensure_list_args, cypher_identifier 4 | from ..client import inject_client 5 | 6 | 7 | class SynapseCriteria: 8 | """ 9 | Synapse selection criteria. 10 | 11 | Specifies which fields to filter by when searching for Synapses. 12 | This class does not send queries itself, but you use it to specify search 13 | criteria for various query functions. 14 | """ 15 | 16 | @inject_client 17 | @ensure_list_args(['rois']) 18 | def __init__(self, matchvar='s', *, rois=None, type=None, confidence=None, primary_only=True, client=None): # noqa 19 | """ 20 | Except for ``matchvar``, all parameters must be passed as keyword arguments. 21 | 22 | Args: 23 | matchvar (str): 24 | An arbitrary cypher variable name to use when this 25 | ``SynapseCriteria`` is used to construct cypher queries. 26 | 27 | rois (str or list): 28 | Optional. 29 | If provided, limit the results to synapses that reside within any of the given roi(s). 30 | 31 | type: 32 | If provided, limit results to either 'pre' or 'post' synapses. 33 | 34 | confidence (float, 0.0-1.0): 35 | Limit results to synapses of at least this confidence rating. 36 | By default, use the dataset's default synapse confidence threshold, 37 | which will include the same synapses that were counted in each 38 | neuron-neuron ``weight`` (as opposed to ``weightHP`` or ``weightHR``). 39 | 40 | primary_only (boolean): 41 | If True, only include primary ROI names in the results. 42 | Disable this with caution. 43 | 44 | Note: 45 | This parameter does NOT filter by ROI. (See the ``rois`` argument for that.) 46 | It merely determines whether or not each synapse should be associated with exactly 47 | one ROI in the query output, or with multiple ROIs (one for every non-primary 48 | ROI the synapse intersects). 49 | 50 | If you set ``primary_only=False``, then the table will contain duplicate entries 51 | for each synapse -- one per intersecting ROI. 52 | client: 53 | Used to validate ROI names. 54 | If not provided, the global default :py:class:`.Client` will be used. 55 | """ 56 | unknown_rois = {*rois} - {*client.all_rois} 57 | assert not unknown_rois, f"Unrecognized synapse rois: {unknown_rois}" 58 | 59 | type = type or None 60 | assert type in ('pre', 'post', None), \ 61 | f"Invalid synapse type: {type}. Choices are 'pre' and 'post'." 62 | 63 | nonprimary = {*rois} - {*client.primary_rois} 64 | assert not nonprimary or not primary_only, \ 65 | f"You listed non-primary ROIs ({nonprimary}) but did not specify include_nonprimary=True" 66 | 67 | if confidence is None: 68 | confidence = client.meta.get('postHighAccuracyThreshold', 0.0) 69 | 70 | self.matchvar = matchvar 71 | self.rois = rois 72 | self.type = type 73 | self.confidence = confidence 74 | self.primary_only = primary_only 75 | 76 | def condition(self, *matchvars, prefix='', comments=True): 77 | """ 78 | Construct a cypher WITH..WHERE clause to filter for synapse criteria. 79 | 80 | Any match variables you wish to "carry through" for subsequent clauses 81 | in your query must be named in the ``vars`` arguments. 82 | """ 83 | if not matchvars: 84 | matchvars = [self.matchvar] 85 | 86 | assert self.matchvar in matchvars, \ 87 | ("Please pass all match vars, including the one that " 88 | f"belongs to this criteria ('{self.matchvar}').") 89 | 90 | if isinstance(prefix, int): 91 | prefix = ' '*prefix 92 | 93 | roi_expr = conf_expr = type_expr = "" 94 | if self.rois: 95 | roi_expr = '(' + ' OR '.join([f'{self.matchvar}.{cypher_identifier(roi)}' for roi in self.rois]) + ')' 96 | 97 | if self.confidence: 98 | conf_expr = f'({self.matchvar}.confidence > {self.confidence})' 99 | 100 | if self.type: 101 | type_expr = f"({self.matchvar}.type = '{self.type}')" 102 | 103 | exprs = [*filter(None, [roi_expr, conf_expr, type_expr])] 104 | 105 | if not exprs: 106 | return "" 107 | 108 | cond = dedent(f"""\ 109 | WITH {', '.join(matchvars)} 110 | WHERE {' AND '.join(exprs)} 111 | """) 112 | 113 | if comments: 114 | cond = f"// -- Filter synapse '{self.matchvar}' --\n" + cond 115 | 116 | cond = indent(cond, prefix)[len(prefix):] 117 | return cond 118 | 119 | def __eq__(self, other): 120 | return ( (self.matchvar == other.matchvar) 121 | and (self.rois == other.rois) 122 | and (self.type == other.type) 123 | and (self.confidence == other.confidence) 124 | and (self.primary_only == other.primary_only)) 125 | 126 | def __repr__(self): 127 | s = f"SynapseCriteria('{self.matchvar}'" 128 | 129 | args = [] 130 | 131 | if self.rois: 132 | args.append("rois=[" + ", ".join(f"'{roi}'" for roi in self.rois) + "]") 133 | 134 | if self.type: 135 | args.append(f"type='{self.type}'") 136 | 137 | if self.confidence: 138 | args.append(f"confidence={self.confidence}") 139 | 140 | if self.primary_only: 141 | args.append("primary_only=True") 142 | 143 | if args: 144 | s += ', ' + ', '.join(args) 145 | 146 | s += ")" 147 | return s 148 | -------------------------------------------------------------------------------- /neuprint/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | NEUPRINT_SERVER = 'neuprint.janelia.org' 4 | DATASET = 'hemibrain:v1.2.1' 5 | 6 | try: 7 | TOKEN = os.environ['NEUPRINT_APPLICATION_CREDENTIALS'] 8 | except KeyError: 9 | raise RuntimeError("These tests assume that NEUPRINT_APPLICATION_CREDENTIALS is defined in your environment!") 10 | -------------------------------------------------------------------------------- /neuprint/tests/test_arrow_endpoint.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import patch, MagicMock 3 | from requests.exceptions import HTTPError 4 | from neuprint import Client 5 | from neuprint.tests import NEUPRINT_SERVER, DATASET 6 | 7 | 8 | def test_arrow_endpoint_version_check(): 9 | c = Client(NEUPRINT_SERVER, DATASET) 10 | 11 | # Test with version higher than 1.7.3 12 | with patch.object(c, '_fetch_json') as mock_fetch: 13 | mock_fetch.return_value = {'Version': '1.8.0'} 14 | assert c.arrow_endpoint() is True 15 | 16 | # Test with version exactly 1.7.3 17 | with patch.object(c, '_fetch_json') as mock_fetch: 18 | mock_fetch.return_value = {'Version': '1.7.3'} 19 | assert c.arrow_endpoint() is True 20 | 21 | # Test with version lower than 1.7.3 22 | with patch.object(c, '_fetch_json') as mock_fetch: 23 | mock_fetch.return_value = {'Version': '1.7.1'} 24 | assert c.arrow_endpoint() is False 25 | 26 | # Test with invalid version format 27 | with patch.object(c, '_fetch_json') as mock_fetch: 28 | mock_fetch.return_value = {'Version': 'invalid-version'} 29 | assert c.arrow_endpoint() is False 30 | 31 | # Test with missing Version key 32 | with patch.object(c, '_fetch_json') as mock_fetch: 33 | mock_fetch.return_value = {} 34 | assert c.arrow_endpoint() is False 35 | 36 | # Test with exception 37 | with patch.object(c, '_fetch_json') as mock_fetch: 38 | mock_fetch.side_effect = Exception("Connection error") 39 | assert c.arrow_endpoint() is False 40 | 41 | 42 | def test_fetch_version_error_handling(): 43 | c = Client(NEUPRINT_SERVER, DATASET) 44 | 45 | # Test normal operation 46 | with patch.object(c, '_fetch_json') as mock_fetch: 47 | mock_fetch.return_value = {'Version': '1.8.0'} 48 | assert c.fetch_version() == '1.8.0' 49 | 50 | # Test HTTP error 51 | with patch.object(c, '_fetch_json') as mock_fetch: 52 | mock_fetch.side_effect = HTTPError("404 Client Error: Not Found for url: https://test/api/version") 53 | with pytest.raises(HTTPError): 54 | c.fetch_version() 55 | 56 | # Test missing Version key 57 | with patch.object(c, '_fetch_json') as mock_fetch: 58 | mock_fetch.return_value = {} 59 | with pytest.raises(KeyError): 60 | c.fetch_version() 61 | 62 | # Test unexpected error 63 | with patch.object(c, '_fetch_json') as mock_fetch: 64 | mock_fetch.side_effect = Exception("Unexpected error") 65 | with pytest.raises(Exception): 66 | c.fetch_version() 67 | 68 | 69 | if __name__ == "__main__": 70 | args = ['-s', '--tb=native', '--pyargs', 'neuprint.tests.test_arrow_endpoint'] 71 | pytest.main(args) -------------------------------------------------------------------------------- /neuprint/tests/test_client.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pandas as pd 3 | from neuprint import Client, default_client, set_default_client 4 | from neuprint.client import inject_client 5 | from neuprint.tests import NEUPRINT_SERVER, DATASET 6 | 7 | EXAMPLE_BODY = 5813037876 # Delta6G, Delta6G_04, Traced, non-cropped 8 | 9 | 10 | def test_members(): 11 | set_default_client(None) 12 | with pytest.raises(RuntimeError): 13 | default_client() 14 | c = Client(NEUPRINT_SERVER, DATASET) 15 | assert c.server == f'https://{NEUPRINT_SERVER}' 16 | assert c.dataset == DATASET 17 | 18 | assert default_client() == c 19 | 20 | df = c.fetch_custom("MATCH (m:Meta) RETURN m.primaryRois as rois") 21 | assert isinstance(df, pd.DataFrame) 22 | assert df.columns == ['rois'] 23 | assert len(df) == 1 24 | assert isinstance(df['rois'].iloc[0], list) 25 | 26 | assert isinstance(c.fetch_available(), list) 27 | assert isinstance(c.fetch_help(), str) 28 | assert c.fetch_server_info() is True 29 | assert isinstance(c.fetch_version(), str) 30 | assert isinstance(c.fetch_database(), dict) 31 | assert isinstance(c.fetch_datasets(), dict) 32 | assert isinstance(c.fetch_db_version(), str) 33 | assert isinstance(c.fetch_profile(), dict) 34 | assert isinstance(c.fetch_token(), str) 35 | assert isinstance(c.fetch_daily_type(), tuple) 36 | assert isinstance(c.fetch_roi_completeness(), pd.DataFrame) 37 | assert isinstance(c.fetch_roi_connectivity(), pd.DataFrame) 38 | assert isinstance(c.fetch_roi_mesh('AB(R)'), bytes) 39 | assert isinstance(c.fetch_skeleton(EXAMPLE_BODY), pd.DataFrame) 40 | assert isinstance(c.fetch_neuron_keys(), list) 41 | 42 | 43 | def test_fetch_skeleton(): 44 | c = Client(NEUPRINT_SERVER, DATASET) 45 | orig_df = c.fetch_skeleton(5813027016, False) 46 | healed_df = c.fetch_skeleton(5813027016, True) 47 | 48 | assert len(orig_df) == len(healed_df) 49 | assert (healed_df['link'] == -1).sum() == 1 50 | assert healed_df['link'].iloc[0] == -1 51 | 52 | 53 | @pytest.mark.xfail 54 | def test_broken_members(): 55 | """ 56 | These endpoints are listed in the neuprintHTTP API, 57 | but don't seem to work. 58 | """ 59 | c = Client(NEUPRINT_SERVER, DATASET) 60 | 61 | # Broken. neuprint returns error 500 62 | assert isinstance(c.fetch_instances(), list) 63 | 64 | 65 | @pytest.mark.skip 66 | def test_keyvalue(): 67 | # TODO: 68 | # What is an appropriate key/value to test with? 69 | c = Client(NEUPRINT_SERVER, DATASET) 70 | c.post_raw_keyvalue(instance, key, b'test-test-test') 71 | c.fetch_raw_keyvalue(instance, key) 72 | 73 | 74 | def test_inject_client(): 75 | c = Client(NEUPRINT_SERVER, DATASET, verify=True) 76 | c2 = Client(NEUPRINT_SERVER, DATASET, verify=False) 77 | 78 | set_default_client(c) 79 | 80 | @inject_client 81 | def f(*, client): 82 | return client 83 | 84 | # Uses default client unless client was specified 85 | assert f() == c 86 | assert f(client=c2) == c2 87 | 88 | with pytest.raises(AssertionError): 89 | # Wrong signature -- asserts 90 | @inject_client 91 | def f2(client): 92 | pass 93 | 94 | 95 | if __name__ == "__main__": 96 | args = ['-s', '--tb=native', '--pyargs', 'neuprint.tests.test_client'] 97 | #args += ['-k', 'fetch_skeleton'] 98 | pytest.main(args) 99 | -------------------------------------------------------------------------------- /neuprint/tests/test_neuroncriteria.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | import pytest 4 | import numpy as np 5 | import pandas as pd 6 | 7 | from neuprint import Client, default_client, set_default_client, NeuronCriteria as NC, NotNull, IsNull 8 | from neuprint.queries.neuroncriteria import where_expr 9 | from neuprint.tests import NEUPRINT_SERVER, DATASET 10 | 11 | 12 | @pytest.fixture(scope='module') 13 | def client(): 14 | c = Client(NEUPRINT_SERVER, DATASET) 15 | set_default_client(c) 16 | assert default_client() == c 17 | return c 18 | 19 | 20 | def test_NeuronCriteria(client): 21 | assert NC(bodyId=1).bodyId == [1] 22 | assert NC(bodyId=[1,2,3]).bodyId == [1,2,3] 23 | 24 | # It's important that bodyIds and ROIs are stored as plain lists, 25 | # since we naively serialize them into Cypher queries with that assumption. 26 | assert NC(bodyId=np.array([1,2,3])).bodyId == [1,2,3] 27 | assert NC(bodyId=pd.Series([1,2,3])).bodyId == [1,2,3] 28 | 29 | ## 30 | ## basic_exprs() 31 | ## 32 | assert NC(bodyId=123).basic_exprs() == ["n.bodyId = 123"] 33 | assert NC('m', bodyId=123).basic_exprs() == ["m.bodyId = 123"] 34 | assert NC(bodyId=[123, 456]).basic_exprs() == ["n.bodyId in [123, 456]"] 35 | 36 | assert NC(type='foo.*').regex 37 | assert not NC(type='foo').regex 38 | assert NC(instance='foo.*').regex 39 | assert not NC(instance='foo').regex 40 | 41 | # Cell types really contain parentheses sometimes, 42 | # so we don't want to automatically upgrade to regex mode for parentheses. 43 | assert not NC(type='foo(bar)').regex 44 | assert not NC(instance='foo(bar)').regex 45 | 46 | assert NC(instance="foo").basic_exprs() == ["n.instance = 'foo'"] 47 | assert NC(instance="foo", regex=True).basic_exprs() == ["n.instance =~ 'foo'"] 48 | assert NC(instance=["foo", "bar"]).basic_exprs() == ["n.instance in ['foo', 'bar']"] 49 | assert NC(instance=["foo", "bar"], regex=True).basic_exprs() == ["n.instance =~ '(foo)|(bar)'"] 50 | 51 | assert NC(type="foo").basic_exprs() == ["n.type = 'foo'"] 52 | assert NC(type="foo", regex=True).basic_exprs() == ["n.type =~ 'foo'"] 53 | assert NC(type=["foo", "bar"]).basic_exprs() == ["n.type in ['foo', 'bar']"] 54 | assert NC(type=["foo", "bar"], regex=True).basic_exprs() == ["n.type =~ '(foo)|(bar)'"] 55 | 56 | assert NC(status="foo").basic_exprs() == ["n.status = 'foo'"] 57 | assert NC(status="foo", regex=True).basic_exprs() == ["n.status = 'foo'"] # not regex (status doesn't use regex) 58 | assert NC(status=["foo", "bar"]).basic_exprs() == ["n.status in ['foo', 'bar']"] 59 | assert NC(status=["foo", "bar"], regex=True).basic_exprs() == ["n.status in ['foo', 'bar']"] 60 | 61 | assert NC(cropped=True).basic_exprs() == ["n.cropped"] 62 | assert NC(cropped=False).basic_exprs() == ["(NOT n.cropped OR NOT exists(n.cropped))"] 63 | 64 | assert NC(somaLocation=NotNull).basic_exprs() == ["exists(n.somaLocation)"] 65 | assert NC(somaLocation=IsNull).basic_exprs() == ["NOT exists(n.somaLocation)"] 66 | 67 | assert NC(inputRois=['SMP(R)', 'FB'], outputRois=['FB', 'SIP(R)'], roi_req='all').basic_exprs() == ['(n.FB AND n.`SIP(R)` AND n.`SMP(R)`)'] 68 | assert NC(inputRois=['SMP(R)', 'FB'], outputRois=['FB', 'SIP(R)'], roi_req='any').basic_exprs() == ['(n.FB OR n.`SIP(R)` OR n.`SMP(R)`)'] 69 | 70 | assert NC(min_pre=5).basic_exprs() == ["n.pre >= 5"] 71 | assert NC(min_post=5).basic_exprs() == ["n.post >= 5"] 72 | 73 | assert NC(bodyId=np.arange(1,6)).basic_exprs() == ["n.bodyId in n_search_bodyId"] 74 | 75 | ## 76 | ## basic_conditions() 77 | ## 78 | assert NC().basic_conditions() == "" 79 | assert NC().all_conditions() == "" 80 | assert NC.combined_conditions([NC(), NC(), NC()]) == "" 81 | 82 | # If 3 or fewer items are supplied, then they are used inline within the WHERE clause. 83 | bodies = [1,2,3] 84 | assert NC(bodyId=bodies).basic_conditions(comments=False) == "n.bodyId in [1, 2, 3]" 85 | 86 | # If more than 3 items are specified, then the items are stored in a global variable 87 | # which is referred to within the WHERE clause. 88 | bodies = [1,2,3,4,5] 89 | nc = NC(bodyId=bodies) 90 | assert nc.global_with() == dedent(f"""\ 91 | WITH {bodies} as n_search_bodyId""") 92 | assert nc.basic_conditions(comments=False) == dedent("n.bodyId in n_search_bodyId") 93 | 94 | statuses = ['Traced', 'Orphan'] 95 | nc = NC(status=statuses) 96 | assert nc.basic_conditions(comments=False) == f"n.status in {statuses}" 97 | 98 | statuses = ['Traced', 'Orphan', 'Assign', 'Unimportant'] 99 | nc = NC(status=statuses) 100 | assert nc.global_with() == dedent(f"""\ 101 | WITH {statuses} as n_search_status""") 102 | assert nc.basic_conditions(comments=False) == "n.status in n_search_status" 103 | 104 | # If None is included, then exists() should be checked. 105 | statuses = ['Traced', 'Orphan', 'Assign', None] 106 | nc = NC(status=statuses) 107 | assert nc.global_with() == dedent("""\ 108 | WITH ['Traced', 'Orphan', 'Assign'] as n_search_status""") 109 | assert nc.basic_conditions(comments=False) == dedent("n.status in n_search_status OR NOT exists(n.status)") 110 | 111 | types = ['aaa', 'bbb', 'ccc'] 112 | nc = NC(type=types) 113 | assert nc.basic_conditions(comments=False) == f"n.type in {types}" 114 | 115 | types = ['aaa', 'bbb', 'ccc', 'ddd'] 116 | nc = NC(type=types) 117 | assert nc.global_with() == dedent(f"""\ 118 | WITH {types} as n_search_type""") 119 | assert nc.basic_conditions(comments=False) == "n.type in n_search_type" 120 | 121 | instances = ['aaa', 'bbb', 'ccc'] 122 | nc = NC(instance=instances) 123 | assert nc.basic_conditions(comments=False) == f"n.instance in {instances}" 124 | 125 | instances = ['aaa', 'bbb', 'ccc', 'ddd'] 126 | nc = NC(instance=instances) 127 | assert nc.global_with() == dedent(f"""\ 128 | WITH {instances} as n_search_instance""") 129 | assert nc.basic_conditions(comments=False) == "n.instance in n_search_instance" 130 | 131 | # Special case: 132 | # If both type and instance are supplied, then we combine them with 'OR' 133 | typeinst = ['aaa', 'bbb', 'ccc'] 134 | nc = NC(type=typeinst, instance=typeinst) 135 | assert nc.basic_conditions(comments=False) == f"(n.type in {typeinst} OR n.instance in {typeinst})" 136 | 137 | typeinst = ['aaa', 'bbb', 'ccc', 'ddd'] 138 | nc = NC(type=typeinst, instance=typeinst) 139 | assert nc.basic_conditions(comments=False) == "(n.type in n_search_type OR n.instance in n_search_instance)" 140 | 141 | 142 | def test_where_expr(): 143 | assert where_expr('bodyId', [1], matchvar='m') == 'm.bodyId = 1' 144 | assert where_expr('bodyId', [1,2], matchvar='m') == 'm.bodyId in [1, 2]' 145 | assert where_expr('bodyId', np.array([1,2]), matchvar='m') == 'm.bodyId in [1, 2]' 146 | assert where_expr('bodyId', []) == "" 147 | assert where_expr('instance', ['foo.*'], regex=True, matchvar='m') == "m.instance =~ 'foo.*'" 148 | assert where_expr('instance', ['foo.*', 'bar.*', 'baz.*'], regex=True, matchvar='m') == "m.instance =~ '(foo.*)|(bar.*)|(baz.*)'" 149 | 150 | # We use backticks in the cypher when necessary (but not otherwise). 151 | assert where_expr('foo/bar', [1], matchvar='m') == 'm.`foo/bar` = 1' 152 | assert where_expr('foo/bar', [1,2], matchvar='m') == 'm.`foo/bar` in [1, 2]' 153 | assert where_expr('foo/bar', np.array([1,2]), matchvar='m') == 'm.`foo/bar` in [1, 2]' 154 | assert where_expr('foo/bar', []) == "" 155 | 156 | 157 | if __name__ == "__main__": 158 | args = ['-s', '--tb=native', '--pyargs', 'neuprint.tests.test_neuroncriteria'] 159 | pytest.main(args) 160 | -------------------------------------------------------------------------------- /neuprint/tests/test_queries.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | import pandas as pd 4 | 5 | from neuprint import Client, default_client, set_default_client 6 | from neuprint import (NeuronCriteria as NC, 7 | MitoCriteria as MC, 8 | SynapseCriteria as SC, 9 | fetch_custom, fetch_neurons, fetch_meta, 10 | fetch_all_rois, fetch_primary_rois, fetch_simple_connections, 11 | fetch_adjacencies, fetch_shortest_paths, 12 | fetch_mitochondria, fetch_synapses_and_closest_mitochondria, 13 | fetch_synapses, fetch_mean_synapses, fetch_synapse_connections) 14 | 15 | from neuprint.tests import NEUPRINT_SERVER, DATASET 16 | 17 | @pytest.fixture(scope='module') 18 | def client(): 19 | c = Client(NEUPRINT_SERVER, DATASET) 20 | set_default_client(c) 21 | assert default_client() == c 22 | return c 23 | 24 | 25 | def test_fetch_custom(client): 26 | df = fetch_custom("MATCH (m:Meta) RETURN m.primaryRois as rois") 27 | assert isinstance(df, pd.DataFrame) 28 | assert df.columns == ['rois'] 29 | assert len(df) == 1 30 | assert isinstance(df['rois'].iloc[0], list) 31 | 32 | 33 | def test_fetch_neurons(client): 34 | bodyId = [294792184, 329566174, 329599710, 417199910, 420274150, 35 | 424379864, 425790257, 451982486, 480927537, 481268653] 36 | 37 | # This works but takes a long time. 38 | #neurons, roi_counts = fetch_neurons(NC()) 39 | 40 | neurons, roi_counts = fetch_neurons(NC(bodyId=bodyId)) 41 | assert len(neurons) == len(bodyId) 42 | assert set(roi_counts['bodyId']) == set(bodyId) 43 | 44 | neurons, roi_counts = fetch_neurons(NC(instance='APL_R')) 45 | assert len(neurons) == 1, "There's only one APL neuron in the hemibrain" 46 | assert neurons.loc[0, 'type'] == "APL" 47 | assert neurons.loc[0, 'instance'] == "APL_R" 48 | 49 | neurons, roi_counts = fetch_neurons(NC(instance='APL[^ ]*', regex=True)) 50 | assert len(neurons) == 1, "There's only one APL neuron in the hemibrain" 51 | assert neurons.loc[0, 'type'] == "APL" 52 | assert neurons.loc[0, 'instance'] == "APL_R" 53 | 54 | neurons, roi_counts = fetch_neurons(NC(type='APL.*', regex=True)) 55 | assert len(neurons) == 1, "There's only one APL neuron in the hemibrain" 56 | assert neurons.loc[0, 'type'] == "APL" 57 | assert neurons.loc[0, 'instance'] == "APL_R" 58 | 59 | neurons, roi_counts = fetch_neurons(NC(type=['.*01', '.*02'], regex=True)) 60 | assert len(neurons), "Didn't find any neurons of the given type pattern" 61 | assert all(lambda t: t.endswith('01') or t.endswith('02') for t in neurons['type']) 62 | assert any(lambda t: t.endswith('01') for t in neurons['type']) 63 | assert any(lambda t: t.endswith('02') for t in neurons['type']) 64 | 65 | neurons, roi_counts = fetch_neurons(NC(instance=['.*_L', '.*_R'], regex=True)) 66 | assert len(neurons), "Didn't find any neurons of the given instance pattern" 67 | assert all(lambda t: t.endswith('_L') or t.endswith('_R') for t in neurons['instance']) 68 | 69 | neurons, roi_counts = fetch_neurons(NC(status=['Traced', 'Orphan'], cropped=False)) 70 | assert neurons.eval('status == "Traced" or status == "Orphan"').all() 71 | assert not neurons['cropped'].any() 72 | 73 | neurons, roi_counts = fetch_neurons(NC(inputRois='AL(R)', outputRois='SNP(R)')) 74 | assert all(['AL(R)' in rois for rois in neurons['inputRois']]) 75 | assert all(['SNP(R)' in rois for rois in neurons['outputRois']]) 76 | assert sorted(roi_counts.query('roi == "AL(R)" and post > 0')['bodyId']) == sorted(neurons['bodyId']) 77 | assert sorted(roi_counts.query('roi == "SNP(R)" and pre > 0')['bodyId']) == sorted(neurons['bodyId']) 78 | 79 | neurons, roi_counts = fetch_neurons(NC(min_pre=1000, min_post=2000)) 80 | assert neurons.eval('pre >= 1000 and post >= 2000').all() 81 | 82 | 83 | def test_fetch_simple_connections(client): 84 | bodyId = [294792184, 329566174, 329599710, 417199910, 420274150, 85 | 424379864, 425790257, 451982486, 480927537, 481268653] 86 | 87 | conn_df = fetch_simple_connections(NC(bodyId=bodyId)) 88 | assert set(conn_df['bodyId_pre'].unique()) == set(bodyId) 89 | 90 | conn_df = fetch_simple_connections(None, NC(bodyId=bodyId)) 91 | assert set(conn_df['bodyId_post'].unique()) == set(bodyId) 92 | 93 | APL_R = 425790257 94 | 95 | conn_df = fetch_simple_connections(NC(instance='APL_R')) 96 | assert (conn_df['bodyId_pre'] == APL_R).all() 97 | 98 | conn_df = fetch_simple_connections(NC(type='APL')) 99 | assert (conn_df['bodyId_pre'] == APL_R).all() 100 | 101 | conn_df = fetch_simple_connections(None, NC(instance='APL_R')) 102 | assert (conn_df['bodyId_post'] == APL_R).all() 103 | 104 | conn_df = fetch_simple_connections(None, NC(type='APL')) 105 | assert (conn_df['bodyId_post'] == APL_R).all() 106 | 107 | conn_df = fetch_simple_connections(NC(bodyId=APL_R), min_weight=10) 108 | assert (conn_df['bodyId_pre'] == APL_R).all() 109 | assert (conn_df['weight'] >= 10).all() 110 | 111 | conn_df = fetch_simple_connections(NC(bodyId=APL_R), min_weight=10, properties=['somaLocation']) 112 | assert 'somaLocation_pre' in conn_df 113 | assert 'somaLocation_post' in conn_df 114 | 115 | conn_df = fetch_simple_connections(NC(bodyId=APL_R), min_weight=10, properties=['roiInfo']) 116 | assert 'roiInfo_pre' in conn_df 117 | assert 'roiInfo_post' in conn_df 118 | assert isinstance(conn_df['roiInfo_pre'].iloc[0], dict) 119 | 120 | 121 | def test_fetch_shortest_paths(client): 122 | src = 329566174 123 | dst = 294792184 124 | paths_df = fetch_shortest_paths(src, dst, min_weight=10) 125 | assert (paths_df.groupby('path')['bodyId'].first() == src).all() 126 | assert (paths_df.groupby('path')['bodyId'].last() == dst).all() 127 | 128 | assert (paths_df.groupby('path')['weight'].first() == 0).all() 129 | 130 | 131 | @pytest.mark.skip 132 | def test_fetch_traced_adjacencies(client): 133 | pass 134 | 135 | 136 | def test_fetch_adjacencies(client): 137 | bodies = [294792184, 329566174, 329599710, 417199910, 420274150, 138 | 424379864, 425790257, 451982486, 480927537, 481268653] 139 | neuron_df, roi_conn_df = fetch_adjacencies(NC(bodyId=bodies), NC(bodyId=bodies)) 140 | 141 | # Should not include non-primary ROIs (except 'NotPrimary') 142 | assert not ({*roi_conn_df['roi'].unique()} - {*fetch_primary_rois()} - {'NotPrimary'}) 143 | 144 | # 145 | # For backwards compatibility with the previous API, 146 | # You can also pass a list of bodyIds to this function (instead of NeuronCriteria). 147 | # 148 | bodies = [294792184, 329566174, 329599710, 417199910, 420274150, 149 | 424379864, 425790257, 451982486, 480927537, 481268653] 150 | neuron_df2, roi_conn_df2 = fetch_adjacencies(bodies, bodies) 151 | 152 | # Should not include non-primary ROIs (except 'NotPrimary') 153 | assert not ({*roi_conn_df2['roi'].unique()} - {*fetch_primary_rois()} - {'NotPrimary'}) 154 | 155 | assert (neuron_df.fillna('') == neuron_df2.fillna('')).all().all() 156 | assert (roi_conn_df == roi_conn_df2).all().all() 157 | 158 | # What happens if results are empty 159 | neuron_df, roi_conn_df = fetch_adjacencies(879442155, 5813027103) 160 | assert len(neuron_df) == 0 161 | assert len(roi_conn_df) == 0 162 | assert neuron_df.columns.tolist() == ['bodyId', 'instance', 'type'] 163 | 164 | 165 | def test_fetch_meta(client): 166 | meta = fetch_meta() 167 | assert isinstance(meta, dict) 168 | 169 | 170 | def test_fetch_all_rois(client): 171 | all_rois = fetch_all_rois() 172 | assert isinstance(all_rois, list) 173 | 174 | 175 | def test_fetch_primary_rois(client): 176 | primary_rois = fetch_primary_rois() 177 | assert isinstance(primary_rois, list) 178 | 179 | 180 | def test_fetch_mitochondria(client): 181 | nc = NC(type='ExR.*', regex=True, rois=['EB']) 182 | mc = MC(rois=['FB', 'LAL(R)'], mitoType='dark', size=100_000, primary_only=True) 183 | mito_df = fetch_mitochondria(nc, mc) 184 | assert set(mito_df['roi']) == {'FB', 'LAL(R)'} 185 | assert (mito_df['mitoType'] == 'dark').all() 186 | assert (mito_df['size'] >= 100_000).all() 187 | 188 | neuron_df, _count_df = fetch_neurons(nc) 189 | mito_df = mito_df.merge(neuron_df[['bodyId', 'type']], 'left', on='bodyId', suffixes=['_mito', '_body']) 190 | assert mito_df['type'].isnull().sum() == 0 191 | assert mito_df['type'].apply(lambda s: s.startswith('ExR')).all() 192 | 193 | 194 | def test_fetch_synapses(client): 195 | nc = NC(type='ExR.*', regex=True, rois=['EB']) 196 | sc = SC(rois=['FB', 'LAL(R)'], primary_only=True) 197 | syn_df = fetch_synapses(nc, sc) 198 | assert set(syn_df['roi']) == {'FB', 'LAL(R)'} 199 | 200 | # Ensure proper body set used. 201 | neuron_df, _count_df = fetch_neurons(nc) 202 | syn_df = syn_df.merge(neuron_df[['bodyId', 'type']], 'left', on='bodyId', suffixes=['_syn', '_body']) 203 | assert syn_df['type_body'].isnull().sum() == 0 204 | assert syn_df['type_body'].apply(lambda s: s.startswith('ExR')).all() 205 | 206 | 207 | def test_fetch_mean_synapses(client): 208 | nc = NC(type='ExR.*', regex=True, rois=['EB']) 209 | sc = SC(rois=['FB', 'LAL(R)'], primary_only=True) 210 | mean_df = fetch_mean_synapses(nc, sc) 211 | mean_df = mean_df.sort_values(['bodyId', 'roi', 'type'], ignore_index=True) 212 | assert set(mean_df['roi']) == {'FB', 'LAL(R)'} 213 | 214 | # Ensure proper body set used. 215 | neuron_df, _count_df = fetch_neurons(nc) 216 | mean_df = mean_df.merge(neuron_df[['bodyId', 'type']], 'left', on='bodyId', suffixes=['_syn', '_body']) 217 | assert mean_df['type_body'].isnull().sum() == 0 218 | assert mean_df['type_body'].apply(lambda s: s.startswith('ExR')).all() 219 | 220 | # Compare with locally averaged results 221 | syn_df = fetch_synapses(nc, sc) 222 | expected_df = syn_df.groupby(['bodyId', 'roi', 'type'], observed=True).agg({'x': ['count', 'mean'], 'y': 'mean', 'z': 'mean', 'confidence': 'mean'}).reset_index() 223 | expected_df.columns = ['bodyId', 'roi', 'type', 'count', *'xyz', 'confidence'] 224 | expected_df = expected_df.sort_values(['bodyId', 'roi', 'type'], ignore_index=True) 225 | assert np.allclose(mean_df[[*'xyz', 'confidence']].values, expected_df[[*'xyz', 'confidence']].values) 226 | 227 | 228 | def test_fetch_synapses_and_closest_mitochondria(client): 229 | syn_mito_distances = fetch_synapses_and_closest_mitochondria(NC(type='ExR2'), SC(type='pre')) 230 | assert len(syn_mito_distances), "Shouldn't be empty!" 231 | 232 | 233 | def test_fetch_synapse_connections(client): 234 | rois = ['PED(R)', 'SMP(R)'] 235 | syn_df = fetch_synapse_connections(792368888, None, SC(rois=rois, primary_only=True), batch_size=2) 236 | assert syn_df.eval('roi_pre in @rois and roi_post in @rois').all() 237 | dtypes = syn_df.dtypes.to_dict() 238 | 239 | # Empty results 240 | syn_df = fetch_synapse_connections(879442155, 5813027103) 241 | assert len(syn_df) == 0 242 | assert syn_df.dtypes.to_dict() == dtypes 243 | 244 | 245 | if __name__ == "__main__": 246 | args = ['-s', '--tb=native', '--pyargs', 'neuprint.tests.test_queries'] 247 | #args += ['-k', 'test_fetch_synapse_connections'] 248 | #args += ['-k', 'fetch_synapses_and_closest_mitochondria'] 249 | #args += ['-k', 'fetch_mean_synapses'] 250 | pytest.main(args) 251 | -------------------------------------------------------------------------------- /neuprint/tests/test_skeleton.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import numpy as np 3 | import pandas as pd 4 | import networkx as nx 5 | 6 | from neuprint import Client, default_client, set_default_client 7 | from neuprint import (fetch_skeleton, heal_skeleton, reorient_skeleton, skeleton_df_to_nx, skeleton_df_to_swc, skeleton_swc_to_df) 8 | 9 | from neuprint.tests import NEUPRINT_SERVER, DATASET 10 | 11 | 12 | @pytest.fixture(scope='module') 13 | def client(): 14 | c = Client(NEUPRINT_SERVER, DATASET) 15 | set_default_client(c) 16 | assert default_client() == c 17 | return c 18 | 19 | 20 | @pytest.fixture 21 | def linear_skeleton(): 22 | """ 23 | A test fixture to produce a fake 'skeleton' 24 | with no branches, just 10 nodes in a line. 25 | """ 26 | rows = np.arange(1,11) 27 | coords = np.zeros((10,3), dtype=int) 28 | coords[:,0] = rows**2 29 | radii = rows.astype(np.float32) 30 | links = [-1, *range(1,10)] 31 | 32 | df = pd.DataFrame({'rowId': rows, 33 | 'x': coords[:,0], 34 | 'y': coords[:,1], 35 | 'z': coords[:,2], 36 | 'radius': radii, 37 | 'link': links}) 38 | return df 39 | 40 | 41 | def test_skeleton_df_to_nx(linear_skeleton): 42 | g = skeleton_df_to_nx(linear_skeleton, directed=False) 43 | assert not isinstance(g, nx.DiGraph) 44 | expected_edges = linear_skeleton[['rowId', 'link']].values[1:] 45 | expected_edges.sort(axis=1) 46 | assert (np.array(g.edges) == expected_edges).all() 47 | 48 | g = skeleton_df_to_nx(linear_skeleton, directed=True) 49 | assert isinstance(g, nx.DiGraph) 50 | assert (np.array(g.edges) == linear_skeleton[['rowId', 'link']].values[1:]).all() 51 | 52 | g = skeleton_df_to_nx(linear_skeleton, with_attributes=True) 53 | assert (np.array(g.edges) == linear_skeleton[['rowId', 'link']].values[1:]).all() 54 | for row in linear_skeleton.itertuples(): 55 | attrs = g.nodes[row.rowId] 56 | assert tuple(attrs[k] for k in [*'xyz', 'radius']) == (row.x, row.y, row.z, row.radius) 57 | 58 | 59 | def test_skeleton_df_to_swc(linear_skeleton): 60 | swc = skeleton_df_to_swc(linear_skeleton) 61 | roundtrip_df = skeleton_swc_to_df(swc) 62 | assert (roundtrip_df == linear_skeleton).all().all() 63 | 64 | 65 | def test_reorient_skeleton(linear_skeleton): 66 | s = linear_skeleton.copy() 67 | reorient_skeleton(s, 10) 68 | assert (s['link'] == [*range(2,11), -1]).all() 69 | 70 | s = linear_skeleton.copy() 71 | reorient_skeleton(s, xyz=(100,0,0)) 72 | assert (s['link'] == [*range(2,11), -1]).all() 73 | 74 | s = linear_skeleton.copy() 75 | reorient_skeleton(s, use_max_radius=True) 76 | assert (s['link'] == [*range(2,11), -1]).all() 77 | 78 | 79 | def test_reorient_broken_skeleton(linear_skeleton): 80 | broken_skeleton = linear_skeleton.copy() 81 | broken_skeleton.loc[2, 'link'] = -1 82 | broken_skeleton.loc[7, 'link'] = -1 83 | 84 | s = broken_skeleton.copy() 85 | reorient_skeleton(s, 10) 86 | assert (s['link'].iloc[7:10] == [9,10,-1]).all() 87 | 88 | # reorienting shouldn't change the number of roots, 89 | # though they may change locations. 90 | assert len(s.query('link == -1')) == 3 91 | 92 | 93 | def test_heal_skeleton(linear_skeleton): 94 | broken_skeleton = linear_skeleton.copy() 95 | broken_skeleton.loc[2, 'link'] = -1 96 | broken_skeleton.loc[7, 'link'] = -1 97 | 98 | healed_skeleton = heal_skeleton(broken_skeleton) 99 | assert (healed_skeleton == linear_skeleton).all().all() 100 | 101 | 102 | def test_heal_skeleton_with_threshold(linear_skeleton): 103 | broken_skeleton = linear_skeleton.copy() 104 | broken_skeleton.loc[2, 'link'] = -1 105 | broken_skeleton.loc[7, 'link'] = -1 106 | 107 | healed_skeleton = heal_skeleton(broken_skeleton, 10.0) 108 | 109 | # With a threshold of 10, the first break could be healed, 110 | # but not the second. 111 | expected_skeleton = linear_skeleton.copy() 112 | expected_skeleton.loc[7, 'link'] = -1 113 | assert (healed_skeleton == expected_skeleton).all().all() 114 | 115 | 116 | def test_fetch_skeleton(client): 117 | orig_df = fetch_skeleton(5813027016, False) 118 | healed_df = fetch_skeleton(5813027016, True) 119 | 120 | assert len(orig_df) == len(healed_df) 121 | assert (healed_df['link'] == -1).sum() == 1 122 | assert healed_df['link'].iloc[0] == -1 123 | 124 | 125 | @pytest.mark.skip("Need to write a test for skeleton_segments()") 126 | def test_skeleton_segments(linear_skeleton): 127 | pass 128 | 129 | 130 | if __name__ == "__main__": 131 | args = ['-s', '--tb=native', '--pyargs', 'neuprint.tests.test_skeleton'] 132 | #args += ['-k', 'heal_skeleton'] 133 | pytest.main(args) 134 | -------------------------------------------------------------------------------- /neuprint/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import numpy as np 3 | from neuprint.utils import ensure_list, ensure_list_args 4 | 5 | 6 | def test_ensure_list(): 7 | assert ensure_list(None) == [] 8 | assert ensure_list([None]) == [None] 9 | 10 | assert ensure_list(1) == [1] 11 | assert ensure_list([1]) == [1] 12 | 13 | assert isinstance(ensure_list(np.array([1,2,3])), list) 14 | 15 | 16 | def test_ensure_list_args(): 17 | 18 | @ensure_list_args(['a', 'c', 'd']) 19 | def f(a, b, c, d='d', *, e=None): 20 | return (a,b,c,d,e) 21 | 22 | # Must preserve function signature 23 | spec = inspect.getfullargspec(f) 24 | assert spec.args == ['a', 'b', 'c', 'd'] 25 | assert spec.defaults == ('d',) 26 | assert spec.kwonlyargs == ['e'] 27 | assert spec.kwonlydefaults == {'e': None} 28 | 29 | # Check results 30 | assert f('a', 'b', 'c', 'd') == (['a'], 'b', ['c'], ['d'], None) 31 | -------------------------------------------------------------------------------- /neuprint/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions for manipulating neuprint-python output. 3 | """ 4 | import re 5 | import os 6 | import sys 7 | import inspect 8 | import functools 9 | import warnings 10 | from textwrap import dedent 11 | from collections.abc import Iterable, Iterator, Collection 12 | 13 | import numpy as np 14 | import pandas as pd 15 | from requests import Session 16 | import ujson 17 | 18 | 19 | class NotNull: 20 | """Filter for existing properties. 21 | 22 | Translates to:: 23 | 24 | WHERE neuron.{property} IS NOT NULL 25 | 26 | """ 27 | 28 | 29 | class IsNull: 30 | """Filter for missing properties. 31 | 32 | Translates to:: 33 | 34 | WHERE neuron.{property} IS NULL 35 | 36 | """ 37 | 38 | 39 | CYPHER_KEYWORDS = [ 40 | "CALL", "CREATE", "DELETE", "DETACH", "FOREACH", "LOAD", "MATCH", "MERGE", "OPTIONAL", "REMOVE", "RETURN", "SET", "START", "UNION", "UNWIND", "WITH", 41 | "LIMIT", "ORDER", "SKIP", "WHERE", "YIELD", 42 | "ASC", "ASCENDING", "ASSERT", "BY", "CSV", "DESC", "DESCENDING", "ON", 43 | "ALL", "CASE", "COUNT", "ELSE", "END", "EXISTS", "THEN", "WHEN", 44 | "AND", "AS", "CONTAINS", "DISTINCT", "ENDS", "IN", "IS", "NOT", "OR", "STARTS", "XOR", 45 | "CONSTRAINT", "CREATE", "DROP", "EXISTS", "INDEX", "NODE", "KEY", "UNIQUE", 46 | "INDEX", "JOIN", "SCAN", "USING", 47 | "FALSE", "NULL", "TRUE", 48 | "ADD", "DO", "FOR", "MANDATORY", "OF", "REQUIRE", "SCALAR" 49 | ] 50 | 51 | # Technically this pattern is too strict, as it doesn't allow for non-ascii letters, 52 | # but that's okay -- we just might use backticks a little more often than necessary. 53 | CYPHER_IDENTIFIER_PATTERN = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$') 54 | 55 | 56 | def cypher_identifier(name): 57 | """ 58 | Wrap the given name in backticks if it wouldn't be a vlid cypher identifier without them. 59 | """ 60 | if name.upper() in CYPHER_KEYWORDS or not CYPHER_IDENTIFIER_PATTERN.match(name): 61 | return f"`{name}`" 62 | return name 63 | 64 | 65 | # 66 | # Import the notebook-aware version of tqdm if 67 | # we appear to be running within a notebook context. 68 | # 69 | try: 70 | import ipykernel.iostream 71 | if isinstance(sys.stdout, ipykernel.iostream.OutStream): 72 | from tqdm.notebook import tqdm 73 | 74 | try: 75 | import ipywidgets 76 | ipywidgets 77 | except ImportError: 78 | msg = dedent("""\ 79 | 80 | Progress bar will not work well in the notebook without ipywidgets. 81 | Run the following commands (for notebook and jupyterlab users): 82 | 83 | conda install -c conda-forge ipywidgets 84 | jupyter nbextension enable --py widgetsnbextension 85 | jupyter labextension install @jupyter-widgets/jupyterlab-manager 86 | 87 | ...and then reload your jupyter session, and restart your kernel. 88 | """) 89 | warnings.warn(msg) 90 | else: 91 | from tqdm import tqdm 92 | 93 | except ImportError: 94 | from tqdm import tqdm 95 | 96 | 97 | class tqdm(tqdm): 98 | """ 99 | Same as tqdm, but auto-disable the progress bar if there's only one item. 100 | """ 101 | def __init__(self, iterable=None, *args, disable=None, **kwargs): 102 | if disable is None: 103 | disable = (iterable is not None 104 | and hasattr(iterable, '__len__') 105 | and len(iterable) <= 1) 106 | 107 | super().__init__(iterable, *args, disable=disable, **kwargs) 108 | 109 | 110 | def trange(*args, **kwargs): 111 | return tqdm(range(*args), **kwargs) 112 | 113 | 114 | def UMAP(*args, **kwargs): 115 | """ 116 | UMAP is an optional dependency, so this wrapper emits 117 | a nicer error message if it's not available. 118 | """ 119 | try: 120 | from umap import UMAP 121 | except ImportError as ex: 122 | msg = ( 123 | "The 'umap' dimensionality reduction package is required for some " 124 | "plotting functionality, but it isn't currently installed.\n\n" 125 | "Please install it:\n\n" 126 | " conda install -c conda-forge umap-learn\n\n" 127 | ) 128 | raise RuntimeError(msg) from ex 129 | 130 | return UMAP(*args, **kwargs) 131 | 132 | 133 | def ensure_list(x): 134 | """ 135 | If ``x`` is already a list, return it unchanged. 136 | If ``x`` is Series or ndarray, convert to plain list. 137 | If ``x`` is ``None``, return an empty list ``[]``. 138 | Otherwise, wrap it in a list. 139 | """ 140 | if x is None: 141 | return [] 142 | 143 | if isinstance(x, (np.ndarray, pd.Series)): 144 | return x.tolist() 145 | 146 | if isinstance(x, Collection) and not isinstance(x, str): 147 | # Note: 148 | # This is a convenient way to handle all of these cases: 149 | # np.array([1, 2, 3]) -> [1, 2, 3] 150 | # [1, 2, 3] -> [1, 2, 3] 151 | # [np.int64(1), np.int64(2), np.int64(3)] -> [1, 2, 3] 152 | return np.asarray(x).tolist() 153 | else: 154 | return [x] 155 | 156 | 157 | def ensure_list_args(argnames): 158 | """ 159 | Returns a decorator. 160 | For the given argument names, the decorator converts the 161 | arguments into iterables via ``ensure_list()``. 162 | """ 163 | def decorator(f): 164 | 165 | @functools.wraps(f) 166 | def wrapper(*args, **kwargs): 167 | callargs = inspect.getcallargs(f, *args, **kwargs) 168 | for name in argnames: 169 | callargs[name] = ensure_list(callargs[name]) 170 | return f(**callargs) 171 | 172 | wrapper.__signature__ = inspect.signature(f) 173 | return wrapper 174 | 175 | return decorator 176 | 177 | 178 | def ensure_list_attrs(attributes): 179 | """ 180 | Returns a *class* decorator. 181 | For the given attribute names, the decorator adds "private" 182 | attributes (e.g. bodyId -> _bodyId) and declares getter/setter properties. 183 | The setter property converts the new value to a list before storing 184 | it in the private attribute. 185 | 186 | Classes which require their members to be a true list can allow users to 187 | set attributes as np.array. 188 | """ 189 | def decorator(cls): 190 | for attr in attributes: 191 | private_attr = f"_{attr}" 192 | 193 | def getter(self, private_attr=private_attr): 194 | return getattr(self, private_attr) 195 | 196 | def setter(self, value, private_attr=private_attr): 197 | value = ensure_list(value) 198 | setattr(self, private_attr, value) 199 | 200 | setattr(cls, attr, property(getter, setter)) 201 | 202 | return cls 203 | return decorator 204 | 205 | 206 | @ensure_list_args(['properties']) 207 | def merge_neuron_properties(neuron_df, conn_df, properties=['type', 'instance']): 208 | """ 209 | Merge neuron properties to a connection table. 210 | 211 | Given a table of neuron properties and a connection table, append 212 | ``_pre`` and ``_post`` columns to the connection table for each of 213 | the given properties via the appropriate merge operations. 214 | 215 | Args: 216 | neuron_df: 217 | DataFrame with columns for 'bodyId' and any properties you want to merge 218 | 219 | conn_df: 220 | DataFrame with columns ``bodyId_pre`` and ``bodyId_post`` 221 | 222 | properties: 223 | Column names from ``neuron_df`` to merge onto ``conn_df``. 224 | 225 | Returns: 226 | Updated ``conn_df`` with new columns. 227 | 228 | Example: 229 | 230 | .. code-block:: ipython 231 | 232 | In [1]: from neuprint import fetch_adjacencies, NeuronCriteria as NC, merge_neuron_properties 233 | ...: neuron_df, conn_df = fetch_adjacencies(rois='PB', min_roi_weight=120) 234 | ...: print(conn_df) 235 | bodyId_pre bodyId_post roi weight 236 | 0 880875736 1631450739 PB 123 237 | 1 880880259 849421763 PB 141 238 | 2 910442723 849421763 PB 139 239 | 3 910783961 5813070465 PB 184 240 | 4 911129204 724280817 PB 127 241 | 5 911134009 849421763 PB 125 242 | 6 911565419 5813070465 PB 141 243 | 7 911911004 1062526223 PB 125 244 | 8 911919044 973566036 PB 122 245 | 9 5813080838 974239375 PB 136 246 | 247 | In [2]: merge_neuron_properties(neuron_df, conn_df, 'type') 248 | Out[2]: 249 | bodyId_pre bodyId_post roi weight type_pre type_post 250 | 0 880875736 1631450739 PB 123 Delta7_a PEN_b(PEN2) 251 | 1 880880259 849421763 PB 141 Delta7_a PEN_b(PEN2) 252 | 2 910442723 849421763 PB 139 Delta7_a PEN_b(PEN2) 253 | 3 910783961 5813070465 PB 184 Delta7_a PEN_b(PEN2) 254 | 4 911129204 724280817 PB 127 Delta7_a PEN_b(PEN2) 255 | 5 911134009 849421763 PB 125 Delta7_a PEN_b(PEN2) 256 | 6 911565419 5813070465 PB 141 Delta7_a PEN_b(PEN2) 257 | 7 911911004 1062526223 PB 125 Delta7_b PEN_b(PEN2) 258 | 8 911919044 973566036 PB 122 Delta7_a PEN_b(PEN2) 259 | 9 5813080838 974239375 PB 136 EPG PEG 260 | """ 261 | neuron_df = neuron_df[['bodyId', *properties]] 262 | 263 | newcols = [f'{prop}_pre' for prop in properties] 264 | newcols += [f'{prop}_post' for prop in properties] 265 | conn_df = conn_df.drop(columns=newcols, errors='ignore') 266 | 267 | conn_df = conn_df.merge(neuron_df, 'left', left_on='bodyId_pre', right_on='bodyId') 268 | del conn_df['bodyId'] 269 | 270 | conn_df = conn_df.merge(neuron_df, 'left', left_on='bodyId_post', right_on='bodyId', 271 | suffixes=['_pre', '_post']) 272 | del conn_df['bodyId'] 273 | 274 | return conn_df 275 | 276 | 277 | def connection_table_to_matrix(conn_df, group_cols='bodyId', weight_col='weight', sort_by=None, make_square=False): 278 | """ 279 | Given a weighted connection table, produce a weighted adjacency matrix. 280 | 281 | Args: 282 | conn_df: 283 | A DataFrame with columns for pre- and post- identifiers 284 | (e.g. bodyId, type or instance), and a column for the 285 | weight of the connection. 286 | 287 | group_cols: 288 | Which two columns to use as the row index and column index 289 | of the returned matrix, respetively. 290 | Or give a single string (e.g. ``"body"``, in which case the 291 | two column names are chosen by appending the suffixes 292 | ``_pre`` and ``_post`` to your string. 293 | 294 | If a pair of pre/post values occurs more than once in the 295 | connection table, all of its weights will be summed in the 296 | output matrix. 297 | 298 | weight_col: 299 | Which column holds the connection weight, to be aggregated for each unique pre/post pair. 300 | 301 | sort_by: 302 | How to sort the rows and columns of the result. 303 | Can be two strings, e.g. ``("type_pre", "type_post")``, 304 | or a single string, e.g. ``"type"`` in which case the suffixes are assumed. 305 | 306 | make_square: 307 | If True, insert rows and columns to ensure that the same IDs exist in the rows and columns. 308 | Inserted entries will have value 0.0 309 | 310 | Returns: 311 | DataFrame, shape NxM, where N is the number of unique values in 312 | the 'pre' group column, and M is the number of unique values in 313 | the 'post' group column. 314 | 315 | Example: 316 | 317 | .. code-block:: ipython 318 | 319 | In [1]: from neuprint import fetch_simple_connections, NeuronCriteria as NC 320 | ...: kc_criteria = NC(type='KC.*') 321 | ...: conn_df = fetch_simple_connections(kc_criteria, kc_criteria) 322 | In [1]: conn_df.head() 323 | Out[1]: 324 | bodyId_pre bodyId_post weight type_pre type_post instance_pre instance_post conn_roiInfo 325 | 0 1224137495 5813032771 29 KCg KCg KCg KCg(super) {'MB(R)': {'pre': 26, 'post': 26}, 'gL(R)': {'... 326 | 1 1172713521 5813067826 27 KCg KCg KCg(super) KCg-d {'MB(R)': {'pre': 26, 'post': 26}, 'PED(R)': {... 327 | 2 517858947 5813032943 26 KCab-p KCab-p KCab-p KCab-p {'MB(R)': {'pre': 25, 'post': 25}, 'PED(R)': {... 328 | 3 642680826 5812980940 25 KCab-p KCab-p KCab-p KCab-p {'MB(R)': {'pre': 25, 'post': 25}, 'PED(R)': {... 329 | 4 5813067826 1172713521 24 KCg KCg KCg-d KCg(super) {'MB(R)': {'pre': 23, 'post': 23}, 'gL(R)': {'... 330 | 331 | In [2]: from neuprint.utils import connection_table_to_matrix 332 | ...: connection_table_to_matrix(conn_df, 'type') 333 | Out[2]: 334 | type_post KC KCa'b' KCab-p KCab-sc KCg 335 | type_pre 336 | KC 3 139 6 5 365 337 | KCa'b' 154 102337 245 997 1977 338 | KCab-p 7 310 17899 3029 127 339 | KCab-sc 4 2591 3975 247038 3419 340 | KCg 380 1969 79 1526 250351 341 | """ 342 | if isinstance(group_cols, str): 343 | group_cols = (f"{group_cols}_pre", f"{group_cols}_post") 344 | 345 | assert len(group_cols) == 2, \ 346 | "Please provide two group_cols (e.g. 'bodyId_pre', 'bodyId_post')" 347 | 348 | assert group_cols[0] in conn_df, \ 349 | f"Column missing: {group_cols[0]}" 350 | 351 | assert group_cols[1] in conn_df, \ 352 | f"Column missing: {group_cols[1]}" 353 | 354 | assert weight_col in conn_df, \ 355 | f"Column missing: {weight_col}" 356 | 357 | col_pre, col_post = group_cols 358 | dtype = conn_df[weight_col].dtype 359 | 360 | agg_weights_df = conn_df.groupby([col_pre, col_post], sort=False)[weight_col].sum().reset_index() 361 | matrix = agg_weights_df.pivot(index=col_pre, columns=col_post, values=weight_col) 362 | matrix = matrix.fillna(0).astype(dtype) 363 | 364 | if sort_by: 365 | if isinstance(sort_by, str): 366 | sort_by = (f"{sort_by}_pre", f"{sort_by}_post") 367 | 368 | assert len(sort_by) == 2, \ 369 | "Please provide two sort_by column names (e.g. 'type_pre', 'type_post')" 370 | 371 | pre_order = conn_df.sort_values(sort_by[0])[col_pre].unique() 372 | post_order = conn_df.sort_values(sort_by[1])[col_post].unique() 373 | matrix = matrix.reindex(index=pre_order, columns=post_order) 374 | else: 375 | # No sort: Keep the order as close to the input order as possible. 376 | pre_order = conn_df[col_pre].unique() 377 | post_order = conn_df[col_post].unique() 378 | matrix = matrix.reindex(index=pre_order, columns=post_order) 379 | 380 | if make_square: 381 | matrix, _ = matrix.align(matrix.T) 382 | matrix = matrix.fillna(0.0).astype(matrix.dtypes) 383 | 384 | matrix = matrix.rename_axis(col_pre, axis=0).rename_axis(col_post, axis=1) 385 | matrix = matrix.loc[ 386 | sorted(matrix.index, key=lambda s: s if s else ""), 387 | sorted(matrix.columns, key=lambda s: s if s else "") 388 | ] 389 | 390 | return matrix 391 | 392 | 393 | def iter_batches(it, batch_size): 394 | """ 395 | Iterator. 396 | 397 | Consume the given iterator/iterable in batches and 398 | yield each batch as a list of items. 399 | 400 | The last batch might be smaller than the others, 401 | if there aren't enough items to fill it. 402 | 403 | If the given iterator supports the __len__ method, 404 | the returned batch iterator will, too. 405 | """ 406 | if hasattr(it, '__len__'): 407 | return _iter_batches_with_len(it, batch_size) 408 | else: 409 | return _iter_batches(it, batch_size) 410 | 411 | 412 | class _iter_batches: 413 | def __init__(self, it, batch_size): 414 | self.base_iterator = it 415 | self.batch_size = batch_size 416 | 417 | 418 | def __iter__(self): 419 | return self._iter_batches(self.base_iterator, self.batch_size) 420 | 421 | 422 | def _iter_batches(self, it, batch_size): 423 | if isinstance(it, (pd.DataFrame, pd.Series)): 424 | for batch_start in range(0, len(it), batch_size): 425 | yield it.iloc[batch_start:batch_start+batch_size] 426 | return 427 | elif isinstance(it, (list, np.ndarray)): 428 | for batch_start in range(0, len(it), batch_size): 429 | yield it[batch_start:batch_start+batch_size] 430 | return 431 | else: 432 | if not isinstance(it, Iterator): 433 | assert isinstance(it, Iterable) 434 | it = iter(it) 435 | 436 | while True: 437 | batch = [] 438 | try: 439 | for _ in range(batch_size): 440 | batch.append(next(it)) 441 | except StopIteration: 442 | return 443 | finally: 444 | if batch: 445 | yield batch 446 | 447 | 448 | class _iter_batches_with_len(_iter_batches): 449 | def __len__(self): 450 | return int(np.ceil(len(self.base_iterator) / self.batch_size)) 451 | 452 | 453 | def compile_columns(client, core_columns=[]): 454 | """ 455 | Compile list of columns from available :Neuron keys (excluding ROIs). 456 | 457 | Args: 458 | client: 459 | neu.Client to collect columns for. 460 | core_columns: 461 | List of core columns (optional). If provided, new columns will be 462 | added to the end of the list and non-existing columns will be 463 | dropped. 464 | 465 | Returns: 466 | columns: 467 | List of key names. 468 | """ 469 | # Fetch existing keys. This call is cached. 470 | keys = client.fetch_neuron_keys() 471 | 472 | # Drop ROIs 473 | keys = [k for k in keys if k not in client.all_rois] 474 | 475 | # Drop missing columns from core_columns 476 | columns = [k for k in core_columns if k in keys] 477 | 478 | # Add new keys (sort to make deterministic) 479 | columns += [k for k in sorted(keys) if k not in columns] 480 | 481 | return columns 482 | 483 | def available_datasets(server, token=None): 484 | """ 485 | Get a list of available datasets for a specified server. 486 | Args: 487 | server: URL of neuprintHttp server 488 | token: neuPrint token. If null, will use 489 | ``NEUPRINT_APPLICATION_CREDENTIALS`` environment variable. 490 | Your token can be retrieved by clicking on your account in 491 | the NeuPrint web interface. 492 | Returns: 493 | List of available datasets 494 | """ 495 | # Token 496 | if not token: 497 | token = os.environ.get('NEUPRINT_APPLICATION_CREDENTIALS') 498 | if not token: 499 | raise RuntimeError("No token provided. Please provide one or set NEUPRINT_APPLICATION_CREDENTIALS") 500 | if ':' in token: 501 | try: 502 | token = ujson.loads(token)['token'] 503 | except Exception as ex: 504 | raise RuntimeError("Did not understand token. Please provide the entire JSON document or (only) the complete token string") from ex 505 | token = token.replace('"', '') 506 | # Server 507 | if '://' not in server: 508 | server = 'https://' + server 509 | elif server.startswith('http://'): 510 | raise RuntimeError("Server must be https, not http") 511 | elif not server.startswith('https://'): 512 | protocol = server.split('://')[0] 513 | raise RuntimeError(f"Unknown protocol: {protocol}") 514 | while server.endswith('/'): 515 | server = server[:-1] 516 | # Request 517 | with Session() as session: 518 | session.headers.update({'Authorization': f'Bearer {token}'}) 519 | response = session.get(f"{server}/api/dbmeta/datasets") 520 | response.raise_for_status() 521 | return list(response.json()) 522 | -------------------------------------------------------------------------------- /neuprint/wrangle.py: -------------------------------------------------------------------------------- 1 | """ 2 | Miscellaneous utilities for wrangling data from neuprint for various purposes. 3 | """ 4 | import pandas as pd 5 | import numpy as np 6 | 7 | 8 | def syndist_matrix(syndist, rois=None, syn_columns=['pre', 'post'], flatten_column_index=False): 9 | """ 10 | Pivot a synapse ROI counts table (one row per body). 11 | 12 | Given a table of synapse ROI distributions as returned by :py:func:`.fetch_neurons()`, 13 | pivot the ROIs into the columns so the result has one row per body. 14 | 15 | Args: 16 | syndist: 17 | DataFrame in the format returned by ``fetch_neurons()[1]`` 18 | rois: 19 | Optionally filter the input table to process only the listed ROIs. 20 | syn_columns: 21 | Optionally process only the given columns of syndist. 22 | flatten_column_index: 23 | By default, the result columns will use a MultiIndex ``(orig_col, roi)``, 24 | e.g. ``('pre', 'LO(R)')``. If ``flatten_column_index=True``, then the 25 | output column index is flattened to a plain index with names like ``LO(R)-pre``. 26 | Returns: 27 | DataFrame indexed by bodyId and with column count C * R, where C 28 | is the number of original columns (not counting bodId and roi), 29 | and R is the number of unique rois in the input. 30 | 31 | Example: 32 | 33 | .. code-block:: ipython 34 | 35 | In [1]: from neuprint import Client, fetch_neurons, syndist_matrix 36 | ...: c = Client('neuprint.janelia.org', 'hemibrain:v1.2.1') 37 | ...: bodies = [786989471, 925548084, 1102514975, 1129042596, 1292847181, 5813080979] 38 | ...: neurons, syndist = fetch_neurons(bodies) 39 | ...: syndist_matrix(syndist, ['EB', 'FB', 'PB']) 40 | Out[1]: 41 | pre post 42 | roi EB FB PB EB FB PB 43 | bodyId 44 | 786989471 0 110 11 0 1598 157 45 | 925548084 0 542 0 0 977 0 46 | 1102514975 0 236 0 1 1338 0 47 | 1129042596 0 139 0 0 1827 0 48 | 1292847181 916 0 0 1558 0 0 49 | 5813080979 439 0 0 748 0 451 50 | """ 51 | if rois is not None: 52 | syndist = syndist.query('roi in @rois') 53 | if syn_columns is not None and len(syn_columns) > 0: 54 | syndist = syndist[['bodyId', 'roi', *syn_columns]] 55 | 56 | matrix = syndist.set_index(['bodyId', 'roi']).unstack(fill_value=0) 57 | 58 | if flatten_column_index: 59 | matrix.columns = [f"{roi}-{prepost}" for (prepost, roi) in matrix.columns.values] 60 | 61 | return matrix 62 | 63 | 64 | def bilateral_syndist(syndist, bodies=None, rois=None, syn_columns=['pre', 'post']): 65 | """ 66 | Aggregate synapse counts for corresponding left and right ROIs. 67 | 68 | Given a synapse distribution table as returned by :py:func:`.fetch_neurons()` 69 | (in its second return value), group corresponding contralateral ROIs 70 | (suffixed with ``(L)`` and ``(R)``) and aggregate their synapse counts 71 | into total 'bilateral' counts with the suffix ``(LR)``. 72 | 73 | ROIs without a suffix ``(L)``/``(R)`` will be returned in the output unchanged. 74 | 75 | Args: 76 | syndist: 77 | DataFrame in the format returned by ``fetch_neurons()[1]`` 78 | bodies: 79 | Optionally filter the input table to include only the listed body IDs. 80 | rois: 81 | Optionally filter the input table to process only the listed ROIs. 82 | syn_columns: 83 | The names of the statistic columns in the input to process. 84 | Others are ignored. 85 | Returns: 86 | DataFrame, similar to the input table but with left/right ROIs aggregated 87 | and named with a ``(LR)`` suffix. 88 | 89 | Example: 90 | 91 | .. code-block:: ipython 92 | 93 | In [1]: from neuprint import Client, fetch_neurons, bilateral_syndist 94 | ...: c = Client('neuprint.janelia.org', 'hemibrain:v1.2.1') 95 | ...: bodies = [786989471, 925548084, 1102514975, 1129042596, 1292847181, 5813080979] 96 | ...: neurons, syndist = fetch_neurons(bodies) 97 | ...: bilateral_syndist(syndist, rois=c.primary_rois) 98 | Out[1]: 99 | bodyId roi pre post 100 | 0 786989471 CRE(LR) 77 75 101 | 3 786989471 FB 110 1598 102 | 1 786989471 LAL(LR) 2 2 103 | 14 786989471 PB 11 157 104 | 2 925548084 CRE(LR) 1 203 105 | 22 925548084 FB 542 977 106 | 3 925548084 SMP(LR) 1 171 107 | 4 1102514975 CRE(LR) 2 190 108 | 35 1102514975 EB 0 1 109 | 37 1102514975 FB 236 1338 110 | 5 1102514975 ICL(LR) 0 1 111 | 6 1102514975 LAL(LR) 0 3 112 | 7 1102514975 SMP(LR) 0 74 113 | 8 1102514975 b'L(LR) 0 4 114 | 55 1129042596 FB 139 1827 115 | 9 1129042596 ICL(LR) 0 2 116 | 10 1292847181 BU(LR) 5 143 117 | 67 1292847181 EB 916 1558 118 | 11 1292847181 LAL(LR) 0 1 119 | 77 5813080979 EB 439 748 120 | 82 5813080979 NO 105 451 121 | 86 5813080979 PB 0 451 122 | """ 123 | if bodies is not None: 124 | syndist = syndist.query('bodyId in @syndist').copy() 125 | 126 | if rois is not None: 127 | syndist = syndist.query('roi in @rois').copy() 128 | 129 | if syn_columns is not None and len(syn_columns) > 0: 130 | syndist = syndist[['bodyId', 'roi', *syn_columns]] 131 | 132 | lateral_matches = syndist['roi'].str.match(r'.*\((R|L)\)') 133 | syndist_lateral = syndist.loc[lateral_matches].copy() 134 | syndist_medial = syndist.loc[~lateral_matches].copy() 135 | syndist_lateral['roi'] = syndist_lateral['roi'].str.slice(0, -3) 136 | 137 | syndist_bilateral = syndist_lateral.groupby(['bodyId', 'roi'], as_index=False).sum() 138 | syndist_bilateral['roi'] = syndist_bilateral['roi'] + '(LR)' 139 | 140 | syndist_bilateral = pd.concat((syndist_medial, syndist_bilateral)) 141 | syndist_bilateral = syndist_bilateral.sort_values(['bodyId', 'roi']) 142 | return syndist_bilateral 143 | 144 | 145 | def assign_sides_in_groups(neurons, syndist, primary_rois=None, min_pre=50, min_post=100, min_bias=0.7): 146 | """ 147 | Determine which side (left or right) each neuron belongs to, 148 | according to a few heuristics. 149 | 150 | Assigns a column named 'consensusSide' to the given neurons table. 151 | The consensusSide is only assigned for neurons with an assigned ``group``, 152 | and only if every neuron in the group can be assigned a side using 153 | the same heuristic. 154 | 155 | The neurons are processed in groups (according to the ``group`` column). 156 | Multiple heuristics are tried: 157 | 158 | - If all neurons in the group have a valid ``somaSide``, then that's used. 159 | - Otherwise, if all neurons in the group have an instance ending with 160 | ``_L`` or ``_R``, then that is used. 161 | - Otherwise, we inspect the pre- and post-synapse counts in ROIs which end 162 | with ``(L)`` or ``(R)``: 163 | 164 | - If all neurons in the group have significantly more post-synapses 165 | on one side, then the balance post-synapse is used to assign the 166 | neuron side. 167 | - Otherwise, if all neurons in the group have significantly more 168 | pre-synapses on one side, then that's used. 169 | - But we do not use either heuristic if there is any disagreement 170 | on the relative lateral direction in which the neurons in the group 171 | project. If some seem to project contralaterally and others seem to 172 | project ipsilaterally, we do not assign a consensusSide to any neurons 173 | in the group. 174 | 175 | Args: 176 | neurons: 177 | As produced by :py:func:`.fetch_neurons()` 178 | syndist: 179 | As produced by :py:func:`.fetch_neurons()` 180 | primary_rois: 181 | To avoid double-counting synapses in overlapping ROIs, it is best to 182 | restrict the syndist table to non-overlapping ROIs only (e.g. primary ROIs). 183 | Provide the list of such ROIs here, or pre-filter the input yourself. 184 | min_pre: 185 | When determining a neuron's side via synapse counts, don't analyze 186 | pre-synapses in neurons with fewer than ``min_pre`` pre-synapses. 187 | min_post: 188 | When determining a neuron's side via synapse counts, don't analyze 189 | post-synapses in neurons with fewer than ``min_post`` post-synapses. 190 | min_bias: 191 | When determining a neuron's side via synapse counts, don't assign a 192 | consensusSide unless each neuron in the group has a significant fraction 193 | of its lateral synapses on either the left or right, as specified 194 | in this argument. By default, only assign a consensusSide if 70% 195 | of post-synapses are on one side, or 70% of pre-synapses are on one 196 | side (not counting synapses in medial ROIs). 197 | 198 | Returns: 199 | DataFrame, indexed by bodyId, with column ``consensusSide`` (all values 200 | ``L``, ``R``, or ``None``) and various auxiliary columns which indicate 201 | how the consensus was determined. 202 | """ 203 | neurons = neurons.copy() 204 | neurons.index = neurons['bodyId'] 205 | 206 | # According to instance, what side is the neuron on? 207 | neurons['instanceSide'] = neurons['instance'].astype(str).str.extract('.*_(R|L)(_.*)?')[0] 208 | 209 | # According to the fraction of pre and post, what side is the neuron on? 210 | if primary_rois is not None: 211 | syndist = syndist.query('roi in @primary_rois') 212 | 213 | syndist = syndist.copy() 214 | syndist['roiSide'] = syndist['roi'].str.extract(r'.*\((R|L)\)').fillna('M') 215 | body_roi_sums = syndist.groupby(['bodyId', 'roiSide'])[['pre', 'post']].sum() 216 | body_sidecounts = body_roi_sums.unstack(fill_value=0) 217 | 218 | # body_sums = body_roi_sums.groupby('bodyId').sum() 219 | # body_sidefrac = (body_roi_sums / body_sums).unstack(fill_value=0) 220 | # body_sidefrac.columns = [f"{prepost}_frac_{side}" for (prepost, side) in body_sidefrac.columns.values] 221 | 222 | neurons['preSide'] = ( 223 | body_sidecounts['pre'] 224 | .query('(L + M + R) >= @min_pre and (L / (L+R) > @min_bias or R / (L+R) > @min_bias)') 225 | .idxmax(axis=1) 226 | .replace('M', np.nan) 227 | ) 228 | neurons['postSide'] = ( 229 | body_sidecounts['post'] 230 | .query('(L + M + R) >= @min_post and (L / (L+R) > @min_bias or R / (L+R) > @min_bias)') 231 | .idxmax(axis=1) 232 | .replace('M', np.nan) 233 | ) 234 | 235 | sides = {} 236 | methods = {} 237 | for _, df in neurons.groupby('group'): 238 | # For this function, 'M' is considered null 239 | if df['somaSide'].isin(['L', 'R']).all(): 240 | sides |= dict(df['somaSide'].items()) 241 | methods |= {body: 'somaSide' for body in df.index} 242 | continue 243 | 244 | if df['instanceSide'].notnull().all(): 245 | sides |= dict(df['instanceSide'].items()) 246 | methods |= {body: 'instanceSide' for body in df.index} 247 | continue 248 | 249 | # - Either pre or post must be complete (no NaN). 250 | if not (df['preSide'].notnull().all() or df['postSide'].notnull().all()): 251 | continue 252 | 253 | # - If pre and post are both known, then we infer the neuron to be projecting 254 | # ipsilaterally or contralaterally, and all neurons in the group who CAN be 255 | # assigned a projection direction must agree on the direction of the projection. 256 | # But if some cannot be assigned a projection direction and some can, we don't balk. 257 | has_pre_and_post = df['preSide'].notnull() & df['postSide'].notnull() 258 | if has_pre_and_post.any(): 259 | is_ipsi = df.loc[has_pre_and_post, 'preSide'] == df.loc[has_pre_and_post, 'postSide'] 260 | if is_ipsi.any() and not is_ipsi.all(): 261 | continue 262 | 263 | # - Prefer the postSide (if available) as the final answer, 264 | # since somaSide is usually the postSide (in flies, anyway). 265 | if df['postSide'].notnull().all(): 266 | sides |= dict(df['postSide'].items()) 267 | methods |= {body: 'postSide' for body in df.index} 268 | else: 269 | assert df['preSide'].notnull().all() 270 | sides |= dict(df['preSide'].items()) 271 | methods |= {body: 'preSide' for body in df.index} 272 | 273 | neurons['consensusSide'] = pd.Series(sides) 274 | neurons['consensusSide'] = neurons['consensusSide'].replace(np.nan, None) 275 | 276 | neurons['consensusSideMethod'] = pd.Series(methods) 277 | neurons['consensusSideMethod'] = neurons['consensusSideMethod'].replace(np.nan, None) 278 | 279 | def allnotnull(s): 280 | return s.notnull().all() 281 | 282 | def allnull(s): 283 | return s.isnull().all() 284 | 285 | # Sanity check: 286 | # Every group should EITHER have no consensusSide at all, 287 | # or should have a consensusSide for every neuron in the group. 288 | # No groups should have any bodies with a consensusSide 289 | # unless all bodies in the group have a consensusSide. 290 | aa = neurons.groupby('group')['consensusSide'].agg([allnotnull, allnull]) 291 | assert (aa['allnull'] ^ aa['allnotnull']).all() 292 | 293 | return neurons[['instanceSide', 'preSide', 'postSide', 'consensusSide', 'consensusSideMethod']] 294 | -------------------------------------------------------------------------------- /pixi.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | authors = ["FlyEM"] 3 | channels = ["conda-forge"] 4 | description = "Python client utilties for interacting with the neuPrint connectome analysis service" 5 | name = "neuprint-python" 6 | platforms = ["osx-64", "linux-64", "win-64", "osx-arm64"] 7 | version = "0.5" 8 | 9 | [environments] 10 | test = ["test"] 11 | docs = ["docs"] 12 | dev = ["docs", "test", "dev"] 13 | publish = ["publish", "test"] 14 | 15 | [feature.test.tasks] 16 | test = "pytest" 17 | 18 | [feature.docs.tasks] 19 | make-docs = {cwd = "docs", cmd = "export PYTHONPATH=$PIXI_PROJECT_ROOT && make html"} 20 | 21 | [feature.publish.tasks] 22 | upload-to-pypi = "upload-to-pypi.sh" 23 | 24 | [dependencies] # short for [feature.default.dependencies] 25 | requests = ">=2.22" 26 | pandas = ">=2.2.3,<3" 27 | tqdm = ">=4.67.1,<5" 28 | ujson = ">=5.10.0,<6" 29 | asciitree = ">=0.3.3,<0.4" 30 | scipy = ">=1.14.1,<2" 31 | networkx = ">=3.4.2,<4" 32 | packaging = ">=23.0" 33 | 34 | [feature.test.dependencies] 35 | pytest = "*" 36 | pyarrow = "*" 37 | 38 | [feature.publish.dependencies] 39 | conda-build = "*" 40 | anaconda-client = "*" 41 | twine = "*" 42 | setuptools = "*" 43 | 44 | [feature.docs.dependencies] 45 | nbsphinx = "*" 46 | numpydoc = "*" 47 | sphinx_bootstrap_theme = "*" 48 | sphinx = "*" 49 | sphinx_rtd_theme = "*" 50 | ipython = "*" 51 | jupyter = "*" 52 | ipywidgets = "*" 53 | bokeh = "*" 54 | holoviews = "*" 55 | hvplot = "*" 56 | selenium = "*" 57 | phantomjs = "*" 58 | 59 | [feature.dev.dependencies] 60 | line_profiler = "*" 61 | pyarrow = "*" 62 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 160 3 | ignore = E122,E123,E126,E127,E128,E231,E201,E202,E226,E222,E266,E731,E722,W503,W504 4 | exclude = build,neuprint/_version.py,tests,conda.recipe,.git,versioneer.py,benchmarks,.asv 5 | 6 | [pylint] 7 | disable = logging-fstring-interpolation 8 | 9 | [tool:pytest] 10 | norecursedirs= .* *.egg* build dist conda.recipe 11 | addopts = 12 | --ignore setup.py 13 | --ignore run_test.py 14 | --tb native 15 | --strict 16 | --durations=20 17 | env = 18 | PYTHONHASHSEED=0 19 | markers = 20 | serial: execute test serially (to avoid race conditions) 21 | 22 | [versioneer] 23 | VCS = git 24 | versionfile_source = neuprint/_version.py 25 | versionfile_build = neuprint/_version.py 26 | tag_prefix = 27 | parentdir_prefix = neuprint-python- 28 | 29 | [bdist_wheel] 30 | universal=1 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | import versioneer 4 | 5 | with open('dependencies.txt') as f: 6 | requirements = f.read().splitlines() 7 | requirements = [l for l in requirements if not l.strip().startswith('#')] 8 | 9 | with open('README.md', encoding='utf-8') as f: 10 | long_description = f.read() 11 | 12 | setup( 13 | name='neuprint-python', 14 | version=versioneer.get_version(), 15 | cmdclass=versioneer.get_cmdclass(), 16 | description="Python client utilties for interacting with the neuPrint connectome analysis service", 17 | long_description=long_description, 18 | long_description_content_type='text/markdown', 19 | author="Stuart Berg", 20 | author_email='bergs@hhmi.janelia.org', 21 | url='https://github.com/connectome-neuprint/neuprint-python', 22 | packages=find_packages(), 23 | entry_points={}, 24 | install_requires=requirements, 25 | keywords='neuprint-python', 26 | python_requires='>=3.9', 27 | classifiers=[ 28 | 'Programming Language :: Python :: 3.9', 29 | 'Programming Language :: Python :: 3.10', 30 | 'Programming Language :: Python :: 3.11', 31 | 'Programming Language :: Python :: 3.12', 32 | ] 33 | ) 34 | -------------------------------------------------------------------------------- /update-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # update-deps.sh 3 | 4 | set -e 5 | 6 | # Create an environment with the binder dependencies 7 | TUTORIAL_DEPS="ipywidgets bokeh holoviews hvplot" 8 | SIMULATION_DEPS="ngspice umap-learn scikit-learn matplotlib" 9 | BINDER_DEPS="neuprint-python jupyterlab ${TUTORIAL_DEPS} ${SIMULATION_DEPS}" 10 | conda create -y -n neuprint-python -c flyem-forge -c conda-forge ${BINDER_DEPS} 11 | 12 | # Export to environment.yml, but relax the neuprint-python version requirement 13 | conda env export -n neuprint-python > environment.yml 14 | sed --in-place 's/neuprint-python=.*/neuprint-python/g' environment.yml 15 | 16 | git commit -m "Updated environment.yml for binder" environment.yml 17 | git push origin master 18 | -------------------------------------------------------------------------------- /upload-to-pypi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | ## 6 | ## Usage: Run this from within the root of the repo. 7 | ## 8 | 9 | if [[ "$(git describe)" == *-* ]]; then 10 | echo "Error:" 1>&2 11 | echo " Can't package a non-tagged commit." 1>&2 12 | echo " Your current git commit isn't tagged with a proper version." 1>&2 13 | echo " Try 'git tag -a' first" 1>&2 14 | exit 1 15 | fi 16 | 17 | # 18 | # Unlike conda packages, PyPI packages can never be deleted, 19 | # which means you can't move a tag if you notice a problem 20 | # just 5 minutes after you posted the build. 21 | # 22 | # Therefore, make sure the tests pass before you proceed! 23 | # 24 | PYTHONPATH=. pytest neuprint/tests 25 | 26 | rm -rf dist build 27 | python setup.py sdist bdist_wheel 28 | 29 | # The test PyPI server 30 | #python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* 31 | 32 | # The real PyPI server 33 | # This command will use the token from ~/.pypirc 34 | python3 -m twine upload --repository neuprint-python dist/* 35 | --------------------------------------------------------------------------------