├── labbookdb ├── __init__.py ├── db │ ├── __init__.py │ ├── utils.py │ ├── base_classes.py │ ├── add.py │ ├── query.py │ └── common_classes.py ├── report │ ├── __init__.py │ ├── text_templates │ │ ├── standard_footer.tex │ │ ├── standard_header.tex │ │ └── DNAExtractionProtocol.tex │ ├── protocolize.py │ ├── processing.py │ ├── development.py │ ├── formatting.py │ ├── behaviour.py │ ├── examples.py │ ├── utilities.py │ ├── tracking.py │ └── selection.py ├── tests │ ├── __init__.py │ ├── test_basic.py │ ├── test_selection.py │ ├── test_query.py │ └── test_report.py ├── evaluate │ ├── __init__.py │ └── manual.py ├── introspection │ ├── __init__.py │ └── schema.py ├── examples │ ├── animal_weights_per_treatment.py │ └── drinking_by_cage_treatment.py ├── cli.py └── decorators.py ├── test_scripts.sh ├── requirements.txt ├── docs ├── modules.rst ├── index.rst ├── setup.rst ├── objects.rst └── conf.py ├── example_data └── bids_eventsfile.csv ├── .travis.yml ├── .gitignore ├── setup.py ├── LICENSE └── README.rst /labbookdb/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /labbookdb/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /labbookdb/report/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /labbookdb/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /labbookdb/evaluate/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /labbookdb/introspection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /labbookdb/report/text_templates/standard_footer.tex: -------------------------------------------------------------------------------- 1 | \end{document} 2 | -------------------------------------------------------------------------------- /test_scripts.sh: -------------------------------------------------------------------------------- 1 | cd ../demolog/from_python_code && ./test_deploy.sh 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | argh>=0.26.2 2 | numpy 3 | pandas 4 | simplejson>=3.8.0 5 | sqlalchemy>=1.0.17 6 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | labbookdb 2 | ========= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | labbookdb 8 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | .. toctree:: 4 | :hidden: 5 | :maxdepth: 3 6 | 7 | modules 8 | objects 9 | -------------------------------------------------------------------------------- /docs/setup.rst: -------------------------------------------------------------------------------- 1 | setup module 2 | ============ 3 | 4 | .. automodule:: setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/objects.rst: -------------------------------------------------------------------------------- 1 | Objects Reference 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | labbookdb.db.common_classes 7 | labbookdb.db.base_classes 8 | -------------------------------------------------------------------------------- /labbookdb/tests/test_basic.py: -------------------------------------------------------------------------------- 1 | from labbookdb.db import add 2 | 3 | def test_load(): 4 | session, engine = add.load_session("/tmp/somepath.db") 5 | session.close() 6 | engine.dispose() 7 | -------------------------------------------------------------------------------- /labbookdb/report/text_templates/standard_header.tex: -------------------------------------------------------------------------------- 1 | \documentclass{article} 2 | \usepackage[a4paper, total={170mm,260mm},tmargin=10mm]{geometry} 3 | \usepackage{siunitx} 4 | 5 | \setlength{\parindent}{0cm} % suppress paragraph indenting 6 | 7 | \DeclareSIUnit\rpm{rpm} 8 | \DeclareSIUnit\Molar{\textsc{m}} 9 | 10 | \begin{document} 11 | -------------------------------------------------------------------------------- /example_data/bids_eventsfile.csv: -------------------------------------------------------------------------------- 1 | ,onset,duration,pulse_width,frequency,trial_type,wavelength,strength,strength_unit 2 | 0,192.0,20.0,0.005,20.0,,490.0,30.0,mW 3 | 1,342.0,20.0,0.005,20.0,,490.0,30.0,mW 4 | 2,492.0,20.0,0.005,20.0,,490.0,30.0,mW 5 | 3,642.0,20.0,0.005,20.0,,490.0,30.0,mW 6 | 4,792.0,20.0,0.005,20.0,,490.0,30.0,mW 7 | 5,942.0,20.0,0.005,20.0,,490.0,30.0,mW 8 | 6,1092.0,20.0,0.005,20.0,,490.0,30.0,mW 9 | 7,1242.0,20.0,0.005,20.0,,490.0,30.0,mW 10 | -------------------------------------------------------------------------------- /labbookdb/db/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | def arange_by_date(attribute): 4 | strs = [attribute[i].__str__() for i in range(len(attribute))] 5 | try: 6 | dates = [dt_format(attribute[i].date) for i in range(len(attribute))] 7 | except: 8 | dates = [dt_format(attribute[i].start_date) for i in range(len(attribute))] 9 | strs = [m for _,m in sorted(zip(dates,strs))] 10 | return strs 11 | 12 | def dt_format(dt): 13 | if not dt: 14 | return "NO DATE" 15 | elif dt.time()==datetime.time(0,0,0): 16 | return str(dt.date()) 17 | else: 18 | return str(dt) 19 | 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.6" 5 | #clone demolog for example DB generation code 6 | before_install: 7 | - cd .. 8 | - git clone https://bitbucket.org/TheChymera/demolog 9 | - git clone https://github.com/TheChymera/behaviopy.git 10 | - (cd behaviopy && python setup.py install && travis_wait 20 pip install -r requirements.txt) 11 | - cd LabbookDB 12 | install: 13 | - travis_wait 20 pip install -r requirements.txt 14 | - pip install . 15 | script: 16 | - export MPLBACKEND="agg" 17 | - ./test_scripts.sh 18 | - py.test 19 | - (cd labbookdb/examples && for f in *.py ; do python "$f" ; done) 20 | -------------------------------------------------------------------------------- /labbookdb/tests/test_selection.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from os import path 3 | 4 | DB_PATH = '~/.demolog/meta.db' 5 | DATA_DIR = path.join(path.dirname(path.realpath(__file__)),'../../example_data/') 6 | 7 | def test_parameterized_animals_base(): 8 | import numpy as np 9 | import pandas as pd 10 | from labbookdb.report.selection import animal_id, parameterized 11 | from datetime import datetime 12 | 13 | db_path=DB_PATH 14 | animal = animal_id(db_path,'ETH/AIC','5684') 15 | info_df = parameterized(db_path, 'animals base', animal_filter=[animal]) 16 | 17 | birth = datetime(2016, 7, 21) 18 | 19 | assert info_df['Animal_sex'].item() == 'm' 20 | assert info_df['Animal_birth_date'][0] == birth 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Data file: 2 | src/dbdata/main.entries.py 3 | main.entries.py 4 | 5 | # Compiled source: 6 | *.com 7 | *.class 8 | *.dll 9 | *.exe 10 | *.o 11 | *.so 12 | 13 | # Logs and databases: 14 | *.log 15 | *.sql 16 | *.sqlite 17 | 18 | # OS generated files: 19 | .DS_Store 20 | .DS_Store? 21 | ._* 22 | .Trashes 23 | ehthumbs.db 24 | Thumbs.db 25 | *~ 26 | 27 | # Python: 28 | *.pyc 29 | *.pyo 30 | 31 | # Data traces: 32 | output/* 33 | input/* 34 | *.png 35 | *.backup/ 36 | 37 | #geany 38 | geany_run_script.sh 39 | 40 | #Folders with non-free data: 41 | img*/ 42 | 43 | #Archive folders: 44 | */*/.old 45 | */.old 46 | .old 47 | 48 | #config files: 49 | *.cfg 50 | 51 | #git 52 | *.py.* 53 | 54 | #specific configs 55 | profiles/* 56 | 57 | #databases 58 | .db 59 | -------------------------------------------------------------------------------- /labbookdb/tests/test_query.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from labbookdb.db import add 4 | 5 | def test_related_entry_separators(): 6 | session, engine = add.load_session("/tmp/somepath.db") 7 | with pytest.raises(BaseException) as excinfo: 8 | add.get_related_ids(session, engine, "Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC/cdb&#&identifier.275511") 9 | assert excinfo.value.args[0] == 'No entry was found with a value of "275511" on the "identifier" column of the "AnimalExternalIdentifier" CATEGORY, in the database.' 10 | with pytest.raises(BaseException) as excinfo: 11 | add.get_related_ids(session, engine, "Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC/cdb&&identifier.275511") 12 | assert excinfo.value.args[0] == 'No entry was found with a value of "ETH/AIC/cdb" on the "database" column of the "AnimalExternalIdentifier" CATEGORY, in the database.' 13 | -------------------------------------------------------------------------------- /labbookdb/examples/animal_weights_per_treatment.py: -------------------------------------------------------------------------------- 1 | from labbookdb.report.tracking import animal_weights, qualitative_dates 2 | from behaviopy.plotting import qualitative_times 3 | 4 | fuzzy_matching = { 5 | "ofM":[-14,-15,-13,-7,-8,-6], 6 | "ofMaF":[0,-1], 7 | "ofMcF1":[14,13,15], 8 | "ofMcF2":[28,27,29], 9 | "ofMpF":[45,44,46], 10 | } 11 | 12 | df = animal_weights('~/.demolog/meta.db', {'cage':['cFluDW','cFluDW_']}) 13 | df['relative_date'] = df['relative_date'].dt.days.astype(int) 14 | df = df[['Animal_id', 'relative_date', 'weight', 'Cage_TreatmentProtocol_code', 'ETH/AIC']] 15 | df = qualitative_dates(df, fuzzy_matching=fuzzy_matching) 16 | qualitative_times(df, 17 | x='qualitative_date', 18 | condition='Cage_TreatmentProtocol_code', 19 | err_style="boot_traces", 20 | order=['ofM','ofMaF','ofMcF1','ofMcF2','ofMpF'], 21 | save_as='animal_weights_per_treatment.png', 22 | ) 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | if __name__ == '__main__': 4 | setup( 5 | name="LabbookDB", 6 | version="0.0.1", 7 | description = "A Wet-Work-Tracking Database Application Framework", 8 | author = "Horea Christian", 9 | author_email = "horea.christ@yandex.com", 10 | url = "https://github.com/TheChymera/LabbookDB", 11 | keywords = [ 12 | "laboratory notebook", 13 | "labbook", 14 | "wet work", 15 | "record keeping", 16 | "reports", 17 | "life science", 18 | "biology", 19 | "neuroscience", 20 | "behaviour", 21 | "relational database", 22 | "SQL", 23 | ], 24 | classifiers = [], 25 | install_requires = [], 26 | provides = ["labbookdb"], 27 | packages = [ 28 | "labbookdb", 29 | "labbookdb.db", 30 | "labbookdb.evaluate", 31 | "labbookdb.introspection", 32 | "labbookdb.report", 33 | ], 34 | entry_points = {'console_scripts' : \ 35 | ['LDB = labbookdb.cli:main'] 36 | } 37 | ) 38 | -------------------------------------------------------------------------------- /labbookdb/cli.py: -------------------------------------------------------------------------------- 1 | __author__ = "Horea Christian" 2 | 3 | import argh 4 | 5 | #the order here is very important, if these modules are called in the wrong order, the declarative base will end up being regenerated and will complain about duplicates along the lines of: 6 | #sqlalchemy.exc.InvalidRequestError: Table 'genotype_associations' is already defined for this MetaData instance. Specify 'extend_existing=True' to redefine options and columns on an existing Table object. 7 | #OR 8 | #ImportError: Gtk3 backend requires pygobject to be installed. 9 | from labbookdb.report.tracking import further_cages, animals_id, animals_info 10 | from .db.add import add_generic, append_parameter 11 | from .db.query import animal_info, cage_info 12 | 13 | def main(): 14 | argh.dispatch_commands([ 15 | add_generic, 16 | animal_info, 17 | animals_id, 18 | animals_info, 19 | append_parameter, 20 | cage_info, 21 | further_cages, 22 | ]) 23 | 24 | if __name__ == '__main__': 25 | main() 26 | -------------------------------------------------------------------------------- /labbookdb/examples/drinking_by_cage_treatment.py: -------------------------------------------------------------------------------- 1 | from labbookdb.report.tracking import treatment_group, append_external_identifiers, qualitative_dates, cage_consumption 2 | from labbookdb.report.selection import cage_periods, cage_drinking_measurements 3 | from behaviopy.plotting import qualitative_times 4 | 5 | db_path = '~/.demolog/meta.db' 6 | df = cage_drinking_measurements(db_path, ['cFluDW']) 7 | df = cage_consumption(db_path, df) 8 | 9 | fuzzy_matching = { 10 | "ofM":[-14,-15,-13,-7,-8,-6], 11 | "ofMaF":[0,-1], 12 | "ofMcF1":[14,13,15], 13 | "ofMcF2":[28,27,29], 14 | "ofMpF":[45,44,46,43,47], 15 | } 16 | df = qualitative_dates(df, 17 | iterator_column='Cage_id', 18 | date_column='relative_end_date', 19 | label='qualitative_date', 20 | fuzzy_matching=fuzzy_matching, 21 | ) 22 | qualitative_times(df, 23 | x='qualitative_date', 24 | y='day_animal_consumption', 25 | unit='Cage_id', 26 | order=['ofM','ofMaF','ofMcF1','ofMcF2','ofMpF'], 27 | condition='TreatmentProtocol_code', 28 | err_style="boot_traces", 29 | save_as='drinking_by_cage_treatment.png' 30 | ) 31 | -------------------------------------------------------------------------------- /labbookdb/decorators.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argh import arg 3 | 4 | def environment_db_path(): 5 | """Add a default value to the `db_path` positional argument, based on the `LDB_PATH` environment variable, and fail elegantly if not.""" 6 | try: 7 | return arg("db_path", 8 | default=os.environ["LDB_PATH"], 9 | nargs="?", 10 | help="The path of the LabbookDB database file to query. " 11 | "We detect that your `LDB_PATH` environment variable is set to `{}` in this prompt. " 12 | "This is automatically used by LabbookDB if no custom value is specified.".format(os.environ["LDB_PATH"]), 13 | ) 14 | except KeyError: 15 | return arg("db_path", 16 | help="The path of the LabbookDB database file to query. " 17 | "We detect that your `LDB_PATH` environment variable IS NOT set in this prompt. " 18 | "This environment variable can be automatically used by LabbookDB if no custom value is specified.", 19 | ) 20 | 21 | -------------------------------------------------------------------------------- /labbookdb/report/text_templates/DNAExtractionProtocol.tex: -------------------------------------------------------------------------------- 1 | \title{$name} 2 | \author{$authors_full_name} 3 | \maketitle 4 | 5 | Perform the following steps for each sample: 6 | \begin{itemize} 7 | \item Add \SI{$digestion_buffer_volume}{$volume_unit_siunitx} $digestion_buffer_name ($digestion_buffer_code) and \SI{$proteinase_volume}{$volume_unit_siunitx} $proteinase_name. 8 | \item \MakeUppercase $digestion_movement at \SI{$digestion_revolutions_per_minute}{\rpm} and \SI{$digestion_temperature}{$digestion_temperature_unit_siunitx} for \SI{$digestion_duration}{$digestion_duration_unit_siunitx}. 9 | \item Add \SI{$lysis_buffer_volume}{$volume_unit_siunitx} $lysis_buffer_name. 10 | \item \MakeUppercase $lysis_movement at \SI{$lysis_revolutions_per_minute}{\rpm} and \SI{$lysis_temperature}{$lysis_temperature_unit_siunitx} for \SI{$lysis_duration}{$lysis_duration_unit_siunitx}. 11 | \item Incubate at \SI{$inactivation_temperature}{$inactivation_temperature_unit_siunitx} for \SI{$inactivation_duration}{$inactivation_duration_unit_siunitx}. 12 | \item Incubate at \SI{$cooling_temperature}{$cooling_temperature_unit_siunitx} for \SI{$cooling_duration}{$cooling_duration_unit_siunitx}. 13 | \item \MakeUppercase $centrifugation_movement at \SI{$centrifugation_revolutions_per_minute}{\rpm} for \SI{$centrifugation_duration}{$centrifugation_duration_unit_siunitx}. 14 | \end{itemize} 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Horea Christian 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /labbookdb/report/protocolize.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import subprocess 4 | import tempfile 5 | import pandas as pd 6 | from string import Template 7 | from sqlalchemy import create_engine, literal, inspection 8 | from sqlalchemy.orm import aliased, relation, scoped_session, sessionmaker, eagerload, subqueryload, joinedload, lazyload, Load 9 | from ..db.common_classes import * 10 | from ..db.query import get_for_protocolize 11 | 12 | 13 | from sqlalchemy.ext.declarative import declarative_base 14 | Base = declarative_base() 15 | 16 | def compose_tex(db_path, class_name, code): 17 | """Create a TeX document containing the protocol of the class_name entry identified by a given code. 18 | """ 19 | 20 | #!!! for a system-wide install the location should likely be redefined! 21 | thisscriptspath = os.path.dirname(os.path.realpath(__file__)) 22 | templates_path = os.path.join(thisscriptspath,"text_templates") 23 | 24 | mydf = get_for_protocolize(db_path, class_name, code) 25 | 26 | template_keys = [i for i in mydf.columns.tolist()] 27 | template_values = mydf.ix[0].tolist() 28 | for ix, template_value in enumerate(template_values): 29 | try: 30 | if template_value.is_integer(): 31 | template_values[ix] = int(template_value) 32 | except AttributeError: 33 | pass 34 | 35 | templating_dictionary = dict(zip(template_keys, template_values)) 36 | 37 | with open(os.path.join(templates_path,'standard_header.tex'), 'r') as myfile: 38 | standard_header=myfile.read() 39 | with open(os.path.join(templates_path,'standard_footer.tex'), 'r') as myfile: 40 | standard_footer=myfile.read() 41 | with open(os.path.join(templates_path,class_name+'.tex'), 'r') as myfile: 42 | template_file=myfile.read() 43 | tex_template=Template(template_file) 44 | tex_template = tex_template.substitute(templating_dictionary) 45 | 46 | tex_document = standard_header 47 | tex_document += tex_template 48 | tex_document += standard_footer 49 | 50 | return tex_document 51 | 52 | def print_document(tex, pdfname="protocol"): 53 | current = os.getcwd() 54 | temp = tempfile.mkdtemp() 55 | os.chdir(temp) 56 | 57 | f = open('protocol.tex','w') 58 | f.write(tex) 59 | f.close() 60 | 61 | proc=subprocess.Popen(['pdflatex','protocol.tex']) 62 | subprocess.Popen(['pdflatex',tex]) 63 | proc.communicate() 64 | 65 | os.rename('protocol.pdf',pdfname) 66 | shutil.copy(pdfname,current) 67 | shutil.rmtree(temp) 68 | -------------------------------------------------------------------------------- /labbookdb/introspection/schema.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import pydotplus 3 | import sadisplay 4 | from os import path 5 | from labbookdb.db.query import ALLOWED_CLASSES 6 | from labbookdb.db.common_classes import * 7 | 8 | def generate( 9 | extent="all", 10 | save_dotfile="", 11 | save_plot="", 12 | label="", 13 | linker_tables=False 14 | ): 15 | """Retreive the LabbookDB schema and save either a DOT file, or a PNG or PDF plot. 16 | """ 17 | 18 | if extent == "all": 19 | nodes = ALLOWED_CLASSES.values() 20 | elif type(extent) is list: 21 | nodes = [ALLOWED_CLASSES[key] for key in extent] 22 | 23 | if linker_tables: 24 | nodes.extend(linker_tables) 25 | 26 | desc = sadisplay.describe(nodes) 27 | 28 | if save_dotfile: 29 | save_dotfile = path.abspath(path.expanduser(save_dotfile)) 30 | with codecs.open(save_dotfile, 'w', encoding='utf-8') as f: 31 | f.write(sadisplay.dot(desc)) 32 | 33 | if save_plot: 34 | save_plot = path.abspath(path.expanduser(save_plot)) 35 | dot = sadisplay.dot(desc) 36 | dot = dot.replace('label = "generated by sadisplay v0.4.8"', 'label = "{}"'.format(label)) 37 | graph = pydotplus.graph_from_dot_data(dot) 38 | filename, extension = path.splitext(save_plot) 39 | if extension in [".png",".PNG"]: 40 | graph.write_png(save_plot) 41 | elif extension in [".pdf",".PDF"]: 42 | graph.write_pdf(save_plot) 43 | 44 | if __name__ == '__main__': 45 | # generate(extent="all", save_plot="~/full_schema.png") 46 | # generate(extent=["Animal","CageStay","Cage","OpticFiberImplantProtocol","Operation","OrthogonalStereotacticTarget","Protocol"], save_plot="~/cagestay_schema.pdf", linker_tables=[operation_association]) 47 | # generate(extent=["Animal","CageStay","Cage",], save_plot="~/cagestay_schema.pdf", linker_tables=[cage_stay_association]) 48 | # generate( 49 | # extent=[ 50 | # "Animal", 51 | # "ForcedSwimTestMeasurement", 52 | # "Evaluation", 53 | # "CageStay", 54 | # "Cage", 55 | # "Measurement", 56 | # "Treatment", 57 | # "Protocol", 58 | # ], 59 | # save_plot="~/fst_schema.pdf", 60 | # linker_tables=[ 61 | # cage_stay_association, 62 | # treatment_cage_association, 63 | # ], 64 | # ) 65 | # generate(extent=["Animal","CageStay","Cage","SucrosePreferenceMeasurement"], save_plot="~/measurements_schema.pdf") 66 | generate(extent=["Animal","Operation","Protocol","Operator"], save_plot="~/basic_schema.pdf", linker_tables=[authors_association,operation_association]) 67 | # generate(extent=["Animal","FMRIMeasurement","OpenFieldTestMeasurement","WeightMeasurement"], save_plot="~/measurements_schema.pdf") 68 | -------------------------------------------------------------------------------- /labbookdb/evaluate/manual.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | import sys, os 5 | from behaviopy import tracking 6 | 7 | from labbookdb.db.query import get_df 8 | 9 | def behaviour(db_path, test_type, animal_ids=[], animals_id_column="id_eth", dates=[], evaluations_dir="~/data/behaviour", author="", volume=0, random_order=True): 10 | """Wrapper for evaluate() passing data from a LabbookDB model database. 11 | """ 12 | 13 | if not author: 14 | print("It is advisable to add your name's three-letter abbreviation via the \"author\" argument. This helps identify your work and protects it from being overwritten.") 15 | 16 | db_path = os.path.expanduser(db_path) 17 | evaluations_dir = os.path.expanduser(evaluations_dir) 18 | 19 | if test_type == "forced_swim_test": 20 | measurements_table = "ForcedSwimTestMeasurement" 21 | if os.path.basename(os.path.normpath(evaluations_dir)) != "forced_swim_test": 22 | evaluations_dir = os.path.join(evaluations_dir,"forced_swim_test") 23 | if not os.path.exists(evaluations_dir): 24 | os.makedirs(evaluations_dir) 25 | trial_duration = 360 #in seconds 26 | events = {"s":"swimming","i":"immobility"} 27 | else: 28 | raise ValueError("The function does not support test_type=\'"+test_type+"\'.") 29 | 30 | 31 | col_entries=[ 32 | ("Animal","id"), 33 | (measurements_table,), 34 | ] 35 | join_entries=[ 36 | ("Animal.measurements",), 37 | (measurements_table,), 38 | ] 39 | 40 | filters = [] 41 | if animal_ids: 42 | filter_entry = ["Animal",animals_id_column] 43 | for animal_id in animal_ids: 44 | filter_entry.extend([animal_id]) 45 | filters.append(filter_entry) 46 | if dates: 47 | filter_entry = ["measurements_table","date"] 48 | for date in dates: 49 | filter_entry.extend([date]) 50 | filters.append(filter_entry) 51 | 52 | reference_df = get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=filters) 53 | if random_order: 54 | reference_df = reference_df.iloc[np.random.permutation(len(reference_df))] 55 | 56 | for _, measurement_df in reference_df.iterrows(): 57 | recording_path = measurement_df.loc[measurements_table+"_data_path"] 58 | if measurements_table+"_recording_bracket" in reference_df.columns: 59 | bracket = measurement_df.loc[measurements_table+"_recording_bracket"] 60 | else: 61 | bracket = "" 62 | outfile_name = os.path.splitext(os.path.basename(recording_path))[0]+"_"+bracket+"_"+author 63 | output_path = os.path.join(evaluations_dir,outfile_name) 64 | tracking.manual_events(recording_path,trial_duration,events=events,bracket=bracket, volume=volume, output_path=output_path) 65 | 66 | if __name__ == '__main__': 67 | behaviour("~/syncdata/meta.db","forced_swim_test",animal_ids=[],author="chr",volume=0.1) 68 | -------------------------------------------------------------------------------- /labbookdb/report/processing.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | def timedelta_sums(evaluation_path, index_name="", period_start=False, period_end=False): 4 | """Return the per-behaviour sums of timedelta intervals. 5 | 6 | Parameters 7 | ---------- 8 | 9 | timedelta_df : pandas_dataframe 10 | A pandas dataframe containing a "behaviour" and a "timedelta" column 11 | index_name : string, optional 12 | The name to add as an index of the retunred series (useful for concatenating multiple outputs) 13 | period_start : float, optional 14 | The timepoint at which the evaluation period for the timedelta sums starts. 15 | period_end : float, optional 16 | The timepoint at which the evaluation period for the timedelta sums ends. 17 | """ 18 | 19 | timedelta_df = timedeltas(evaluation_path, period_start=period_start, period_end=period_end) 20 | sums = {} 21 | for behaviour in list(set(timedelta_df["behaviour"])): 22 | sums[behaviour] = timedelta_df.loc[timedelta_df["behaviour"] == behaviour, "timedelta"].sum() 23 | sum_df = pd.DataFrame(sums, index=[index_name]) 24 | return sum_df 25 | 26 | def timedeltas(evaluation_path, period_start=False, period_end=False): 27 | """Return per-behaviour timedelta intervals. 28 | 29 | Parameters 30 | ---------- 31 | 32 | timedelta_df : pandas_dataframe 33 | A pandas dataframe containing a "behaviour" and a "start" column 34 | period_start : float, optional 35 | The timepoint at which the evaluation period for the timedelta starts. 36 | period_end : float, optional 37 | The timepoint at which the evaluation period for the timedelta ends. 38 | """ 39 | 40 | df = pd.read_csv(evaluation_path) 41 | 42 | #edit df to fit restricted summary period 43 | if period_start: 44 | cropped_df = df[df["start"] > period_start] 45 | # we perform this check so that the period is not extended beyond the data range 46 | if not len(cropped_df) == len(df): 47 | startpoint_behaviour = list(df[df["start"] <= period_start].tail(1)["behaviour"])[0] 48 | startpoint = pd.DataFrame({"start":period_start,"behaviour":startpoint_behaviour}, index=[""]) 49 | df = pd.concat([startpoint,cropped_df]) 50 | if period_end: 51 | cropped_df = df[df["start"] < period_end] 52 | # we perform this check so that the period is not extended beyond the data range 53 | if not len(cropped_df) == len(df): 54 | endpoint = pd.DataFrame({"start":period_end,"behaviour":"ANALYSIS_ENDPOINT"}, index=[""]) 55 | df = pd.concat([cropped_df,endpoint]) 56 | 57 | # timedelta calculation 58 | df["timedelta"] = (df["start"].shift(-1)-df["start"]).fillna(0) 59 | 60 | #the last index gives the experiment end time, not meaningful behaviour. We remove that here. 61 | df = df[:-1] 62 | 63 | return df 64 | 65 | def rounded_days(datetime_obj): 66 | days = datetime_obj.days 67 | # rounding number of days up, as timedelta.days returns the floor only: 68 | if datetime_obj.seconds / 43199.5 >= 1: 69 | days += 1 70 | return days 71 | -------------------------------------------------------------------------------- /labbookdb/report/development.py: -------------------------------------------------------------------------------- 1 | 2 | def animal_multiselect(db_path, 3 | cage_treatments=[], 4 | implant_targets=['dr_impl'], 5 | virus_targets=['dr_skull','dr_dura','dr_dura_shallow','dr_skull_perpendicular'], 6 | genotypes=['eptg'], 7 | external_id='', 8 | ): 9 | from labbookdb.report.selection import animals_by_genotype, animal_id, animal_treatments, animal_operations 10 | 11 | if implant_targets: 12 | df_implants = animal_operations(db_path, implant_targets=implant_targets)['Animal_id'].tolist() 13 | if virus_targets: 14 | df_virus = animal_operations(db_path, virus_targets=virus_targets)['Animal_id'].tolist() 15 | if genotypes: 16 | df_genotype = animals_by_genotype(db_path, genotypes)['Animal_id'].tolist() 17 | if cage_treatments: 18 | df_treatments = animal_treatments(db_path, cage_treatments=cage_treatments)['Animal_id'].tolist() 19 | 20 | if external_id: 21 | if implant_targets: 22 | df_implants = [animal_id(db_path, external_id, i, reverse=True) for i in df_implants] 23 | if virus_targets: 24 | df_virus = [animal_id(db_path, external_id, i, reverse=True) for i in df_virus] 25 | if genotypes: 26 | df_genotype = [animal_id(db_path, external_id, i, reverse=True) for i in df_genotype] 27 | if cage_treatments: 28 | df_treatments = [animal_id(db_path, external_id, i, reverse=True) for i in df_treatments] 29 | 30 | d = [] 31 | if implant_targets: 32 | d.append(df_implants) 33 | if virus_targets: 34 | d.append(df_virus) 35 | if genotypes: 36 | d.append(df_genotype) 37 | if cage_treatments: 38 | d.append(df_treatments) 39 | selection = list(set(d[0]).intersection(*d)) 40 | return selection 41 | 42 | def animal_weights_(): 43 | import matplotlib.pyplot as mpl 44 | from labbookdb.report.tracking import animal_weights, qualitative_dates 45 | from behaviopy.plotting import qualitative_times 46 | 47 | fuzzy_matching = { 48 | "ofM":[-14,-15,-13,-7,-8,-6], 49 | "ofMaF":[0,-1], 50 | "ofMcF1":[14,13,15], 51 | "ofMcF2":[28,27,29], 52 | "ofMpF":[45,44,46], 53 | } 54 | 55 | df = animal_weights('~/syncdata/meta.db', {'animal':['aFluIV','aFluIV_']}) 56 | df['relative_date'] = df['relative_date'].dt.days.astype(int) 57 | df = df[['Animal_id', 'relative_date', 'weight', 'TreatmentProtocol_code', 'ETH/AIC']] 58 | df = qualitative_dates(df, fuzzy_matching=fuzzy_matching) 59 | qualitative_times(df, order=['ofM','ofMaF','ofMcF1','ofMcF2','ofMpF'], condition='TreatmentProtocol_code', err_style="boot_traces", time='qualitative_date') 60 | df = df[df['qualitative_date']=='ofMpF'] 61 | print(df) 62 | mpl.show() 63 | 64 | def drinking_water_by_cage_treatment( 65 | treatment_relative_date=True, 66 | rounding='D', 67 | ): 68 | from labbookdb.report.tracking import treatment_group, append_external_identifiers, qualitative_dates, cage_consumption 69 | from labbookdb.report.selection import cage_periods, cage_drinking_measurements 70 | from behaviopy.plotting import weights 71 | import matplotlib.pyplot as plt 72 | 73 | 74 | DB_PATH = '~/syncdata/meta.db' 75 | df = cage_drinking_measurements(DB_PATH,['cFluDW','cFluDW_']) 76 | df = cage_consumption(DB_PATH,df) 77 | 78 | fuzzy_matching = { 79 | "ofM":[-14,-15,-13,-7,-8,-6], 80 | "ofMaF":[0,-1], 81 | "ofMcF1":[14,13,15], 82 | "ofMcF2":[28,27,29], 83 | "ofMpF":[45,44,46,43,47], 84 | } 85 | df = qualitative_dates(df, 86 | iterator_column='Cage_id', 87 | date_column='relative_end_date', 88 | label='qualitative_date', 89 | fuzzy_matching=fuzzy_matching, 90 | ) 91 | weights(df, 92 | weight='day_animal_consumption', 93 | unit='Cage_id', 94 | order=['ofM','ofMaF','ofMcF1','ofMcF2','ofMpF'], 95 | condition='TreatmentProtocol_code', 96 | err_style="boot_traces", 97 | time='qualitative_date', 98 | ) 99 | plt.show() 100 | -------------------------------------------------------------------------------- /labbookdb/tests/test_report.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from os import path 3 | 4 | DB_PATH = '~/.demolog/meta.db' 5 | DATA_DIR = path.join(path.dirname(path.realpath(__file__)),'../../example_data/') 6 | 7 | def test_implant_angle_filter(): 8 | from labbookdb.report.selection import animal_id, animal_treatments, animal_operations 9 | import numpy as np 10 | 11 | db_path=DB_PATH 12 | df = animal_operations(db_path) 13 | #validate target by code 14 | df = df[~df['OrthogonalStereotacticTarget_code'].isnull()] 15 | df = df[df['OrthogonalStereotacticTarget_code'].str.contains('dr')] 16 | #check pitch 17 | df = df[~df['OrthogonalStereotacticTarget_pitch'].isin([0,np.NaN])] 18 | animals = df['Animal_id'].tolist() 19 | animals_eth = [animal_id(db_path,'ETH/AIC',i,reverse=True) for i in animals] 20 | 21 | assert animals_eth == ['5684'] 22 | 23 | def test_animal_cage_treatment_control_in_report(): 24 | """Check if animal which died before the cagetreatment was applied to its last home cage is indeed not showing a cage treatment, but still showing the animal treatment.""" 25 | from labbookdb.report.tracking import animals_info 26 | 27 | df = animals_info(DB_PATH, 28 | save_as=None, 29 | functional_scan_responders=True, 30 | treatments=True, 31 | ) 32 | assert df[df['ETH/AIC']=='6255']['cage_treatment'].values[0] == "" 33 | assert df[df['ETH/AIC']=='6255']['animal_treatment'].values[0] == 'aFluIV_' 34 | 35 | def test_animal_id(): 36 | """Check if LabbookDB animal ID is correctly reported based on external database identifier.""" 37 | from labbookdb.report.selection import animal_id 38 | 39 | my_id = animal_id(DB_PATH, 40 | database='ETH/AIC', 41 | identifier='6255' 42 | ) 43 | assert my_id == 41 44 | 45 | def test_bids_eventsfile(): 46 | """Check if correct BIDS events file can be sourced.""" 47 | from labbookdb.report.tracking import bids_eventsfile 48 | import pandas as pd 49 | 50 | df = bids_eventsfile(DB_PATH,'chr_longSOA') 51 | bids_eventsfile = path.join(DATA_DIR,'bids_eventsfile.csv') 52 | df_ = pd.read_csv(bids_eventsfile, index_col=0) 53 | 54 | assert df[['onset','duration']].equals(df_[['onset','duration']]) 55 | 56 | def test_drinking_by_cage_treatment( 57 | treatment_relative_date=True, 58 | rounding='D', 59 | ): 60 | from labbookdb.report.tracking import treatment_group, append_external_identifiers, qualitative_dates, cage_consumption 61 | from labbookdb.report.selection import cage_periods, cage_drinking_measurements 62 | 63 | known_cage_ids = [25, 38, 41] 64 | known_consumption_values = [2.35, 2.51, 2.94, 2.95, 3.16, 3.17, 3.22, 3.23, 3.24, 3.25, 3.49, 3.63, 3.72, 4.04, 4.09, 4.58, 4.98, 5.15, 5.31, 5.39, 5.54, 5.97, 6.73, 6.78] 65 | 66 | df = cage_drinking_measurements(DB_PATH,['cFluDW']) 67 | df = cage_consumption(DB_PATH,df) 68 | 69 | fuzzy_matching = { 70 | "ofM":[-14,-15,-13,-7,-8,-6], 71 | "ofMaF":[0,-1], 72 | "ofMcF1":[14,13,15], 73 | "ofMcF2":[28,27,29], 74 | "ofMpF":[45,44,46,43,47], 75 | } 76 | df = qualitative_dates(df, 77 | iterator_column='Cage_id', 78 | date_column='relative_end_date', 79 | label='qualitative_date', 80 | fuzzy_matching=fuzzy_matching, 81 | ) 82 | 83 | cage_ids = sorted(df['Cage_id'].unique()) 84 | assert cage_ids == known_cage_ids 85 | 86 | consumption_values = df['day_animal_consumption'].values 87 | consumption_values = [round(i, 2) for i in consumption_values] 88 | consumption_values = sorted(list(set(consumption_values))) 89 | assert consumption_values == known_consumption_values 90 | 91 | def test_groups(): 92 | """Create a `pandas.DataFrame` containing treatment and genotype group assignments""" 93 | from labbookdb.report.tracking import treatment_group, append_external_identifiers 94 | 95 | known_sorted_ids = [ 96 | '5667', 97 | '5668', 98 | '5673', 99 | '5674', 100 | '5675', 101 | '5689', 102 | '5690', 103 | '5691', 104 | '5692', 105 | '5694', 106 | '5699', 107 | '5700', 108 | '5704', 109 | '5705', 110 | '5706', 111 | '6254', 112 | '6256', 113 | '6262', 114 | ] 115 | 116 | df = treatment_group(DB_PATH, ['cFluDW','cFluDW_'], level='cage') 117 | df = append_external_identifiers(DB_PATH, df, ['Genotype_code']) 118 | sorted_ids = sorted(df['ETH/AIC'].tolist()) 119 | 120 | assert sorted_ids == known_sorted_ids 121 | -------------------------------------------------------------------------------- /labbookdb/report/formatting.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from . import processing 3 | 4 | def plottable_sums(reference_df, behaviour, identifier_column="Animal_id", periods={}, period_label="period", metadata_columns={"TreatmentProtocol_code":"Treatment"}): 5 | identifiers = list(set(reference_df[identifier_column])) 6 | evaluation_df = pd.DataFrame({}) 7 | for identifier in identifiers: 8 | identifier_df = reference_df[reference_df[identifier_column]==identifier] 9 | evaluation_path = identifier_df["Evaluation_path"].values[0] 10 | identifier_data = {} 11 | for metadata_column in metadata_columns: 12 | identifier_data[metadata_columns[metadata_column]] = identifier_df[metadata_column].values[0] 13 | for period in periods: 14 | period_start, period_end = periods[period] 15 | sums = processing.timedelta_sums(evaluation_path, index_name=identifier, period_start=period_start, period_end=period_end) 16 | #We need to calculate this explicitly since the start/end of th experiment may not align perfecty with the theoretical period 17 | real_period_duration = sums.sum(axis=1).values[0] 18 | #if the behaviour key is not found, there was none of that behaviour type in the period 19 | try: 20 | behaviour_ratio = sums[behaviour].values[0]/real_period_duration 21 | except KeyError: 22 | behaviour_ratio = 0 23 | identifier_data[behaviour.title()+" Ratio"] = behaviour_ratio 24 | identifier_data[period_label] = period 25 | identifier_data["Identifier"] = identifier 26 | period_df_slice = pd.DataFrame(identifier_data, index=[identifier]) 27 | evaluation_df = pd.concat([evaluation_df, period_df_slice]) 28 | 29 | #data is usually ordered as it comes, for nicer plots we sort it here 30 | evaluation_df = evaluation_df.sort_values([period_label], ascending=True) 31 | evaluation_df = evaluation_df.sort_values(list(metadata_columns.values()), ascending=False) 32 | return evaluation_df 33 | 34 | def plottable_sucrosepreference_df(reference_df): 35 | cage_ids = list(set(reference_df["Cage_id"])) 36 | preferences_df = pd.DataFrame({}) 37 | for cage_id in cage_ids: 38 | cage_id_df = reference_df[reference_df["Cage_id"]==cage_id] 39 | reference_dates = list(set(cage_id_df["SucrosePreferenceMeasurement_reference_date"])) 40 | reference_dates.sort() 41 | measurement_dates = list(set(cage_id_df["SucrosePreferenceMeasurement_date"])) 42 | measurement_dates.sort() 43 | first_date = reference_dates[0] 44 | preferences={} 45 | for measurement_date in measurement_dates: 46 | cage_id_measurement_df = cage_id_df[cage_id_df["SucrosePreferenceMeasurement_date"] == measurement_date] 47 | start_date = cage_id_measurement_df["SucrosePreferenceMeasurement_reference_date"].tolist()[0] 48 | relative_start_day = start_date-first_date 49 | rounded_relative_start_day = processing.rounded_days(relative_start_day) 50 | relative_end_day = measurement_date-first_date 51 | rounded_relative_end_day = processing.rounded_days(relative_end_day) 52 | key = "{} to {}".format(rounded_relative_start_day, rounded_relative_end_day) 53 | water_start = cage_id_measurement_df["SucrosePreferenceMeasurement_water_start_amount"].tolist()[0] 54 | water_end = cage_id_measurement_df["SucrosePreferenceMeasurement_water_end_amount"].tolist()[0] 55 | sucrose_start = cage_id_measurement_df["SucrosePreferenceMeasurement_sucrose_start_amount"].tolist()[0] 56 | sucrose_end = cage_id_measurement_df["SucrosePreferenceMeasurement_sucrose_end_amount"].tolist()[0] 57 | water_consumption = water_end - water_start 58 | sucrose_consumption = sucrose_end - sucrose_start 59 | sucrose_prefernce = sucrose_consumption/(water_consumption + sucrose_consumption) 60 | preferences["Period [days]"] = key 61 | preferences["Sucrose Preference Ratio"] = sucrose_prefernce 62 | preferences["Sucrose Bottle Position"] = cage_id_measurement_df["SucrosePreferenceMeasurement_sucrose_bottle_position"].tolist()[0] 63 | preferences["Sucrose Concentration"] = cage_id_measurement_df["SucrosePreferenceMeasurement_sucrose_concentration"].tolist()[0] 64 | preferences["Treatment"] = cage_id_measurement_df["TreatmentProtocol_code"].tolist()[0] 65 | preferences["Cage ID"] = cage_id # this may not actually be needed, as the same info is contained in the index 66 | preferences_df_slice = pd.DataFrame(preferences, index=[cage_id]) 67 | preferences_df = pd.concat([preferences_df, preferences_df_slice]) 68 | 69 | return preferences_df 70 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | LabbookDB 2 | ========= 3 | 4 | .. image:: https://readthedocs.org/projects/labbookdb/badge/?version=latest 5 | :target: http://labbookdb.readthedocs.io/en/latest/?badge=latest 6 | :alt: Documentation Status 7 | .. image:: https://travis-ci.org/TheChymera/LabbookDB.svg?branch=master 8 | :target: https://travis-ci.org/TheChymera/LabbookDB 9 | 10 | This package contains a relational database structure for life science research, and a number of functions to conveniently add and retrieve information - and generate summaries. 11 | The core concept of LabbookDB is that most of the information classically tracked in a lab book can be more efficiently and more reliably stored in a relational database. 12 | 13 | In comparison to a paper notebook, an **electronic** lab book is: 14 | 15 | * More easily stored 16 | * More easily shared 17 | * More easily backed up 18 | 19 | In comparison with other electronic formats based on a document concept, a **database** of experimental metadata is: 20 | 21 | * More easily browsed 22 | * More easily queried 23 | * More easily integrated into data analysis functions 24 | 25 | 26 | Presentations 27 | ------------- 28 | 29 | Video 30 | ~~~~~ 31 | 32 | * `LabbookDB - A Relational Framework for Laboratory Metadata `_, at SciPy 2017 in Austin (TX,USA). 33 | 34 | Publications 35 | ~~~~~~~~~~~~ 36 | 37 | * `LabbookDB - A Wet-Work-Tracking Database Application Framework `_, in Proceedings of the 15th Python in Science Conference (SciPy 2017). 38 | 39 | Installation 40 | ------------ 41 | 42 | Gentoo Linux 43 | ~~~~~~~~~~~~ 44 | 45 | LabbookDB is available for Portage (the package manager of Gentoo Linux, derivative distributions, as well as BSD) via the `Gentoo Science Overlay `_. 46 | Upon enabling the overlay, the package can be emerged: 47 | 48 | .. code-block:: console 49 | 50 | emerge labbookdb 51 | 52 | 53 | Python Package Manager (Users) 54 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 55 | 56 | Python's `setuptools` allows you to install Python packages independently of your distribution (or operating system, even). 57 | This approach cannot manage any of our numerous non-Python dependencies (by design) and at the moment will not even manage Python dependencies; 58 | as such, given any other alternative, **we do not recommend this approach**: 59 | 60 | .. code-block:: console 61 | 62 | git clone git@github.com:TheChymera/LabbookDB.git 63 | cd LabbookDB 64 | python setup.py install --user 65 | 66 | Python Package Manager (Developers) 67 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 68 | 69 | Python's `setuptools` allows you to install Python packages independently of your distribution (or operating system, even); 70 | it also allows you to install a "live" version of the package - dynamically linking back to the source code. 71 | This permits you to test code (with real module functionality) as you develop it. 72 | This method is sub-par for dependency management (see above notice), but - as a developer - you should be able to manually ensure that your package manager provides the needed packages. 73 | 74 | .. code-block:: console 75 | 76 | git clone git@github.com:TheChymera/LabbookDB.git 77 | cd LabbookDB 78 | echo "export PATH=\$HOME/.local/bin/:\$PATH" >> ~/.bashrc 79 | source ~/.bashrc 80 | python setup.py develop --user 81 | 82 | Example Input 83 | ------------- 84 | 85 | LabbookDB is designed to organize complex wet work data. 86 | We publish example input to generate a relationship-rich database in a separate repository, `demolog `_. 87 | 88 | Dependencies 89 | ------------ 90 | 91 | * `Argh`_ 92 | * `Pandas`_ 93 | * `simplejson`_ 94 | * `SQLAlchemy`_ 95 | 96 | Optional Dependencies for Introspection 97 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 98 | 99 | * `Sadisplay`_ 100 | 101 | Optional Dependencies for PDF Protocol Generation 102 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 103 | 104 | * `TeX Live`_ 105 | 106 | Optional Dependencies for Plotting 107 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 108 | 109 | * `BehavioPy`_ 110 | * `Matplotlib`_ 111 | 112 | 113 | .. _Argh: https://github.com/neithere/argh/ 114 | .. _BehavioPy: https://github.com/TheChymera/behaviopy 115 | .. _Matplotlib: https://matplotlib.org/ 116 | .. _Pandas: http://pandas.pydata.org/ 117 | .. _Sadisplay: https://bitbucket.org/estin/sadisplay/wiki/Home 118 | .. _simplejson: https://github.com/simplejson/simplejson 119 | .. _SQLAlchemy: http://www.sqlalchemy.org/ 120 | .. _TeX Live: https://www.tug.org/texlive/ 121 | -------------------------------------------------------------------------------- /labbookdb/report/behaviour.py: -------------------------------------------------------------------------------- 1 | try: 2 | from . import selection, formatting 3 | except (SystemError, ValueError): 4 | from labbookdb.report import selection, formatting 5 | from os import path 6 | from behaviopy import plotting 7 | 8 | def sucrose_preference(db_path, treatment_start_dates, 9 | bp_style=True, 10 | colorset=None, 11 | comparisons={"Period [days]":[]}, 12 | compare="Treatment", 13 | rename_treatments={"cFluDW":"Fluoxetine","cFluDW_":"Control"}, 14 | save_df="", 15 | ): 16 | """Plot sucrose preference scatterplot. 17 | 18 | Parameters 19 | ---------- 20 | 21 | db_path : string 22 | Path to a LabbookDB formatted database. 23 | 24 | treatment_start_dates : list 25 | A list containing the treatment start date or dates by which to filter the cages for the sucrose preference measurements. 26 | Items should be strings in datetime format, e.g. "2016,4,25,19,30". 27 | 28 | bp_style : bool, optional 29 | Whether to let behaviopy apply its default style. 30 | 31 | compare : string, optional 32 | Which parameter to categorize the comparison by. Must be a column name from df. 33 | 34 | comparisons : dict, optional 35 | A dictionary, the key of which indicates which df column to generate comparison insances from. If only a subset of the available rows are to be included in the comparison, the dictionary needs to specify a value, consisting of a list of acceptable values on the column given by the key. 36 | 37 | datacolumn_label : string, optional 38 | A column name from df, the values in which column give the data to plot. 39 | 40 | rename_treatments : dict, optional 41 | Dictionary specifying a rename scheme for the treatments. Keys are names to change and values are what to change the names to. 42 | 43 | save_df : string, optional 44 | Path under which to save the plotted dataframe. ".csv" will be appended to the string, and the data will be saved in CSV format. 45 | """ 46 | 47 | animals_df = selection.animals_by_cage_treatment(db_path, start_dates=treatment_start_dates) 48 | animals = list(set(animals_df["Animal_id"])) 49 | raw_df = selection.by_animals(db_path, "sucrose preference", animals=animals) 50 | full_df = animals_df.merge(raw_df, on="Animal_id", suffixes=("_Treatment","")) 51 | plottable_df = formatting.plottable_sucrosepreference_df(full_df) 52 | plotting.expandable_ttest(plottable_df, 53 | compare=compare, 54 | comparisons=comparisons, 55 | datacolumn_label="Sucrose Preference Ratio", 56 | rename_treatments=rename_treatments, 57 | colorset=colorset, 58 | ) 59 | 60 | if save_df: 61 | df_path = path.abspath(path.expanduser(save_df)) 62 | if not(df_path.endswith(".csv") or df_path.endswith(".CSV")): 63 | df_path += ".csv" 64 | plottable_df.to_csv(df_path) 65 | 66 | return plottable_df 67 | 68 | def forced_swim(db_path, plot_style, treatment_start_dates, 69 | colorset=None, 70 | columns=["2 to 4"], 71 | rename_treatments={"cFluDW":"Fluoxetine","cFluDW_":"Control"}, 72 | time_label="", 73 | plot_behaviour="immobility", 74 | save_df="", 75 | ): 76 | """Plot forced swim scatterplot. 77 | 78 | Parameters 79 | ---------- 80 | 81 | db_path : string 82 | Path to a LabbookDB formatted database. 83 | 84 | treatment_start_dates : list 85 | A list containing the treatment start date or dates by which to filter the cages for the sucrose preference measurements. 86 | Items should be strings in datetime format, e.g. "2016,4,25,19,30". 87 | 88 | rename_treatments : dict 89 | Dictionary specifying a rename scheme for the treatments. Keys are names to change and values are what to change the names to. 90 | 91 | save_df : string, optional 92 | Path under which to save the plotted dataframe. ".csv" will be appended to the string if not yet presenr, and the data will be saved in CSV format. 93 | """ 94 | 95 | raw_df = selection.parameterized(db_path, "forced swim", treatment_start_dates=treatment_start_dates) 96 | 97 | if plot_style in ["tsplot", "pointplot"]: 98 | if not time_label: 99 | time_label = "Interval [1 min]" 100 | if time_label == "Interval [1 min]": 101 | periods={1:[0,60],2:[60,120],3:[120,180],4:[180,240],5:[240,300],6:[300,360]} 102 | plottable_df = formatting.plottable_sums(raw_df, plot_behaviour, identifier_column="Animal_id", periods=periods, period_label=time_label) 103 | elif time_label == "Interval [2 min]": 104 | periods={1:[0,120],2:[120,240],3:[240,360]} 105 | plottable_df = formatting.plottable_sums(raw_df, plot_behaviour, identifier_column="Animal_id", periods=periods, period_label=time_label) 106 | if colorset: 107 | plotting.forced_swim_timecourse(plottable_df, legend_loc="best", rename_treatments=rename_treatments, time_label=time_label, plotstyle=plot_style, datacolumn_label="Immobility Ratio", colorset=colorset) 108 | else: 109 | plotting.forced_swim_timecourse(plottable_df, legend_loc="best", rename_treatments=rename_treatments, time_label=time_label, plotstyle=plot_style, datacolumn_label="Immobility Ratio") 110 | elif plot_style == "ttest": 111 | periods = {} 112 | for column_name in columns: 113 | start_minute, end_minute = column_name.split(" to ") 114 | start = int(start_minute)*60 115 | end = int(end_minute)*60 116 | periods[column_name] = [start,end] 117 | plottable_df = formatting.plottable_sums(raw_df, plot_behaviour, period_label="Interval [minutes]", periods=periods) 118 | if colorset: 119 | plotting.expandable_ttest(plottable_df, compare="Treatment", comparisons={"Interval [minutes]":[]}, datacolumn_label="Immobility Ratio", rename_treatments=rename_treatments, colorset=colorset) 120 | else: 121 | plotting.expandable_ttest(plottable_df, compare="Treatment", comparisons={"Interval [minutes]":[]}, datacolumn_label="Immobility Ratio", rename_treatments=rename_treatments) 122 | 123 | if save_df: 124 | df_path = path.abspath(path.expanduser(save_df)) 125 | if not(df_path.endswith(".csv") or df_path.endswith(".CSV")): 126 | df_path += ".csv" 127 | plottable_df.to_csv(df_path) 128 | -------------------------------------------------------------------------------- /labbookdb/report/examples.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | from behaviopy import plotting 3 | try: 4 | from .behaviour import forced_swim, sucrose_preference 5 | from .protocolize import compose_tex, print_document 6 | from .tracking import treatments_plot 7 | from .selection import animals_by_treatment 8 | except (SystemError, ValueError): 9 | from labbookdb.report.behaviour import forced_swim, sucrose_preference 10 | from labbookdb.report.protocolize import compose_tex, print_document 11 | from labbookdb.report.tracking import treatments_plot 12 | from labbookdb.report.selection import animals_by_treatment 13 | 14 | COHORTS = [ 15 | {"treatment_start":"2015,11,11", "window_end":"2015,12,30"}, 16 | {"treatment_start":"2016,4,25,19,30", "window_end":""}, 17 | {"treatment_start":"2016,5,19,23,5", "window_end":""}, 18 | {"treatment_start":"2016,11,24,21,30", "window_end":""}, 19 | {"treatment_start":"2017,1,31,22,0", "window_end":"2017,3,21"}, 20 | ] 21 | 22 | def my_sucrose_preference(db_path, cohorts, compare): 23 | if cohorts == "cage": 24 | treatment_start_dates = ["2016,4,25,19,30","2016,5,19,23,5",] 25 | elif cohorts == "animal": 26 | treatment_start_dates = ["2016,11,24,21,30"] 27 | elif cohorts == "aileen_switching_sides": 28 | treatment_start_dates = ["2017,1,31,22,0"] 29 | elif cohorts == "all": 30 | treatment_start_dates = [i["treatment_start"] for i in COHORTS] 31 | else: 32 | treatment_start_dates = cohorts 33 | if compare == "treatment": 34 | sucrose_preference(db_path, treatment_start_dates=treatment_start_dates, comparisons={"Period [days]":[]}, compare="Treatment",save_df="") 35 | elif compare == "side_preference": 36 | sucrose_preference(db_path, treatment_start_dates=treatment_start_dates, comparisons={"Cage ID":[]}, compare="Sucrose Bottle Position", save_df="") 37 | 38 | def treatments(db_path, cohorts, 39 | per_cage=True, 40 | ): 41 | 42 | if isinstance(cohorts,str): 43 | cohorts = [cohorts] 44 | shade = ["FMRIMeasurement_date"] 45 | draw = [ 46 | {"TreatmentProtocol_code":["aFluIV_","Treatment_start_date"]}, 47 | {"TreatmentProtocol_code":["aFluIV","Treatment_start_date"]}, 48 | "OpenFieldTestMeasurement_date", 49 | "ForcedSwimTestMeasurement_date", 50 | {"TreatmentProtocol_code":["aFluSC","Treatment_start_date"]}, 51 | ] 52 | saturate = [ 53 | {"Cage_TreatmentProtocol_code":["cFluDW","Cage_Treatment_start_date","Cage_Treatment_end_date"]}, 54 | {"Cage_TreatmentProtocol_code":["cFluDW_","Cage_Treatment_start_date","Cage_Treatment_end_date"]}, 55 | {"TreatmentProtocol_code":["cFluIP","Treatment_start_date","Treatment_end_date"]}, 56 | ] 57 | if per_cage: 58 | filters = [["Cage_Treatment","start_date",i["treatment_start"]] for i in cohorts] 59 | else: 60 | df = animals_by_treatment(db_path, start_dates=[i["treatment_start"] for i in cohorts],) 61 | animals = list(set(df["Animal_id"])) 62 | animals = [str(i) for i in animals] 63 | myfilter = ["Animal","id",] 64 | myfilter.extend(animals) 65 | filters = [myfilter] 66 | window_end = [i["window_end"] for i in cohorts if i["window_end"] not in ("", None)] 67 | window_end.sort() 68 | if window_end: 69 | window_end = window_end[-1] 70 | treatments_plot(db_path, 71 | draw=draw, 72 | filters=filters, 73 | saturate=saturate, 74 | default_join="outer", 75 | #This loads only entries with fMRI measurements: 76 | # join_types=["outer","inner","outer","outer","outer","outer","outer","outer","outer","outer"], 77 | #This loads all entries: 78 | join_types=["outer","outer","outer","outer","outer","outer","outer","outer","outer","outer"], 79 | save_df="~/lala.csv", 80 | save_plot="~/lala.png", 81 | shade=shade, 82 | window_end=window_end, 83 | ) 84 | 85 | def protocol(db_path, 86 | class_name="DNAExtractionProtocol", 87 | code="EPDqEP", 88 | ): 89 | tex = compose_tex(db_path, class_name,code) 90 | print_document(tex, class_name+"_"+code+".pdf") 91 | 92 | if __name__ == '__main__': 93 | db_path="~/syncdata/meta.db" 94 | # treatments(db_path,COHORTS[0:1], per_cage=False) 95 | # treatments(db_path,COHORTS[2:3]) 96 | # forced_swim(db_path, "tsplot", treatment_start_dates=["2016,4,25,19,30","2016,5,19,23,5"]) 97 | 98 | ################################################ 99 | #####Safe for usage with logging_examples####### 100 | ################################################ 101 | treatments(db_path,COHORTS[4:5]) 102 | # forced_swim(db_path, "tsplot", treatment_start_dates=["2017,1,31,22,0","2016,11,24,21,30"]) 103 | # forced_swim(db_path, "pointplot", treatment_start_dates=["2017,1,31,22,0","2016,11,24,21,30"]) 104 | # forced_swim(db_path, "tsplot", treatment_start_dates=["2017,1,31,22,0"]) 105 | # forced_swim(db_path, "pointplot", treatment_start_dates=["2017,1,31,22,0"]) 106 | # forced_swim(db_path, "tsplot", treatment_start_dates=["2016,11,24,21,30"]) 107 | # forced_swim(db_path, "tsplot", treatment_start_dates=[i["treatment_start"] for i in COHORTS], save_df="") 108 | # forced_swim(db_path, "pointplot", treatment_start_dates=[i["treatment_start"] for i in COHORTS], save_df="") 109 | # forced_swim(db_path, "ttest", treatment_start_dates=["2016,11,24,21,30","2017,1,31,22,0"], columns=["2 to 4", "2 to 6"]) 110 | # forced_swim(db_path, "ttest", treatment_start_dates=["2016,4,25,19,30","2016,5,19,23,5","2017,1,31,22,0"], columns=["2 to 4", "2 to 6"]) 111 | # forced_swim(db_path, "ttest", treatment_start_dates=["2016,4,25,19,30","2016,5,19,23,5","2016,11,24,21,30","2017,1,31,22,0"], columns=["2 to 4", "2 to 6"], colorset=["#56B4E9", "#E69F00", "#56B4E9", "#000000","#F0E442", "#0072B2", "#D55E00", "#CC79A7"]) 112 | # forced_swim(db_path, "ttest", treatment_start_dates=["2017,1,31,22,0"], columns=["2 to 4"]) 113 | # forced_swim(db_path, "ttest", treatment_start_dates=["2016,11,24,21,30"], columns=["2 to 4", "2 to 6"]) 114 | # forced_swim(db_path, "ttest", treatment_start_dates=[i["treatment_start"] for i in COHORTS], columns=["2 to 4", "2 to 6"], save_df="") 115 | # my_sucrose_preference(db_path, "aileen_switching_sides", "treatment") 116 | # protocolize_dna_extraction(db_path) 117 | 118 | # plt.savefig("fst.pdf") 119 | plt.show() 120 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # LabbookDB documentation build configuration file, created by 5 | # sphinx-quickstart on Wed May 24 01:26:34 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc', 35 | 'sphinx.ext.doctest', 36 | 'sphinx.ext.intersphinx', 37 | 'sphinx.ext.coverage', 38 | 'sphinx.ext.mathjax', 39 | 'sphinx.ext.napoleon', 40 | 'sphinx.ext.viewcode'] 41 | 42 | # Add any paths that contain templates here, relative to this directory. 43 | templates_path = ['_templates'] 44 | 45 | # The suffix(es) of source filenames. 46 | # You can specify multiple suffix as a list of string: 47 | # 48 | # source_suffix = ['.rst', '.md'] 49 | source_suffix = '.rst' 50 | 51 | # The master toctree document. 52 | master_doc = 'index' 53 | 54 | # General information about the project. 55 | project = 'LabbookDB' 56 | copyright = '2017, Horea Christian' 57 | author = 'Horea Christian' 58 | 59 | # The version info for the project you're documenting, acts as replacement for 60 | # |version| and |release|, also used in various other places throughout the 61 | # built documents. 62 | # 63 | # The short X.Y version. 64 | version = '0.0.1' 65 | # The full version, including alpha/beta/rc tags. 66 | release = '0.0.1' 67 | 68 | # The language for content autogenerated by Sphinx. Refer to documentation 69 | # for a list of supported languages. 70 | # 71 | # This is also used if you do content translation via gettext catalogs. 72 | # Usually you set "language" from the command line for these cases. 73 | language = None 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | # This patterns also effect to html_static_path and html_extra_path 78 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 79 | 80 | # The name of the Pygments (syntax highlighting) style to use. 81 | pygments_style = 'sphinx' 82 | 83 | # If true, `todo` and `todoList` produce output, else they produce nothing. 84 | todo_include_todos = False 85 | 86 | 87 | # -- Options for HTML output ---------------------------------------------- 88 | 89 | # The theme to use for HTML and HTML Help pages. See the documentation for 90 | # a list of builtin themes. 91 | # 92 | html_theme = 'nature' 93 | 94 | # Theme options are theme-specific and customize the look and feel of a theme 95 | # further. For a list of options available for each theme, see the 96 | # documentation. 97 | # 98 | # html_theme_options = {} 99 | 100 | # Add any paths that contain custom static files (such as style sheets) here, 101 | # relative to this directory. They are copied after the builtin static files, 102 | # so a file named "default.css" will overwrite the builtin "default.css". 103 | html_static_path = ['_static'] 104 | 105 | 106 | # -- Options for HTMLHelp output ------------------------------------------ 107 | 108 | # Output file base name for HTML help builder. 109 | htmlhelp_basename = 'LabbookDBdoc' 110 | 111 | 112 | # -- Options for LaTeX output --------------------------------------------- 113 | 114 | latex_elements = { 115 | # The paper size ('letterpaper' or 'a4paper'). 116 | # 117 | # 'papersize': 'letterpaper', 118 | 119 | # The font size ('10pt', '11pt' or '12pt'). 120 | # 121 | # 'pointsize': '10pt', 122 | 123 | # Additional stuff for the LaTeX preamble. 124 | # 125 | # 'preamble': '', 126 | 127 | # Latex figure (float) alignment 128 | # 129 | # 'figure_align': 'htbp', 130 | } 131 | 132 | # Grouping the document tree into LaTeX files. List of tuples 133 | # (source start file, target name, title, 134 | # author, documentclass [howto, manual, or own class]). 135 | latex_documents = [ 136 | (master_doc, 'LabbookDB.tex', 'LabbookDB Documentation', 137 | 'Horea Christian', 'manual'), 138 | ] 139 | 140 | 141 | # -- Options for manual page output --------------------------------------- 142 | 143 | # One entry per manual page. List of tuples 144 | # (source start file, name, description, authors, manual section). 145 | man_pages = [ 146 | (master_doc, 'labbookdb', 'LabbookDB Documentation', [author], 1) 147 | ] 148 | 149 | 150 | # -- Options for Texinfo output ------------------------------------------- 151 | 152 | # Grouping the document tree into Texinfo files. List of tuples 153 | # (source start file, target name, title, author, 154 | # dir menu entry, description, category) 155 | texinfo_documents = [ 156 | (master_doc, 'LabbookDB', 'LabbookDB Documentation', author, 'LabbookDB', 'One line description of project.', 'Miscellaneous'), 157 | ] 158 | 159 | def run_apidoc(_): 160 | from sphinx.apidoc import main 161 | import os 162 | import sys 163 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 164 | cur_dir = os.path.abspath(os.path.dirname(__file__)) 165 | module = os.path.join(cur_dir,"..","labbookdb") 166 | main(['-e', '-o', cur_dir, module, '--force']) 167 | 168 | def setup(app): 169 | app.connect('builder-inited', run_apidoc) 170 | 171 | 172 | # Example configuration for intersphinx: refer to the Python standard library. 173 | intersphinx_mapping = {'https://docs.python.org/': None} 174 | -------------------------------------------------------------------------------- /labbookdb/report/utilities.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | def concurrent_cagetreatment(df, cagestays, 4 | protect_duplicates=[ 5 | 'Animal_id', 6 | 'Cage_id', 7 | 'Cage_Treatment_start_date', 8 | 'Cage_Treatment_end_date', 9 | 'Cage_TreatmentProtocol_code', 10 | 'Treatment_end_date', 11 | 'Treatment_end_date', 12 | 'TreatmentProtocol_code', 13 | ], 14 | ): 15 | """ 16 | Return a `pandas.DataFrame` object containing only `Cage_Treatment*` entries which are concurrent with the animal stay in the cage to which they were administered. 17 | 18 | Parameters 19 | ---------- 20 | 21 | df : pandas.DataFrame 22 | Pandas Dataframe, with columns containing: 23 | `Animal_id`, 24 | `Animal_death_date`, 25 | `CageStay_start_date`, 26 | `Cage_Treatment_start_date`, 27 | `Cage_TreatmentProtocol_code`. 28 | cagestays : pandas.DataFrame 29 | Pandas Dataframe, with columns containing: 30 | `Animal_id`, 31 | `CageStay_end_date`, 32 | `CageStay_start_date`, 33 | 34 | Notes 35 | ----- 36 | 37 | This function checks whether cage-level treatment onsets indeed happened during the period in which the animal was housed in the cage. 38 | We do not check for the treatment end dates, as an animal which has received a partial treatment has received a treatment. 39 | Checks for treatment discontinuation due to e.g. death should be performed elsewhere. 40 | """ 41 | drop_idx = [] 42 | for subject in list(df['Animal_id'].unique()): 43 | stay_starts = df[df['Animal_id']==subject]['CageStay_start_date'].tolist() 44 | # The per-animal treatment info is recorded in each table row, but if the animal only has one cage stay without a cage treatment, it will be deleted, taking the animal treatment information with it. 45 | # We avoid this here: 46 | blank_cells_only = False 47 | if len(stay_starts) == 1: 48 | if df.loc[df['Animal_id']==subject, 'TreatmentProtocol_code'].item() != None: 49 | blank_cells_only = True 50 | for stay_start in stay_starts: 51 | stay_end = cagestays[(cagestays['Animal_id']==subject)&(cagestays['CageStay_start_date']==stay_start)]['CageStay_end_date'].tolist()[0] 52 | treatment_start = df[(df['Animal_id']==subject)&(df['CageStay_start_date']==stay_start)]['Cage_Treatment_start_date'].tolist()[0] 53 | death_date = df[df['Animal_id']==subject]['Animal_death_date'].tolist()[0] 54 | # We do not check for treatment end dates, because often you may want to include recipients of incomplete treatments (e.g. due to death) when filtering based on cagestays. 55 | # Filtering based on death should be done elsewhere. 56 | if treatment_start <= stay_start or treatment_start >= stay_end or treatment_start >= death_date: 57 | if blank_cells_only: 58 | df.loc[df['Animal_id']==subject, ['Cage_TreatmentProtocol_code', 'Cage_Treatment_start_date', 'Cage_Treatment_end_date', 'Cage_Treatment_protocol_id']] = None 59 | else: 60 | drop_idx.extend(df[(df['Animal_id']==subject)&(df['CageStay_start_date']==stay_start)].index.tolist()) 61 | df = df.drop(drop_idx) 62 | #df = df.drop_duplicates(subset=protect_duplicates) 63 | return df 64 | 65 | def make_identifier_short_form(df, 66 | index_name="Animal_id"): 67 | """ 68 | Convert the long form `AnimalExternalIdentifier_identifier` column of a `pandas.DataFrame` to short-form identifier columns named after the corresponding values on the `AnimalExternalIdentifier_database` column. 69 | 70 | Parameters 71 | ---------- 72 | df : pandas.DataFrame 73 | A `pandas.DataFrame` object containing a long-form `AnimalExternalIdentifier_identifier` column and a dedicated `AnimalExternalIdentifier_database` column. 74 | index_name : str, optonal 75 | The name of a column from `df`, the values of which can be rendered unique. This column will serve as the index o the resulting dataframe. 76 | """ 77 | df = df.rename(columns={'AnimalExternalIdentifier_animal_id': 'Animal_id'}) 78 | df = df.set_index([index_name, 'AnimalExternalIdentifier_database'])['AnimalExternalIdentifier_identifier'] 79 | df = df.unstack(1) 80 | return df 81 | 82 | def collapse_rename(df, groupby, collapse, 83 | rename=False, 84 | ): 85 | """ 86 | Collapse long form columns according to a lambda function, so that groupby column values are rendered unique 87 | 88 | Parameters 89 | ---------- 90 | 91 | df : pandas.DataFrame 92 | A `pandas.DataFrame` object which you want to collapse. 93 | groupby : string 94 | The name of a column from `df`, the values of which you want to render unique. 95 | collapse : dict 96 | A dictionary the keys of which are columns you want to collapse, and the values of which are lambda functions instructing how to collapse (e.g. concatenate) the values. 97 | rename : dict, optional 98 | A dictionary the keys of which are names of columns from `df`, and the values of which are new names for these columns. 99 | """ 100 | df = df.groupby(groupby).agg(collapse) 101 | if rename: 102 | df = df.rename(columns=rename) 103 | 104 | return df 105 | 106 | def relativize_dates(df, 107 | date_suffix='_date', 108 | rounding='D', 109 | rounding_type='round', 110 | reference_date=True, 111 | ): 112 | """ 113 | Express dates on each row of a Pandas Dataframe as datetime objects relative to the row value on the 'reference_date' column. 114 | 115 | Parameters 116 | ---------- 117 | 118 | df : pandas.DataFrame 119 | Pandas Dataframe, with columns containing 'reference_date' and strings ending in `date_suffix`. 120 | date_suffix : str, optional 121 | String sufix via which to identify date columns needing manipulation. 122 | rounding : str, optional 123 | Datetime increment for date rounding. 124 | rounding_type : {'round','floor','ceil'}, optional 125 | Whether to round the dates (splits e.g. days apart at noon, hours at 30 minutes, etc.) or to take the floor or the ceiling. 126 | """ 127 | 128 | if isinstance(reference_date, bool) and reference_date: 129 | df['reference_date'] = df['Cage_Treatment_start_date'] 130 | elif isinstance(reference_date, str): 131 | df['reference_date'] = df[reference_date] 132 | date_columns = [i for i in df.columns.tolist() if i.endswith(date_suffix)] 133 | for date_column in date_columns: 134 | try: 135 | df[date_column] = df[date_column]-df['reference_date'] 136 | except TypeError: 137 | pass 138 | else: 139 | if rounding: 140 | start = pd.to_datetime('1970-01-01') 141 | df[date_column] = df[date_column] + start 142 | if rounding_type == 'round': 143 | df[date_column] = df[date_column].dt.round(rounding) 144 | elif rounding_type == 'floor': 145 | df[date_column] = df[date_column].dt.floor(rounding) 146 | elif rounding_type == 'ceil': 147 | df[date_column] = df[date_column].dt.ceil(rounding) 148 | return df 149 | -------------------------------------------------------------------------------- /labbookdb/db/base_classes.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from sqlalchemy import Column, Integer, String, Sequence, Table, ForeignKey, Float, DateTime, Boolean, ForeignKeyConstraint 3 | from sqlalchemy.orm import validates, backref, relationship 4 | from sqlalchemy.ext.declarative import declarative_base 5 | Base = declarative_base() 6 | 7 | from .utils import * 8 | 9 | authors_association = Table('authors_associations', Base.metadata, 10 | Column('protocols_id', Integer, ForeignKey('protocols.id')), 11 | Column('operators_id', Integer, ForeignKey('operators.id')) 12 | ) 13 | measurements_irregularities_association = Table('measurements_irregularities_association', Base.metadata, 14 | Column('measurements_id', Integer, ForeignKey('measurements.id')), 15 | Column('irregularities_id', Integer, ForeignKey('irregularities.id')) 16 | ) 17 | operations_irregularities_association = Table('operations_irregularities_association', Base.metadata, 18 | Column('operations_id', Integer, ForeignKey('operations.id')), 19 | Column('irregularities_id', Integer, ForeignKey('irregularities.id')) 20 | ) 21 | 22 | class Genotype(Base): 23 | __tablename__ = "genotypes" 24 | id = Column(Integer, primary_key=True) 25 | code = Column(String, unique=True) 26 | construct = Column(String) 27 | zygosity = Column(String) 28 | 29 | def __repr__(self): 30 | return ""\ 31 | % (self.code, self.construct, self.zygosity) 32 | 33 | class Irregularity(Base): 34 | __tablename__ = "irregularities" 35 | id = Column(Integer, primary_key=True) 36 | description = Column(String, unique=True) 37 | 38 | class MeasurementUnit(Base): 39 | __tablename__ = "measurement_units" 40 | id = Column(Integer, primary_key=True) 41 | code = Column(String, unique=True) 42 | long_name = Column(String) 43 | siunitx = Column(String) 44 | 45 | class Operator(Base): 46 | __tablename__ = "operators" 47 | id = Column(Integer, primary_key=True) 48 | code = Column(String, unique=True) 49 | full_name = Column(String) 50 | affiliation = Column(String) 51 | email = Column(String) 52 | 53 | @validates('email') 54 | def validate_email(self, key, address): 55 | assert '@' in address 56 | return address 57 | 58 | 59 | #fMRI classes: 60 | 61 | class FMRIScannerSetup(Base): 62 | __tablename__ = "fmri_scanner_setups" 63 | id = Column(Integer, primary_key=True) 64 | code = Column(String, unique=True) 65 | coil = Column(String) 66 | scanner = Column(String) 67 | support = Column(String) 68 | resonator = Column(String) 69 | 70 | class StimulationEvent(Base): 71 | __tablename__ = "stimulation_events" 72 | id = Column(Integer, primary_key=True) 73 | #tme values specified in seconds, frequencies in hertz, wavelength in nm 74 | onset = Column(Float) 75 | duration = Column(Float) 76 | frequency = Column(Float) 77 | pulse_width = Column(Float) 78 | trial_type = Column(String) 79 | target = Column(String) 80 | wavelength = Column(Float) 81 | strength = Column(Float) 82 | unit_id = Column(Integer, ForeignKey('measurement_units.id')) 83 | unit = relationship("MeasurementUnit") 84 | 85 | #meta classes: 86 | 87 | class Biopsy(Base): 88 | __tablename__ = "biopsies" 89 | id = Column(Integer, primary_key=True) 90 | start_date = Column(DateTime) 91 | animal_id = Column(Integer, ForeignKey('animals.id')) 92 | extraction_protocol_id = Column(Integer, ForeignKey('protocols.id')) 93 | extraction_protocol = relationship("Protocol", foreign_keys=[extraction_protocol_id]) 94 | sample_location = Column(String) #id of the physical sample 95 | fluorescent_microscopy = relationship("FluorescentMicroscopyMeasurement", backref="biopsy") 96 | type = Column(String(50)) 97 | __mapper_args__ = { 98 | 'polymorphic_identity': 'biopsy', 99 | 'polymorphic_on': type 100 | } 101 | 102 | class Protocol(Base): 103 | __tablename__ = "protocols" 104 | id = Column(Integer, primary_key=True) 105 | code = Column(String, unique=True) 106 | name = Column(String, unique=True) 107 | authors = relationship("Operator", secondary=authors_association) 108 | 109 | type = Column(String(50)) 110 | __mapper_args__ = { 111 | 'polymorphic_identity': 'protocol', 112 | 'polymorphic_on': type 113 | } 114 | 115 | def __str__(self): 116 | return "Protocol(code: {code})"\ 117 | .format(code=self.code) 118 | 119 | class Virus(Base): 120 | __tablename__ = "viruses" 121 | id = Column(Integer, primary_key=True) 122 | code = Column(String, unique=True) 123 | addgene_identifier = Column(String) 124 | capsid = Column(String) 125 | concentration = Column(Float) # in vg/ml 126 | plasmid_summary = Column(String) 127 | credit = Column(String) 128 | source = Column(String) 129 | 130 | def __str__(self): 131 | if self.addgene_id: 132 | return "Virus(code: {code}, Addgene: #{addgene_id})"\ 133 | .format(code=self.code, addgene_id=self.addgene_id) 134 | else: 135 | return "Virus(code: {code})"\ 136 | .format(code=self.code) 137 | 138 | class OpticFiberImplant(Base): 139 | __tablename__ = "implants" 140 | id = Column(Integer, primary_key=True) 141 | code = Column(String, unique=True) 142 | long_code = Column(String) #not necessarily uniqu, as we track different transmittances per model 143 | angle = Column(Float) # in degrees, relative to an unangled implant axis 144 | ferrule_diameter = Column(Float) # in mm 145 | cannula_diameter = Column(Float) # in mm 146 | length = Column(Float) # in mm 147 | manufacturer_code = Column(String) 148 | manufacturer = Column(String) 149 | numerical_apperture = Column(Float) 150 | transmittance = Column(Float) # in percent 151 | 152 | def __str__(self): 153 | return "OpticFiberImplant(code: {code}, long_code: {long_code})"\ 154 | .format(code=self.code, long_code=self.long_code) 155 | 156 | class OrthogonalStereotacticTarget(Base): 157 | __tablename__ = "orthogonal_stereotactic_targets" 158 | id = Column(Integer, primary_key=True) 159 | code = Column(String, unique=True) 160 | reference = Column(String) 161 | # coordinates in millimetres 162 | posteroanterior = Column(Float) 163 | leftright = Column(Float) 164 | superoinferior = Column(Float) 165 | # angles in degrees and relative to implant 166 | pitch = Column(Float) 167 | yaw = Column(Float) 168 | roll = Column(Float) 169 | # depth in millimetres 170 | depth = Column(Float) 171 | qualitative_depth_reference = Column(String, default="skull") # set to "dura" if the insertable is lowered to the dura before coordinate setting 172 | 173 | def __str__(self): 174 | return "OrthogonalStereotacticTarget({reference}: {pa}(PA), {lr}(LR), {si}(SI))"\ 175 | .format(reference=self.reference, pa=self.posteroanterior, lr=self.leftright, si=self.superoinferior) 176 | 177 | class Measurement(Base): 178 | __tablename__ = "measurements" 179 | id = Column(Integer, primary_key=True) 180 | date = Column(DateTime) 181 | 182 | animal_id = Column(Integer, ForeignKey('animals.id')) # only set in per-animal measurements 183 | cage_id = Column(Integer, ForeignKey('cages.id')) # only set in per-cage measurements 184 | 185 | irregularities = relationship("Irregularity", secondary=measurements_irregularities_association) 186 | operator_id = Column(Integer, ForeignKey('operators.id')) 187 | operator = relationship("Operator") 188 | 189 | type = Column(String(50)) 190 | __mapper_args__ = { 191 | 'polymorphic_identity': 'measurement', 192 | 'polymorphic_on': type 193 | } 194 | 195 | def __str__(self): 196 | return "{type}(date: {date})"\ 197 | .format(date=dt_format(self.date), type=self.type) 198 | -------------------------------------------------------------------------------- /labbookdb/db/add.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from __future__ import print_function 3 | 4 | import argh 5 | import json 6 | import numpy 7 | 8 | import pandas as pd 9 | 10 | from sqlalchemy import create_engine, literal, update, insert 11 | from sqlalchemy import inspect 12 | from os import path 13 | from sqlalchemy.orm import sessionmaker 14 | import sqlalchemy 15 | 16 | from .common_classes import * 17 | from .query import ALLOWED_CLASSES 18 | 19 | def load_session(db_path): 20 | """Load and return a new SQLalchemy session and engine. 21 | 22 | Parameters 23 | ---------- 24 | db_path : str 25 | Path to desired database location, can be relative or use tilde to specify the user $HOME. 26 | 27 | Returns 28 | ------- 29 | session : sqlalchemy.orm.session.Session 30 | Session instance. 31 | engine : sqlalchemy.engine.Engine 32 | Engine instance. 33 | """ 34 | 35 | db_path = "sqlite:///" + path.abspath(path.expanduser(db_path)) 36 | engine = create_engine(db_path, echo=False) 37 | #it is very important that `autoflush == False`, otherwise if "treatments" or "measurements" entried precede "external_ids" the latter will insert a null on the animal_id column 38 | Session = sessionmaker(bind=engine, autoflush=False) 39 | session = Session() 40 | Base.metadata.create_all(engine) 41 | return session, engine 42 | 43 | def add_to_db(session, engine, myobject): 44 | """Add an object to session and return the .id attribute value. 45 | 46 | Parameters 47 | ---------- 48 | session : sqlalchemy.orm.session.Session 49 | Session instance, as created with labbookdb.db.add.load_session(). 50 | engine : sqlalchemy.engine.Engine 51 | Engine instance correponding to the Session instance under session, as created with labbookdb.db.add.load_session(). 52 | myobject : object 53 | LabbookDB object with SQLAlchemy-compatible attributes (e.g. as found under labbookdb.db.common_classes). 54 | 55 | Returns 56 | ------- 57 | object_id : int 58 | Value of myobject.id attribute 59 | """ 60 | 61 | session.add(myobject) 62 | try: 63 | session.commit() 64 | except sqlalchemy.exc.IntegrityError: 65 | print("Please make sure this was not a double entry:", myobject) 66 | object_id=myobject.id 67 | return object_id 68 | 69 | def instructions(kind): 70 | """Print helpful instructions for common error-causing input issues 71 | 72 | Parameters 73 | ---------- 74 | kind : {"table_identifier",} 75 | Shorthand for the instructin message to be printed. 76 | """ 77 | if kind == "table_identifier": 78 | print("Make sure you have entered the filter value correctly. This value is supposed to refer to the id column of another table and needs to be specified as \'table_identifier\'.\'field_by_which_to_filter\'.\'target_value\'") 79 | 80 | def get_related_ids(session, engine, parameters): 81 | """Return the .id attribute value for existing entries matched by a string following the LabbookDB-syntax. 82 | 83 | Parameters 84 | ---------- 85 | session : sqlalchemy.orm.session.Session 86 | Session instance, as created with labbookdb.db.add.load_session(). 87 | engine : sqlalchemy.engine.Engine 88 | Engine instance correponding to the Session instance under session, as created with labbookdb.db.add.load_session(). 89 | parameters : str 90 | LabbookDB-syntax string specifying an existing entry. 91 | 92 | Returns 93 | ------- 94 | ids : list of int 95 | .id attribute values for the entries matched by the LabbookDB-syntax string. 96 | sql_query : sqlalchemy.orm.query.Query 97 | Query corresponding to the LabbookDB-syntax string 98 | 99 | Examples 100 | -------- 101 | >>> from labbookdb.db import add 102 | >>> session, engine = add.load_session("lala.db") 103 | >>> add.get_related_ids(s,e,"Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC/cdb&#&identifier.275511") 104 | BaseException: No entry was found with a value of "275511" on the "identifier" column of the "AnimalExternalIdentifier" CATEGORY, in the database. 105 | >>> add.get_related_ids(s,e,"Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC/cdb&&identifier.275511") 106 | BaseException: No entry was found with a value of "ETH/AIC/cdb" on the "database" column of the "AnimalExternalIdentifier" CATEGORY, in the database. 107 | 108 | Notes 109 | ----- 110 | Recursivity : 111 | This function calls itself recursively in order to get the .id attribute values of related entries (and related entries of related entries, etc.) specified in the LabbookDB-syntax string. 112 | Multiple constraints are separated by double ampersands which may be separated by none or up to two hashtags, to specify at which level the constrains tshould be applied to. 113 | One hashtag is removed on each recursion step, and the constraint is only evaluated when there are no hashtags left. 114 | "Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC/cdb&#&identifier.275511" will look for both the database and the identifier attributes in the AnimalExternalIdentifier class, while "Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC/cdb&#&identifier.275511" will look for the database attribute on the AnimalExternalIdentifier class, and for the identifier attribute on the Animal class. 115 | """ 116 | 117 | category = parameters.split(":",1)[0] 118 | sql_query=session.query(ALLOWED_CLASSES[category]) 119 | for field_value in parameters.split(":",1)[1].split("&&"): 120 | field, value = field_value.split(".",1) 121 | if "&#&" in value or "&##&" in value: 122 | value=value.replace("&#&", "&&") 123 | value=value.replace("&##&", "&#&") 124 | if ":" in value: 125 | values, objects = get_related_ids(session, engine, value) 126 | for value in values: 127 | value=int(value) # the value is returned as a numpy object 128 | if field[-4:] == "date": # support for date entry matching (the values have to be passes as string but matched as datetime) 129 | value = datetime.datetime(*[int(i) for i in value.split(",")]) 130 | # we are generally looking to match values, but sometimes the parent table does not have an id column, but only a relationship column (e.g. in one to many relationships) 131 | try: 132 | sql_query = sql_query.filter(getattr(ALLOWED_CLASSES[category], field)==value) 133 | except sqlalchemy.exc.InvalidRequestError: 134 | sql_query = sql_query.filter(getattr(ALLOWED_CLASSES[category], field).contains(*[i for i in objects])) 135 | else: 136 | if field[-4:] == "date": # support for date entry matching (the values have to be passes as string but matched as datetime) 137 | value = datetime.datetime(*[int(i) for i in value.split(",")]) 138 | sql_query = sql_query.filter(getattr(ALLOWED_CLASSES[category], field)==value) 139 | mystring = sql_query.with_labels().statement 140 | mydf = pd.read_sql_query(mystring,engine) 141 | category_tablename = ALLOWED_CLASSES[category].__table__.name 142 | related_table_ids = mydf[category_tablename+"_id"] 143 | ids = list(related_table_ids) 144 | ids = [int(i) for i in ids] 145 | if ids == []: 146 | raise BaseException("No entry was found with a value of \""+str(value)+"\" on the \""+field+"\" column of the \""+category+"\" CATEGORY, in the database.") 147 | session.close() 148 | engine.dispose() 149 | return ids, sql_query 150 | 151 | def append_parameter(db_path, entry_identification, parameters): 152 | """Assigns a value to a given parameter of a given entry. 153 | 154 | Parameters 155 | ---------- 156 | 157 | db_path : str 158 | A string especifying the database path 159 | 160 | entry_identification : str 161 | A LabbookDB syntax string specifying an instance of an object for which to update a parameter. 162 | Example strings: "Animal:external_ids.AnimalExternalIdentifier:database.ETH/AIC&#&identifier.5701" , "Cage:id.14" 163 | 164 | parameters : str or dict 165 | A LabbookDB-style dictionary (or JSON interpretable as dictionary), where keys are strings giving the names of attributes of the class selected by entry_identification, and values are either the values to assign (verbatim: string, int, or float) or LabbookDB-syntax strings specifying a related entry, or a list of LabbookDB-syntax strings specifying related entries, or a list of LabbookDB-style dictionaries specifying new entries to be created and linked. 166 | """ 167 | 168 | if isinstance(parameters, str): 169 | parameters = json.loads(parameters) 170 | 171 | session, engine = load_session(db_path) 172 | 173 | entry_class = ALLOWED_CLASSES[entry_identification.split(":")[0]] 174 | my_id = get_related_ids(session, engine, entry_identification)[0][0] 175 | myobject = session.query(entry_class).filter(entry_class.id == my_id)[0] 176 | 177 | for parameter_key in parameters: 178 | parameter_expression = parameters[parameter_key] 179 | if isinstance(parameter_expression, (str, int, float)): 180 | if ":" in parameter_expression and "." in parameter_expression: 181 | related_entry_ids, _ = get_related_ids(session, engine, i) 182 | related_entry_class = ALLOWED_CLASSES[i.split(":")[0]] 183 | for related_entry_id in related_entry_ids: 184 | related_entry = session.query(related_entry_class).filter(related_entry_class.id == related_entry_id)[0] 185 | setattr(myobject, parameter_key, related_entry) 186 | else: 187 | if parameter_key[-4:] == "date": 188 | parameter_expression = datetime.datetime(*[int(i) for i in parameter_expression.split(",")]) 189 | setattr(myobject, parameter_key, parameter_expression) 190 | else: 191 | set_attribute = getattr(myobject, parameter_key) 192 | for parameter_expression_entry in parameter_expression: 193 | if isinstance(parameter_expression_entry, dict): 194 | new_entry, _ = add_generic(db_path, parameter_expression_entry, session=session, engine=engine) 195 | set_attribute.append(new_entry) 196 | elif isinstance(parameter_expression_entry, str): 197 | related_entry_ids, _ = get_related_ids(session, engine, parameter_expression_entry) 198 | related_entry_class = ALLOWED_CLASSES[parameter_expression_entry.split(":")[0]] 199 | for related_entry_id in related_entry_ids: 200 | related_entry = session.query(related_entry_class).filter(related_entry_class.id == related_entry_id)[0] 201 | set_attribute.append(related_entry) 202 | commit_and_close(session, engine) 203 | 204 | def add_generic(db_path, parameters, session=None, engine=None): 205 | """Adds new entries based on a LabbookDB-syntax parameter dictionary. 206 | 207 | Parameters 208 | ---------- 209 | db_path : str 210 | Path to database to open if session and engine parameters are not already passed, can be relative or use tilde to specify the user $HOME. 211 | parameters : str or dict 212 | A LabbookDB-style dictionary (or JSON interpretable as dictionary), where keys are "CATEGORY" and other strings specifying the attribute names for the object to be created, and values are the class name (for "CATEGORY") and either the values to assign (verbatim: string, int, or float) or LabbookDB-syntax strings specifying a related entry, or a list of LabbookDB-syntax strings specifying related entries, or a list of LabbookDB-style dictionaries specifying new entries to be created and linked. 213 | session : sqlalchemy.orm.session.Session, optional 214 | Session instance, as created with labbookdb.db.add.load_session(). 215 | engine : sqlalchemy.engine.Engine, optional 216 | Engine instance correponding to the Session instance under session, as created with labbookdb.db.add.load_session(). 217 | 218 | Returns 219 | ------- 220 | myobject : object 221 | LabbookDB object with SQLAlchemy-compatible attributes (e.g. as found under labbookdb.db.common_classes). 222 | object_id : int 223 | Value of myobject.id attribute. 224 | """ 225 | 226 | if not (session and engine) : 227 | session, engine = load_session(db_path) 228 | close = True 229 | else: 230 | close = False 231 | if isinstance(parameters, str): 232 | parameters = json.loads(parameters) 233 | 234 | category_class = ALLOWED_CLASSES[parameters["CATEGORY"]] 235 | if list(parameters.keys()) == ["CATEGORY"]: 236 | attributes = dir(category_class()) 237 | filtered_attributes = [i for i in attributes if i[0] != "_"] 238 | print("You can list the following keys as part of your parameters: " + ", ".join(filtered_attributes)) 239 | parameters.pop("CATEGORY", None) 240 | 241 | myobject = category_class() 242 | columns = inspect(myobject).mapper.column_attrs 243 | relationships = inspect(myobject).mapper.relationships 244 | all_attributes = [attr.key for attr in columns+relationships] 245 | for key, _ in sorted(parameters.items()): 246 | if key not in all_attributes: 247 | raise ValueError("'"+myobject.__class__.__name__+"' object does not support '"+key+"' attribute. Acceptable attributes are: "+" ,".join(all_attributes)+".") 248 | if key[-4:] == "date": 249 | parameters[key] = datetime.datetime(*[int(i) for i in parameters[key].split(",")]) 250 | if key[-3:] == "_id" and not isinstance(parameters[key], int): 251 | try: 252 | input_values, _ = get_related_ids(session, engine, parameters[key]) 253 | except ValueError: 254 | instructions("table_identifier") 255 | for input_value in input_values: 256 | input_value = int(input_value) 257 | print("Setting", myobject.__class__.__name__+"'s",key,"attribute to",input_value) 258 | setattr(myobject, key, input_value) 259 | #this triggers on-the-fly related-entry creation: 260 | elif isinstance(parameters[key], list): 261 | related_entries=[] 262 | for related_entry in parameters[key]: 263 | if isinstance(related_entry, dict): 264 | related_entry, _ = add_generic(db_path, related_entry, session=session, engine=engine) 265 | related_entries.append(related_entry) 266 | elif isinstance(related_entry, str): 267 | my_id = get_related_ids(session, engine, related_entry)[0][0] 268 | entry_class = ALLOWED_CLASSES[related_entry.split(":")[0]] 269 | related_entry = session.query(entry_class).\ 270 | filter(entry_class.id == my_id).all()[0] 271 | related_entries.append(related_entry) 272 | session.add(myobject) # voodoo (imho) fix for the weird errors about myobject not being attached to a Session 273 | print("Setting", myobject.__class__.__name__+"'s",key,"attribute to",related_entries) 274 | setattr(myobject, key, related_entries) 275 | else: 276 | print("Setting", myobject.__class__.__name__+"'s",key,"attribute to",parameters[key]) 277 | setattr(myobject, key, parameters[key]) 278 | 279 | object_id = add_to_db(session, engine, myobject) 280 | if close: 281 | session.close() 282 | engine.dispose() 283 | return myobject, object_id 284 | 285 | def commit_and_close(session, engine): 286 | """Commit and close session and dispose of engine. 287 | Nonfatal for sqlalchemy.exc.IntegrityError with print notification. 288 | 289 | Parameters 290 | ---------- 291 | session : sqlalchemy.orm.session.Session, optional 292 | Session instance, as created with labbookdb.db.add.load_session(). 293 | engine : sqlalchemy.engine.Engine, optional 294 | Engine instance correponding to the Session instance under session, as created with labbookdb.db.add.load_session(). 295 | """ 296 | 297 | try: 298 | session.commit() 299 | except sqlalchemy.exc.IntegrityError: 300 | print("Please make sure this was not a double entry.") 301 | session.close() 302 | engine.dispose() 303 | -------------------------------------------------------------------------------- /labbookdb/db/query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import argh 4 | import json 5 | import sys 6 | 7 | import pandas as pd 8 | 9 | import datetime 10 | import os 11 | from copy import deepcopy 12 | from sqlalchemy.orm import sessionmaker, aliased, contains_eager 13 | import sqlalchemy 14 | 15 | from .common_classes import * 16 | 17 | ALLOWED_CLASSES = { 18 | "Animal": Animal, 19 | "AnimalExternalIdentifier": AnimalExternalIdentifier, 20 | "AnesthesiaProtocol": AnesthesiaProtocol, 21 | "Arena": Arena, 22 | "Biopsy": Biopsy, 23 | "BrainBiopsy": BrainBiopsy, 24 | "BrainExtractionProtocol": BrainExtractionProtocol, 25 | "Cage": Cage, 26 | "CageStay": CageStay, 27 | "DrinkingMeasurement": DrinkingMeasurement, 28 | "DNAExtractionProtocol": DNAExtractionProtocol, 29 | "Evaluation": Evaluation, 30 | "FluorescentMicroscopyMeasurement": FluorescentMicroscopyMeasurement, 31 | "FMRIMeasurement": FMRIMeasurement, 32 | "FMRIScannerSetup": FMRIScannerSetup, 33 | "ForcedSwimTestMeasurement": ForcedSwimTestMeasurement, 34 | "Genotype": Genotype, 35 | "HandlingHabituationProtocol": HandlingHabituationProtocol, 36 | "HandlingHabituation": HandlingHabituation, 37 | "Incubation": Incubation, 38 | "Ingredient": Ingredient, 39 | "Irregularity": Irregularity, 40 | "MeasurementUnit": MeasurementUnit, 41 | "Measurement": Measurement, 42 | "Observation": Observation, 43 | "OpenFieldTestMeasurement": OpenFieldTestMeasurement, 44 | "Operator": Operator, 45 | "Operation": Operation, 46 | "OpticFiberImplant": OpticFiberImplant, 47 | "OpticFiberImplantProtocol": OpticFiberImplantProtocol, 48 | "OrthogonalStereotacticTarget": OrthogonalStereotacticTarget, 49 | "Protocol": Protocol, 50 | "SectioningProtocol": SectioningProtocol, 51 | "StimulationProtocol": StimulationProtocol, 52 | "StimulationEvent": StimulationEvent, 53 | "Substance": Substance, 54 | "SucrosePreferenceMeasurement": SucrosePreferenceMeasurement, 55 | "Solution": Solution, 56 | "Treatment": Treatment, 57 | "TreatmentProtocol": TreatmentProtocol, 58 | "Virus": Virus, 59 | "VirusInjectionProtocol": VirusInjectionProtocol, 60 | "WeightMeasurement": WeightMeasurement, 61 | } 62 | 63 | def get_related_id(session, engine, parameters): 64 | category = parameters.split(":",1)[0] 65 | sql_query=session.query(ALLOWED_CLASSES[category]) 66 | for field_value in parameters.split(":",1)[1].split("&&"): 67 | field, value = field_value.split(".",1) 68 | if ":" in value: 69 | values = get_related_id(session, engine, value) 70 | for value in values: 71 | value=int(value) # the value is returned as a numpy object 72 | if field[-4:] == "date": # support for date entry matching (the values have to be passes as string but matched as datetime) 73 | value = datetime.datetime(*[int(i) for i in value.split(",")]) 74 | sql_query = sql_query.filter(getattr(ALLOWED_CLASSES[category], field)==value) 75 | else: 76 | if field[-4:] == "date": # support for date entry matching (the values have to be passes as string but matched as datetime) 77 | value = datetime.datetime(*[int(i) for i in value.split(",")]) 78 | sql_query = sql_query.filter(getattr(ALLOWED_CLASSES[category], field)==value) 79 | mystring = sql_query.statement 80 | mydf = pd.read_sql_query(mystring,engine) 81 | mydf = mydf.T.groupby(level=0).first().T #awkward hack to deal with polymorphic tables returning multiple IDs 82 | related_table_ids = mydf["id"] 83 | input_values = list(related_table_ids) 84 | if input_values == []: 85 | raise BaseException("No entry was found with a value of \""+str(value)+"\" on the \""+field+"\" column of the \""+category+"\" CATEGORY, in the database.") 86 | session.close() 87 | engine.dispose() 88 | return input_values 89 | 90 | @argh.arg('-p', '--db_path', type=str) 91 | @argh.arg('database', default=None, nargs="?") 92 | def animal_info(identifier, database, 93 | db_path=None, 94 | ): 95 | """Return the __str__ attribute of an Animal object query filterd by the id column OR by arguments of the external_id objects. 96 | 97 | Parameters 98 | ---------- 99 | 100 | db_path : string 101 | Path to a LabbookDB formatted database. 102 | 103 | identifier : int or string 104 | The identifier of the animal 105 | 106 | database : string or None, optional 107 | If specified gives a constraint on the AnimalExternalIdentifier.database column AND truns the identifier attribute into a constraint on the AnimalExternalIdentifier.identifier column. If unspecified, the identfier argument is used as a constraint on the Animal.id column. 108 | """ 109 | 110 | if not db_path: 111 | db_path = os.environ["LDB_PATH"] 112 | 113 | if database and identifier: 114 | session, engine = load_session(db_path) 115 | sql_query = session.query(Animal) 116 | sql_query = sql_query.join(Animal.external_ids)\ 117 | .filter(AnimalExternalIdentifier.database == database).filter(AnimalExternalIdentifier.identifier == identifier)\ 118 | .options(contains_eager('external_ids')) 119 | identifier = [i for i in sql_query][0].id 120 | session.close() 121 | engine.dispose() 122 | 123 | session, engine = load_session(db_path) 124 | sql_query = session.query(Animal) 125 | sql_query = sql_query.filter(Animal.id == identifier) 126 | try: 127 | animal = [i.__str__() for i in sql_query][0] 128 | except: 129 | if database == None: 130 | print("No entry was found with {id} in the Animal.id column of the database located at {loc}.".format(id=identifier,loc=db_path)) 131 | else: 132 | print("No entry was found with {id} in the AnimalExternalIdentifier.identifier column and {db} in the AnimalExternalIdentifier.database column of the database located at {loc}.".format(id=identifier,db=database,loc=db_path)) 133 | else: 134 | print(animal) 135 | session.close() 136 | engine.dispose() 137 | 138 | def cage_info(db_path, identifier, 139 | ): 140 | """Return the __str__ attribute of an Animal object query filterd by the id column OR by arguments of the external_id objects. 141 | 142 | Parameters 143 | ---------- 144 | 145 | db_path : string 146 | Path to a LabbookDB formatted database. 147 | 148 | identifier : int or string 149 | The identifier of the animal 150 | 151 | database : string or None, optional 152 | If specified gives a constraint on the AnimalExternalIdentifier.database colun AND truns the identifier attribute into a constraint on the AnimalExternalIdentifier.identifier column. If unspecified, the identfier argument is used as a constraint on the Animal.id column. 153 | """ 154 | 155 | session, engine = load_session(db_path) 156 | sql_query = session.query(Cage) 157 | sql_query = sql_query.filter(Cage.id == identifier) 158 | cage = [i.__str__() for i in sql_query][0] 159 | try: 160 | cage = [i.__str__() for i in sql_query][0] 161 | except: 162 | print("No entry was found with {id} in the Cage.id column of the database located at {loc}.".format(id=identifier,loc=db_path)) 163 | else: 164 | print(cage) 165 | session.close() 166 | engine.dispose() 167 | 168 | def load_session(db_path): 169 | db_path = "sqlite:///" + os.path.expanduser(db_path) 170 | engine = sqlalchemy.create_engine(db_path, echo=False) 171 | Session = sessionmaker(bind=engine) 172 | session = Session() 173 | Base.metadata.create_all(engine) 174 | return session, engine 175 | 176 | def commit_and_close(session, engine): 177 | try: 178 | session.commit() 179 | except sqlalchemy.exc.IntegrityError: 180 | print("Please make sure this was not a double entry.") 181 | session.close() 182 | engine.dispose() 183 | 184 | def add_all_columns(cols, class_name): 185 | joinclassobject = ALLOWED_CLASSES[class_name] 186 | 187 | #we need to catch this esception, because for aliased classes a mapper is not directly returned 188 | try: 189 | col_name_cols = sqlalchemy.inspection.inspect(joinclassobject).columns.items() 190 | except AttributeError: 191 | col_name_cols = sqlalchemy.inspection.inspect(joinclassobject).mapper.columns.items() 192 | 193 | for col_name, col in col_name_cols: 194 | column = getattr(joinclassobject, col.key) 195 | cols.append(column.label("{}_{}".format(class_name, col_name))) 196 | 197 | def get_for_protocolize(db_path, class_name, code): 198 | """Return a dataframe containing a specific entry from a given class name, joined with its related tables up to three levels down. 199 | """ 200 | session, engine = load_session(db_path) 201 | cols = [] 202 | joins = [] 203 | classobject = ALLOWED_CLASSES[class_name] 204 | insp = sqlalchemy.inspection.inspect(classobject) 205 | for name, col in insp.columns.items(): 206 | cols.append(col.label(name)) 207 | for name, rel in insp.relationships.items(): 208 | alias = aliased(rel.mapper.class_, name=name) 209 | joins.append((alias, rel.class_attribute)) 210 | for col_name, col in sqlalchemy.inspection.inspect(rel.mapper).columns.items(): 211 | #the id column causes double entries, as it is mapped once on the parent table (related_table_id) and once on the child table (table_id) 212 | if col.key != "id": 213 | aliased_col = getattr(alias, col.key) 214 | cols.append(aliased_col.label("{}_{}".format(name, col_name))) 215 | 216 | sub_insp = sqlalchemy.inspection.inspect(rel.mapper.class_) 217 | for sub_name, sub_rel in sub_insp.relationships.items(): 218 | if "contains" not in sub_name: 219 | sub_alias = aliased(sub_rel.mapper.class_, name=name+"_"+sub_name) 220 | joins.append((sub_alias, sub_rel.class_attribute)) 221 | for sub_col_name, sub_col in sqlalchemy.inspection.inspect(sub_rel.mapper).columns.items(): 222 | #the id column causes double entries, as it is mapped once on the parent table (related_table_id) and once on the child table (table_id) 223 | if sub_col.key != "id": 224 | sub_aliased_col = getattr(sub_alias, sub_col.key) 225 | cols.append(sub_aliased_col.label("{}_{}_{}".format(name, sub_name, sub_col_name))) 226 | 227 | sql_query = session.query(*cols).select_from(classobject) 228 | for join in joins: 229 | sql_query = sql_query.outerjoin(*join) 230 | sql_query = sql_query.filter(classobject.code == code) 231 | 232 | mystring = sql_query.statement 233 | mydf = pd.read_sql_query(mystring,engine) 234 | 235 | session.close() 236 | engine.dispose() 237 | return mydf 238 | 239 | def get_df(db_path, 240 | col_entries=[], 241 | default_join="inner", 242 | filters=[], 243 | join_entries=[], 244 | join_types=[], 245 | ): 246 | """Return a dataframe from a complex query of a LabbookDB-style database 247 | 248 | Arguments 249 | --------- 250 | 251 | db_path : string 252 | Path to database file. 253 | col_entries : list 254 | A list of tuples containing the columns to be queried: 255 | * 1-tuples indicate all attributes of a class are to be queried 256 | * 2-tuples indicate only the attribute specified by the second element, of the class specified by the first element is to be queried 257 | * 3-tuples indicate that an aliased class of the type given by the second element is to be created and named according to the first and second elements, separated by an underscore. The attribute given by the third element will be queried; if the thid element is empty, all attributes will be queried 258 | join_entries : list 259 | A list of tuples specifying the desired join operations: 260 | * 1-tuples give the join 261 | * 2-tuples give the class to be joined on the first element, and the explicit relationship (attribute of another class) on the second element 262 | If any of the elements contains a period, the expression will be evaluated as a class (preceeding the period) attribute (after the period)$ 263 | filters : list 264 | A list of lists giving filters for the query. In each sub-list the first and second elements give the class and attribute to be matched. Every following element specifies a possible value for the class attribute (implemented as inclusive disjunction). If the attribute name ends in "date" the function computes datetime objects from the subsequent strings containing numbers separated by commas. 265 | !!!incomplete documentation 266 | 267 | 268 | Examples 269 | -------- 270 | >>> col_entries=[ 271 | ("Animal","id"), 272 | ("Treatment",), 273 | ("FMRIMeasurement",), 274 | ("TreatmentProtocol","code"), 275 | ("Cage","id"), 276 | ("Cage","Treatment",""), 277 | ("Cage","TreatmentProtocol","code") 278 | ] 279 | >>> join_entries=[ 280 | ("Animal.treatments",), 281 | ("FMRIMeasurement",), 282 | ("Treatment.protocol",), 283 | ("Animal.cage_stays",), 284 | ("CageStay.cage",), 285 | ("Cage_Treatment","Cage.treatments"), 286 | ("Cage_TreatmentProtocol","Cage_Treatment.protocol") 287 | ] 288 | >>> filters = [["Cage_Treatment","start_date","2016,4,25,19,30"]] 289 | >>> reference_df = get_df("~/syncdata/meta.db", col_entries=col_entries, join_entries=join_entries, filters=filters) 290 | 291 | """ 292 | 293 | session, engine = load_session(db_path) 294 | 295 | cols=[] 296 | for col_entry in col_entries: 297 | if len(col_entry) == 1: 298 | add_all_columns(cols, col_entry[0]) 299 | if len(col_entry) == 2: 300 | cols.append(getattr(ALLOWED_CLASSES[col_entry[0]],col_entry[1]).label("{}_{}".format(*col_entry))) 301 | if len(col_entry) == 3: 302 | aliased_class = aliased(ALLOWED_CLASSES[col_entry[1]]) 303 | ALLOWED_CLASSES[col_entry[0]+"_"+col_entry[1]] = aliased_class 304 | if col_entry[2] == "": 305 | add_all_columns(cols, col_entry[0]+"_"+col_entry[1]) 306 | else: 307 | cols.append(getattr(aliased_class,col_entry[2]).label("{}_{}_{}".format(*col_entry))) 308 | 309 | joins=[] 310 | for join_entry in join_entries: 311 | join_parameters = [] 312 | for join_entry_substring in join_entry: 313 | if "." in join_entry_substring: 314 | class_name, table_name = join_entry_substring.split(".") #if this unpacks >2 values, the user specified strings are malformed 315 | join_parameters.append(getattr(ALLOWED_CLASSES[class_name],table_name)) 316 | else: 317 | join_parameters.append(ALLOWED_CLASSES[join_entry_substring]) 318 | joins.append(join_parameters) 319 | 320 | sql_query = session.query(*cols) 321 | #we need to make sure we don't edit the join_types variable passed to this function 322 | join_types = deepcopy(join_types) 323 | while len(join_types) < len(joins): 324 | join_types.append(default_join) 325 | for ix, join in enumerate(joins): 326 | if join_types[ix] == "inner": 327 | sql_query = sql_query.join(*join) 328 | elif join_types[ix] == "outer": 329 | sql_query = sql_query.outerjoin(*join) 330 | 331 | for sub_filter in filters: 332 | if sub_filter: 333 | if sub_filter[1][-4:] == "date" and isinstance(sub_filter[2], str): 334 | for ix, i in enumerate(sub_filter[2:]): 335 | sub_filter[2+ix] = datetime.datetime(*[int(a) for a in i.split(",")]) 336 | if len(sub_filter) == 3: 337 | sql_query = sql_query.filter(getattr(ALLOWED_CLASSES[sub_filter[0]], sub_filter[1]) == sub_filter[2]) 338 | else: 339 | sql_query = sql_query.filter(sqlalchemy.or_(getattr(ALLOWED_CLASSES[sub_filter[0]], sub_filter[1]) == v for v in sub_filter[2:])) 340 | 341 | mystring = sql_query.statement 342 | df = pd.read_sql_query(mystring,engine) 343 | session.close() 344 | engine.dispose() 345 | 346 | return df 347 | 348 | #THIS IS KEPT TO REMEMBER WHENCE THE ABOVE AWKWARD ROUTINES CAME AND HOW THE CODE IS SUPPOSED TO LOOK IF TYPED OUT 349 | # CageTreatment = aliased(Treatment) 350 | # CageTreatmentProtocol = aliased(TreatmentProtocol) 351 | # sql_query = session.query( 352 | # Animal.id.label("Animal_id"), 353 | # TreatmentProtocol.code.label("TreatmentProtocol_code"), 354 | # Cage.id.label("Cage_id"), 355 | # CageTreatment.id.label("Cage_Treatment_id"), 356 | # CageTreatmentProtocol.code.label("Cage_TreatmentProtocol_code"), 357 | # )\ 358 | # .join(Animal.treatments)\ 359 | # .join(Treatment.protocol)\ 360 | # .join(Animal.cage_stays)\ 361 | # .join(CageStay.cage)\ 362 | # .join(CageTreatment, Cage.treatments)\ 363 | # .join(CageTreatmentProtocol, CageTreatment.protocol)\ 364 | # .filter(Animal.id == 43) 365 | # mystring = sql_query.statement 366 | # reference_df = pd.read_sql_query(mystring,engine) 367 | # print reference_df.columns 368 | # print reference_df 369 | -------------------------------------------------------------------------------- /labbookdb/report/tracking.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | from labbookdb.decorators import environment_db_path 4 | from labbookdb.report.utilities import * 5 | from labbookdb.report import selection 6 | from labbookdb.db import query 7 | 8 | 9 | TABLE_COL_SPACE = 150 10 | 11 | 12 | def animals_id(db_path, 13 | save_as=None, 14 | ): 15 | """ 16 | Extract list of animal (database and external) IDs, and either print it to screen or save it as an HTML file. 17 | 18 | Parameters 19 | ---------- 20 | 21 | db_path : string 22 | Path to the database file to query. 23 | 24 | save_as : string or None, optional 25 | Path under which to save the HTML report (".html" is automatically appended). If None, the report is printed to the terminal. 26 | """ 27 | 28 | df = selection.parameterized(db_path, "animals id") 29 | 30 | df = df.rename(columns={'AnimalExternalIdentifier_database': 'External Database:', 'AnimalExternalIdentifier_animal_id': 'ID'}) 31 | df = df.set_index(['ID', 'External Database:'])['AnimalExternalIdentifier_identifier'].unstack(1).reset_index() 32 | df.set_index('ID', inplace=True) 33 | df = df.sort_index(ascending=False) 34 | 35 | if save_as: 36 | df.to_html(os.path.abspath(os.path.expanduser(save_as+".html"), col_space=TABLE_COL_SPACE)) 37 | else: 38 | print(df) 39 | return 40 | 41 | @environment_db_path() 42 | def animals_info(db_path, 43 | save_as=None, 44 | functional_scan_responders=True, 45 | treatments=True, 46 | ): 47 | """ 48 | Extract list of animal (database and external) IDs and their death dates and genotypes, and either print it to screen or save it as an HTML file. 49 | 50 | Parameters 51 | ---------- 52 | 53 | db_path : string 54 | Path to the database file to query. 55 | 56 | save_as : string or None, optional 57 | Path under which to save the HTML report (".html" is automatically appended to the name, if not already present). If None, the report is printed to the terminal. 58 | 59 | functional_scan_responders : bool, optional 60 | Whether to create and list a column tracking how many of the scans in which an animal was exposed to stimulation show ICA results in a qualitative analysis. 61 | 62 | treatments : bool, optional 63 | Whether to create a and list columns tracking what animal-based and cage-based treatements the animal was subjected to. 64 | 65 | """ 66 | 67 | df = selection.parameterized(db_path, "animals info") 68 | 69 | collapse = { 70 | 'Animal_death_date' : lambda x: ', '.join(set([str(i) for i in x])), 71 | 'Genotype_code' : lambda x: ', '.join(set([str(i) for i in x])), 72 | } 73 | short_identifiers = make_identifier_short_form(df) 74 | df = short_identifiers.join(collapse_rename(df, 'AnimalExternalIdentifier_animal_id', collapse)) 75 | df.reset_index().set_index('Animal_id', inplace=True) 76 | 77 | if functional_scan_responders: 78 | count_scans = {'occurences' : lambda x: sum(x),} 79 | 80 | collapse = { 81 | 'StimulationProtocol_code' : lambda x: 0 if list(x) == [] else 1, 82 | "Animal_id" : lambda x: list(x)[0], 83 | } 84 | rename = {'StimulationProtocol_code': 'occurences'} 85 | functional_scan_df = selection.parameterized(db_path, "animals measurements") 86 | functional_scan_df = collapse_rename(functional_scan_df, "Measurement_id", collapse, rename) 87 | functional_scan_df = collapse_rename(functional_scan_df, 'Animal_id', count_scans) 88 | 89 | collapse = { 90 | 'Irregularity_description' : lambda x: 1 if "ICA failed to indicate response to stimulus" in list(x) else 0, 91 | "Animal_id" : lambda x: list(x)[0], 92 | } 93 | rename ={'Irregularity_description': 'occurences'} 94 | nonresponder_df = selection.parameterized(db_path, "animals measurements irregularities") 95 | nonresponder_df = collapse_rename(nonresponder_df, 'Measurement_id', collapse, rename) 96 | nonresponder_df = collapse_rename(nonresponder_df, 'Animal_id', count_scans) 97 | 98 | df['nonresponsive'] = nonresponder_df 99 | df['functional'] = functional_scan_df 100 | df[['nonresponsive', 'functional']] = df[["nonresponsive", 'functional']].fillna(0).astype(int) 101 | df["responsive functional scans"] = df['functional'] - df['nonresponsive'] 102 | df["responsive functional scans"] = df["responsive functional scans"].astype(str) +"/"+ df['functional'].astype(str) 103 | df.drop(['nonresponsive', 'functional'], axis = 1, inplace = True, errors = 'ignore') 104 | 105 | if treatments: 106 | treatments_df = selection.animal_treatments(db_path) 107 | collapse_treatments = { 108 | 'TreatmentProtocol_code' : lambda x: ', '.join(set([str(i) for i in x if i])), 109 | 'Cage_TreatmentProtocol_code' : lambda x: ', '.join(set([i for i in x if i])), 110 | } 111 | treatments_rename = { 112 | 'TreatmentProtocol_code': 'animal_treatment', 113 | 'Cage_TreatmentProtocol_code': 'cage_treatment', 114 | } 115 | treatments_df = treatments_df.groupby('Animal_id').agg(collapse_treatments) 116 | treatments_df = treatments_df.rename(columns=treatments_rename) 117 | df['animal_treatment'] = treatments_df["animal_treatment"] 118 | df['cage_treatment'] = treatments_df["cage_treatment"] 119 | 120 | df = df.sort_index(ascending=False) 121 | df = df.fillna('') 122 | if save_as: 123 | if os.path.splitext(save_as)[1] in [".html",".HTML"]: 124 | df.to_html(os.path.abspath(os.path.expanduser(save_as)), col_space=TABLE_COL_SPACE) 125 | elif os.path.splitext(save_as)[1] in [".tsv",".TSV"]: 126 | df.to_csv(save_as, sep='\t', encoding='utf-8') 127 | elif os.path.splitext(save_as)[1] in [".csv",".CSV", ""]: 128 | df.to_csv(save_as, encoding='utf-8') 129 | else: 130 | print("WARNING: This function currently only supports `.csv`, `.tsv`, or `.html` output. Please append one of the aforementioned extensions to the specified file name (`{}`), or specify no extension - in which case `.csv` will be added and an according output will be created.".format(save_as)) 131 | return df 132 | 133 | def bids_eventsfile(db_path, code, 134 | strict=False): 135 | """ 136 | Return a BIDS-formatted eventfile for a given code 137 | 138 | Parameters 139 | ---------- 140 | 141 | db_path : string 142 | Path to the database file to query. 143 | 144 | code : string 145 | Code (valid `StimulationProtocol.code` value) which identifies the stimulation protocol to format. 146 | strict : bool, optional 147 | Whether to strict about respecting BIDS specifics. 148 | (currently removes coumns with only empty cells) 149 | """ 150 | 151 | df = selection.stimulation_protocol(db_path, code) 152 | bids_df = pd.DataFrame([]) 153 | bids_df['onset'] = df['StimulationEvent_onset'] 154 | bids_df['duration'] = df['StimulationEvent_duration'] 155 | bids_df['frequency'] = df['StimulationEvent_frequency'] 156 | bids_df['pulse_width'] = df['StimulationEvent_pulse_width'] 157 | bids_df['onset'] = df['StimulationEvent_onset'] 158 | bids_df['trial_type'] = df['StimulationEvent_trial_type'] 159 | bids_df['wavelength'] = df['StimulationEvent_wavelength'] 160 | bids_df['strength'] = df['StimulationEvent_strength'] 161 | bids_df['strength_unit'] = df['MeasurementUnit_code'] 162 | 163 | if strict: 164 | bids_df = bids_df.dropna(axis=1, how='all') 165 | 166 | return bids_df 167 | 168 | def cage_consumption(db_path, df, 169 | treatment_relative_date=True, 170 | rounding='D', 171 | ): 172 | """ 173 | Return a `pandas.DataFrame` object containing information about the per-animal drinking solution consumption of single cages. 174 | 175 | Parameters 176 | ---------- 177 | db_path : string 178 | Path to the database file to query. 179 | df : pandas.DataFrame 180 | A `pandas.DataFrame` object with `DrinkingMeasurement_id`, `DrinkingMeasurement_reference_date`, `DrinkingMeasurement_date`, `DrinkingMeasurement_start_amount`, `DrinkingMeasurement_start_amount` columns. 181 | This can be obtained e.g. from `labbookdb.report.selection.cage_drinking_measurements()`. 182 | treatment_relative_date : bool, optional 183 | Whether to express the dates relative to a treatment onset. 184 | It is assumed that only one cage treatment is recorded per cage, if this is not so, this function may not work as expected. 185 | rounding : string, optional 186 | Whether to round dates and timedeltas - use strings as supported by pandas. [1]_ 187 | 188 | Notes 189 | ----- 190 | This function caluclates the per-day consumption based on phase-agnostic and potentially rounded and day values. 191 | This is prone to some inaccuracy, as drinking is generally restricted to specific times of the day. 192 | Ideally, a `waking_hour_consumption` should be estimated based on exact times of day and day cycle. 193 | 194 | References 195 | ---------- 196 | 197 | .. [1] http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases 198 | """ 199 | 200 | selected_cages = list(df['Cage_id'].unique()) 201 | occupancy_df = selection.cage_periods(db_path, cage_filter=selected_cages) 202 | df['occupancy']='' 203 | for measurement in df['DrinkingMeasurement_id'].tolist(): 204 | selected = df[df['DrinkingMeasurement_id']==measurement] 205 | measurement_start = selected['DrinkingMeasurement_reference_date'].values[0] 206 | measurement_end = selected['DrinkingMeasurement_date'].values[0] 207 | cage_id = selected['Cage_id'].values[0] 208 | occupants = occupancy_df[ 209 | (occupancy_df['CageStay_start_date']<=measurement_start)& 210 | ( 211 | (occupancy_df['CageStay_end_date']>=measurement_end)| 212 | (occupancy_df['CageStay_end_date'].isnull()) 213 | )& 214 | (occupancy_df['Cage_id']==cage_id) 215 | ] 216 | if True in occupants['Animal_id'].duplicated().tolist(): 217 | print(occupants) 218 | raise ValueError('An animal ist listed twice in the occupancy list of a cage (printed above). This biases the occupancy evaluation, and is likely diagnostic of a broader processing error.') 219 | occupancy = len(occupants.index) 220 | df.loc[(df['DrinkingMeasurement_id']==measurement),'occupancy'] = occupancy 221 | df['consumption'] = df['DrinkingMeasurement_start_amount']-df['DrinkingMeasurement_end_amount'] 222 | if treatment_relative_date: 223 | df['relative_start_date'] = '' 224 | df['relative_end_date'] = '' 225 | df['relative_start_date'] = df['relative_start_date'].astype('timedelta64[ns]') 226 | df['relative_end_date'] = df['relative_end_date'].astype('timedelta64[ns]') 227 | df["relative_start_date"] = df["DrinkingMeasurement_reference_date"]-df["Treatment_start_date"] 228 | df["relative_end_date"] = df["DrinkingMeasurement_date"]-df["Treatment_start_date"] 229 | if rounding: 230 | df['relative_start_date'] = df['relative_start_date'].dt.round(rounding) 231 | df['relative_end_date'] = df['relative_end_date'].dt.round(rounding) 232 | df['relative_end_date'] = df['relative_end_date'].dt.days.astype(int) 233 | df['relative_start_date'] = df['relative_start_date'].dt.days.astype(int) 234 | 235 | # Here we calculate the day consumption based on phase-agnostic and potentially rounded and day values. 236 | # This is prone to some inaccuracy, as drinking is generally restricted to specific times of the day. 237 | # Ideally, a `waking_hour_consumption` should be estimated based on exact times of day and day cycle. 238 | df['day_consumption'] = df['consumption']/(df['relative_end_date'] - df['relative_start_date']) 239 | df['day_animal_consumption']=df['day_consumption']/df['occupancy'] 240 | return df 241 | 242 | def append_external_identifiers(db_path, df, 243 | concatenate=[], 244 | ): 245 | """ 246 | Append external animal IDs to a dataframe containing an `Animal_id` (`Animal.id`) column. 247 | 248 | Parameters 249 | ---------- 250 | 251 | db_path : str 252 | Path to database fuile to query. 253 | df : pandas.DataFrame 254 | A `pandas.DataFrame` object containing an `Animal_id` (`Animal.id`) column. 255 | concatenate : list, optional 256 | A list containing any combination of 'Animal_death_date', 'Genotype_id', 'Genotype_code', 'Genotype_construct'. 257 | """ 258 | 259 | df_id = selection.parameterized(db_path, "animals info", 260 | animal_filter=list(df['Animal_id'].unique()), 261 | ) 262 | 263 | collapse = {} 264 | if concatenate: 265 | for i in concatenate: 266 | collapse[i] = lambda x: ', '.join(set([str(i) for i in x])) 267 | short_identifiers = make_identifier_short_form(df_id) 268 | df_id = short_identifiers.join(collapse_rename(df_id, 'AnimalExternalIdentifier_animal_id', collapse)) 269 | df_id.reset_index(inplace=True) 270 | df = pd.merge(df_id, df, on='Animal_id', how='inner') 271 | 272 | return df 273 | 274 | 275 | def treatment_group(db_path, treatments, 276 | level="", 277 | ): 278 | """ 279 | Return a `pandas.DataFrame` object containing the per animal start dates of a particular treatment code (applied either at the animal or the cage levels). 280 | 281 | Parameters 282 | ---------- 283 | 284 | db_path : string 285 | Path to database file to query. 286 | code : string 287 | Desired treatment code (`Treatment.code` attribute) to filter for. 288 | level : {"animal", "cage"} 289 | Whether to query animal treatments or cage treatments. 290 | 291 | Notes 292 | ----- 293 | 294 | This function checks whether cage-level treatment onsets indeed happened during the period in which the animal was housed in teh cage. 295 | We do not check for the treatment end dates, as an animal which has received a partial treatment has received a treatment. 296 | Checks for treatment discontinuation due to e.g. death should be performed elsewhere. 297 | """ 298 | if not level: 299 | level = "animal" 300 | if level=="animal": 301 | df = selection.animal_treatments(db_path, animal_treatments=treatments) 302 | elif level=="cage": 303 | df = selection.animal_treatments(db_path, cage_treatments=treatments) 304 | return df 305 | 306 | def qualitative_dates(df, 307 | iterator_column='Animal_id', 308 | date_column='relative_date', 309 | label='qualitative_date', 310 | fuzzy_matching={}, 311 | ): 312 | """ 313 | Assign qualitative date labels. 314 | 315 | Parameters 316 | ---------- 317 | 318 | df : pandas.DataFrame 319 | A `pandas.DataFrame` object containing a date column. 320 | iteraor_column : string, optional 321 | The label of the column which identifies the base entities of which each should be assigned a set of qualitatie dates (most commonly this is `Animal_id`, or `Cage_id`). 322 | date_column : string, optional 323 | The label of the column which serves as the quantitative record which is to be discretized into qualitative dates. 324 | label : string, optional 325 | The label to assign to the new qualitative date column. 326 | fuzzy_assignment : dict, optional 327 | A dictionary the keys of which are qualitative date labels to be assigned, and the values of which are lists giving the quantitative date labels in the order of preference based on which to assign the labels. 328 | """ 329 | 330 | df[label]='' 331 | for i in df[iterator_column]: 332 | try: 333 | for label, dates in fuzzy_matching.iteritems(): 334 | for date in dates: 335 | if date in df[df[iterator_column]==i][date_column].values: 336 | df.loc[(df[iterator_column]==i)&(df[date_column]==date),'qualitative_date']=label 337 | break 338 | except AttributeError: 339 | for label, dates in fuzzy_matching.items(): 340 | for date in dates: 341 | if date in df[df[iterator_column]==i][date_column].values: 342 | df.loc[(df[iterator_column]==i)&(df[date_column]==date),'qualitative_date']=label 343 | break 344 | return df 345 | 346 | def animal_weights(db_path, 347 | reference={}, 348 | rounding="D", 349 | ): 350 | """ 351 | Return a dataframe containing animal weights and dates. 352 | 353 | Parameters 354 | ---------- 355 | 356 | db_path : string 357 | Path to database file to query. 358 | reference : dict, optional 359 | Dictionary based on which to apply a reference date for the dates of each animal. Keys of this dictionary must be "animal" or "cage", and values must be lists of treatment codes. 360 | rounding : string, optional 361 | Whether to round dates and timedeltas - use strings as supported by pandas. [1]_ 362 | 363 | References 364 | ---------- 365 | 366 | .. [1] http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases 367 | """ 368 | import pandas as pd 369 | 370 | df = selection.parameterized(db_path, "animals weights") 371 | short_identifiers = make_identifier_short_form(df, index_name="WeightMeasurement_id") 372 | collapse = { 373 | 'WeightMeasurement_date' : lambda x: list(set(x))[0] if (len(set(x)) == 1) else "WARNING: different values were present for this entry. Data in this entire DataFrame may not be trustworthy.", 374 | 'WeightMeasurement_weight' : lambda x: list(set(x))[0] if (len(set(x)) == 1) else "WARNING: different values were present for this entry. Data in this entire DataFrame may not be trustworthy.", 375 | 'AnimalExternalIdentifier_animal_id' : lambda x: list(set(x))[0] if (len(set(x)) == 1) else "WARNING: different values were present for this entry. Data in this entire DataFrame may not be trustworthy.", 376 | } 377 | rename = { 378 | 'WeightMeasurement_date': 'date', 379 | 'WeightMeasurement_weight': 'weight', 380 | 'AnimalExternalIdentifier_animal_id': 'Animal_id', 381 | } 382 | df = short_identifiers.join(collapse_rename(df, 'WeightMeasurement_id', collapse, rename)) 383 | if reference: 384 | if list(reference.keys())[0] == 'animal': 385 | start_date_label = 'Treatment_start_date' 386 | elif list(reference.keys())[0] == 'cage': 387 | start_date_label = 'Cage_Treatment_start_date' 388 | onsets = treatment_group(db_path, list(reference.values())[0], level=list(reference.keys())[0]) 389 | df['relative_date'] = '' 390 | df['relative_date'] = df['relative_date'].astype('timedelta64[ns]') 391 | for subject in df["Animal_id"]: 392 | try: 393 | df.loc[df["Animal_id"]==subject,"relative_date"] = df.loc[df["Animal_id"]==subject,"date"]-onsets.loc[onsets["Animal_id"]==subject, start_date_label].values[0] 394 | except IndexError: 395 | df.drop(df[df["Animal_id"]==subject].index, inplace=True) 396 | df = pd.merge(df, onsets, on='Animal_id', how='outer') 397 | if rounding: 398 | df['relative_date'] = df['relative_date'].dt.round(rounding) 399 | if rounding: 400 | df['date'] = df['date'].dt.round(rounding) 401 | return df 402 | 403 | def further_cages(db_path): 404 | """ 405 | Returns cage numbers that should be selected for incoming cages. 406 | 407 | Parameters 408 | ---------- 409 | db_path : path to database file to query (needs to be protocolizer-style) 410 | """ 411 | 412 | df = selection.parameterized(db_path, "cage list") 413 | 414 | cages = df["Cage_id"].dropna().tolist() 415 | cages = list(set([int(i) for i in cages])) 416 | 417 | last_cage = cages[-1] 418 | next_cage = last_cage+1 419 | skipped_cages=[] 420 | for cage in cages: 421 | cage_increment = 1 422 | while True: 423 | if cage+cage_increment >= last_cage: 424 | break 425 | if cage+cage_increment not in cages: 426 | skipped_cages.append(cage+cage_increment) 427 | cage_increment += 1 428 | else: 429 | break 430 | 431 | if not skipped_cages: 432 | skipped_cages = ["None"] 433 | 434 | print("Next open cage number: {0}\nSkipped cage numbers: {1}".format(next_cage, ", ".join([str(i) for i in skipped_cages]))) 435 | return 436 | 437 | def overview(db_path, 438 | default_join=False, 439 | filters=[], 440 | join_types=[], 441 | relative_dates=True, 442 | save_as="", 443 | rounding='D', 444 | rounding_type='round', 445 | protect_duplicates=['Animal_id','Cage_id','Cage_Treatment_start_date', 'Cage_TreatmentProtocol_code'], 446 | ): 447 | """Returns an overview of events per animal. 448 | 449 | Parameters 450 | ---------- 451 | 452 | db_path : string 453 | Path to the database file to query. 454 | 455 | outerjoin_all : bool 456 | Pased as outerjoin_all to `..query.get_df()` 457 | 458 | filters : list of list, optional 459 | A list of lists giving filters for the query. It is passed to ..query.get_df(). 460 | 461 | saturate : {list of str, list of dict}, optional 462 | A list of dictionaries or strings specifying by which criteria to saturate cells. It is passed to behaviopy.timetable.multi_plot() 463 | 464 | save_df : string, optional 465 | Path under which to save the plotted dataframe. ".csv" will be appended to the string, and the data will be saved in CSV format. 466 | 467 | window_end : string 468 | A datetime-formatted string (e.g. "2016,12,18") to apply as the timetable end date (overrides autodetected end). 469 | 470 | rounding_type : {'round','floor','ceil'}, optional 471 | Whether to round the dates (splits e.g. days apart at noon, hours at 30 minutes, etc.) or to take the floor or the ceiling. 472 | """ 473 | 474 | df = selection.timetable(db_path, filters, default_join, join_types=join_types) 475 | animals = list(df["Animal_id"].unique()) 476 | cagestays = selection.cage_periods(db_path, animal_filter=animals) 477 | df = concurrent_cagetreatment(df, cagestays, protect_duplicates=protect_duplicates) 478 | 479 | if relative_dates: 480 | if isinstance(relative_dates, dict): 481 | df['reference_date'] = '' 482 | df['reference_date'] = df['reference_date'].astype('datetime64[ns]') 483 | reference_column = list(relative_dates.keys())[0] 484 | matching_column = list(list(relative_dates.values())[0].keys())[0] 485 | reference_matching = list(list(relative_dates.values())[0].values())[0] 486 | for subject in df['Animal_id'].unique(): 487 | reference_date = df[(df['Animal_id']==subject)&(df[matching_column]==reference_matching)][reference_column].values[0] 488 | df.loc[df['Animal_id']==subject,'reference_date'] = reference_date 489 | elif isinstance(relative_dates, bool): 490 | df['reference_date'] = df['Cage_Treatment_start_date'] 491 | elif "," in relative_dates or "-" in relative_dates: 492 | print('WARNING: The `relative_dates` value provided could be interpreted as a date. This feature is however not yet supported. Dates will be made relative to "Cage_Treatment_start_date" instead!') 493 | df['reference_date'] = df['Cage_Treatment_start_date'] 494 | else: 495 | df['reference_date'] = df[relative_dates] 496 | df = relativize_dates(df, 497 | rounding=rounding, 498 | rounding_type=rounding_type, 499 | reference_date=False, 500 | ) 501 | 502 | if save_as: 503 | save_as = os.path.abspath(os.path.expanduser(save_as)) 504 | if not(save_as.endswith(".csv") or save_as.endswith(".CSV")): 505 | save_as += ".csv" 506 | df.to_csv(save_as) 507 | 508 | return df 509 | -------------------------------------------------------------------------------- /labbookdb/report/selection.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from labbookdb.db import query 3 | from labbookdb.report.utilities import concurrent_cagetreatment 4 | 5 | def animal_id(db_path, database, identifier, reverse=False): 6 | """Return the main LabbookDB animal identifier given an external database identifier. 7 | 8 | Parameters 9 | ---------- 10 | 11 | db_path : string 12 | Path to the database file to query. 13 | database : string 14 | Valid `AnimalExternalIdentifier.database` value. 15 | identifier : string 16 | Valid `AnimalExternalIdentifier.identifier` value. 17 | reverse : bool, optional 18 | Whether to reverse the query. 19 | A reverse query means that a LabbookDB `Animal.id` filter is applied and an `AnimalExternalIdentifier.identifier` value is returned. 20 | 21 | Returns 22 | ------- 23 | int 24 | LabbookDB animal identifier. 25 | """ 26 | 27 | col_entries=[ 28 | ("Animal","id"), 29 | ("AnimalExternalIdentifier",), 30 | ] 31 | join_entries=[ 32 | ("Animal.external_ids",), 33 | ] 34 | 35 | my_filters = [] 36 | if reverse: 37 | my_filter = ["Animal","id",identifier] 38 | my_filters.append(my_filter) 39 | else: 40 | my_filter = ["AnimalExternalIdentifier","identifier",identifier] 41 | my_filters.append(my_filter) 42 | my_filter = ["AnimalExternalIdentifier","database",database] 43 | my_filters.append(my_filter) 44 | 45 | df = query.get_df(db_path,col_entries=col_entries, join_entries=join_entries, filters=my_filters) 46 | try: 47 | if reverse: 48 | labbookdb_id = df['AnimalExternalIdentifier_identifier'].item() 49 | else: 50 | labbookdb_id = df['Animal_id'].item() 51 | except ValueError: 52 | labbookdb_id = 'FailedIDQuery' 53 | print('This may be happening because the identifier query value you have provided is of the wrong type (LabbookDB lookups are type-sensitive).') 54 | 55 | return labbookdb_id 56 | 57 | def stimulation_protocol(db_path, code): 58 | """Select a `pandas.DataFrame`object containing all events and associated measurement units for a specific stimulation protocol. 59 | 60 | Parameters 61 | ---------- 62 | 63 | db_path : string 64 | Path to the database file to query. 65 | 66 | code : string 67 | Code (valid `StimulationProtocol.code` value) which identifies the stimulation protocol to format. 68 | """ 69 | 70 | col_entries=[ 71 | ("StimulationProtocol",), 72 | ("StimulationEvent",), 73 | ("MeasurementUnit",), 74 | ] 75 | join_entries=[ 76 | ("StimulationProtocol.events",), 77 | ("StimulationEvent.unit",), 78 | ] 79 | my_filters=[] 80 | my_filter = ["StimulationProtocol","code",code] 81 | my_filters.append(my_filter) 82 | 83 | df = query.get_df(db_path,col_entries=col_entries, join_entries=join_entries, filters=my_filters) 84 | 85 | return df 86 | 87 | def animals_by_cage_treatment(db_path, 88 | codes=[], 89 | end_dates=[], 90 | start_dates=[], 91 | ): 92 | """Select a dataframe of animals and all related tables through to cage treatments based on cage_treatment filters. 93 | 94 | Parameters 95 | ---------- 96 | 97 | db_path : string 98 | Path to a LabbookDB formatted database. 99 | 100 | codes : list, optional 101 | Related TreatmentProtocol.code values based on which to filter dataframe 102 | 103 | end_dates : list, optional 104 | Related Treatment.end_date values based on which to filter dataframe 105 | 106 | start_dates : list, optional 107 | Related Treatment.start_date values based on which to filter dataframe 108 | """ 109 | 110 | col_entries=[ 111 | ("Animal","id"), 112 | ("Cage","id"), 113 | ("Treatment",), 114 | ("TreatmentProtocol","code"), 115 | ] 116 | join_entries=[ 117 | ("Animal.cage_stays",), 118 | ("CageStay.cage",), 119 | ("Cage.treatments",), 120 | ("Treatment.protocol",), 121 | ] 122 | my_filters=[] 123 | if codes: 124 | my_filter = ["TreatmentProtocol","code"] 125 | my_filter.extend(codes) 126 | my_filters.append(my_filter) 127 | if end_dates: 128 | my_filter = ["Treatment","end_date"] 129 | my_filter.extend(end_dates) 130 | my_filters.append(my_filter) 131 | if start_dates: 132 | my_filter = ["Treatment","start_date"] 133 | my_filter.extend(start_dates) 134 | my_filters.append(my_filter) 135 | if not my_filters: 136 | my_filters=[None] 137 | 138 | df = query.get_df(db_path,col_entries=col_entries, join_entries=join_entries, filters=my_filters) 139 | 140 | return df 141 | 142 | def animals_by_treatment(db_path, 143 | codes=[], 144 | end_dates=[], 145 | start_dates=[], 146 | ): 147 | """Select a dataframe of animals and all related tables through to treatments based on treatment filters. 148 | 149 | Parameters 150 | ---------- 151 | 152 | db_path : string 153 | Path to a LabbookDB formatted database. 154 | 155 | codes : list, optional 156 | Related TreatmentProtocol.code values based on which to filter dataframe 157 | 158 | end_dates : list, optional 159 | Related Treatment.end_date values based on which to filter dataframe 160 | 161 | start_dates : list, optional 162 | Related Treatment.start_date values based on which to filter dataframe 163 | """ 164 | 165 | col_entries=[ 166 | ("Animal","id"), 167 | ("Treatment",), 168 | ("TreatmentProtocol","code"), 169 | ] 170 | join_entries=[ 171 | ("Animal.treatments",), 172 | ("Treatment.protocol",), 173 | ] 174 | my_filters=[] 175 | if codes: 176 | my_filter = ["TreatmentProtocol","code"] 177 | my_filter.extend(codes) 178 | my_filters.append(my_filter) 179 | if end_dates: 180 | my_filter = ["Treatment","end_date"] 181 | my_filter.extend(end_dates) 182 | my_filters.append(my_filter) 183 | if start_dates: 184 | my_filter = ["Treatment","start_date"] 185 | my_filter.extend(start_dates) 186 | my_filters.append(my_filter) 187 | if not my_filters: 188 | my_filters=[None] 189 | 190 | df = query.get_df(db_path,col_entries=col_entries, join_entries=join_entries, filters=my_filters) 191 | 192 | return df 193 | 194 | def animal_operations(db_path, 195 | animal_ids=[], 196 | implant_targets=[], 197 | virus_targets=[], 198 | ): 199 | """Select a dataframe of animals having been subjected to operative interventions with the given anatomical targets. 200 | 201 | Parameters 202 | ---------- 203 | 204 | db_path : str 205 | Path to a LabbookDB formatted database. 206 | 207 | animal_ids : list, optional 208 | A List of LabbookDB `Animal.id` values by which to filter the query. 209 | It is faster to filter using this mechanism than to return a dataframe for all animals and then filter that. 210 | 211 | implant_targets : list, optional 212 | A List of LabbookDB `OrthogonalStereotacticTarget.code` values which should be used to filter the query, while being joined to `Operation` objects via the `OpticFiberImplantProtocol` class. 213 | It is faster to filter using this mechanism than to return a dataframe for all animals and then filter that. 214 | 215 | virus_targets : list, optional 216 | A List of LabbookDB `OrthogonalStereotacticTarget.code` values which should be used to filter the query, while being joined to `Operation` objects via the `VirusInjectionProtocol` class. 217 | It is faster to filter using this mechanism than to return a dataframe for all animals and then filter that. 218 | 219 | Notes 220 | ----- 221 | 222 | CAREFUL: Providing both `implant_targets` and `virus_targets` will return entries including only animals which have undergone an operation which has included protocols targeting both areas. 223 | If the areas were targeted by protocols included in different operations, the correspondence will not be detected. 224 | To obtain such a list please call the function twice and create a new dataframe from the intersection of the inputs on the `Animal_id` column. 225 | """ 226 | 227 | filters = [] 228 | join_type = 'outer' 229 | col_entries=[ 230 | ("Animal","id"), 231 | ("Operation",), 232 | ("OpticFiberImplantProtocol",), 233 | ("VirusInjectionProtocol",), 234 | ("OrthogonalStereotacticTarget",), 235 | ("Virus","OrthogonalStereotacticTarget",""), 236 | ] 237 | join_entries=[ 238 | ("Animal.operations",), 239 | ("OpticFiberImplantProtocol","Operation.protocols"), 240 | ("VirusInjectionProtocol","Operation.protocols"), 241 | ("OrthogonalStereotacticTarget","OpticFiberImplantProtocol.stereotactic_target"), 242 | ("Virus_OrthogonalStereotacticTarget","VirusInjectionProtocol.stereotactic_target"), 243 | ] 244 | my_filter=[] 245 | if animal_ids: 246 | my_filter = ["Animal","id"] 247 | my_filter.extend(animal_ids) 248 | if implant_targets: 249 | my_filter = ["OrthogonalStereotacticTarget","code"] 250 | my_filter.extend(implant_targets) 251 | if virus_targets: 252 | my_filter = ["Virus_OrthogonalStereotacticTarget","code"] 253 | my_filter.extend(virus_targets) 254 | filters.append(my_filter) 255 | 256 | df = query.get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=filters, default_join=join_type) 257 | 258 | return df 259 | 260 | 261 | def animal_treatments(db_path, 262 | animal_ids=[], 263 | animal_treatments=[], 264 | cage_treatments=[], 265 | conjunctive=True, 266 | ): 267 | """Select a dataframe of animals and all treatments including animal-level or cage-level treatments. 268 | 269 | Parameters 270 | ---------- 271 | 272 | db_path : str 273 | Path to a LabbookDB formatted database. 274 | 275 | animal_ids : list, optional 276 | A List of LabbookDB `Animal.id` values by which to filter the query. 277 | It is faster to filter using this mechanism than to return a dataframe for all animals and then filter that. 278 | 279 | animal_treatments : list, optional 280 | A List of LabbookDB `Treatment.code` values which should be used to filter the query, while being joined to `Animal` objects. 281 | 282 | cage_treatments : list, optional 283 | A List of LabbookDB `Treatment.code` values which should be used to filter the query, while being joined to `Cage` objects - and further to `Animal` objects via `CageStay` objects. 284 | An onset check is also applied by the function, to ascertain that there is an overlap between the animal's presence in the cage and the cage treatment application. 285 | 286 | conjunctive : bool, optional 287 | Whether both `cage_treatments` and `animal_treatments` need to be satisfied (statements within each list are always disjunctive). 288 | 289 | Notes 290 | ----- 291 | 292 | Currently `conjunctive=False` does not work; cage treatment and animal treatment filters are always conjunctive. 293 | """ 294 | 295 | filters = [] 296 | join_type = 'outer' 297 | col_entries=[ 298 | ("Animal","id"), 299 | ("Animal","death_date"), 300 | ("Treatment",), 301 | ("TreatmentProtocol","code"), 302 | ("CageStay","start_date"), 303 | ("Cage","id"), 304 | ("Cage","Treatment",""), 305 | ("Cage","TreatmentProtocol","code"), 306 | ] 307 | join_entries=[ 308 | ("Animal.treatments",), 309 | ("Treatment.protocol",), 310 | ("Animal.cage_stays",), 311 | ("CageStay.cage",), 312 | ("Cage_Treatment","Cage.treatments"), 313 | ("Cage_TreatmentProtocol","Cage_Treatment.protocol"), 314 | ] 315 | my_filter=[] 316 | if animal_ids: 317 | my_filter = ["Animal","id"] 318 | my_filter.extend(animal_ids) 319 | if animal_treatments: 320 | my_filter = ["TreatmentProtocol","code"] 321 | my_filter.extend(animal_treatments) 322 | if cage_treatments: 323 | my_filter = ["Cage_TreatmentProtocol","code"] 324 | my_filter.extend(cage_treatments) 325 | filters.append(my_filter) 326 | 327 | df = query.get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=filters, default_join=join_type) 328 | 329 | # Generally dataframe operations should be performed in `labbookdb.report.tracking`, however, if animals are selected by cage treatment, we need to determine which animals actually received the treatment. 330 | # The following is therefore a selection issue. 331 | cage_treatment_columns = ['Cage_Treatment_id','Cage_Treatment_end_date','Cage_Treatment_start_date','Cage_TreatmentProtocol_code','Cage_Treatment_protocol_id'] 332 | animals = list(df["Animal_id"].unique()) 333 | cage_stays = cage_periods(db_path, animal_filter=animals) 334 | df = concurrent_cagetreatment(df, cage_stays) 335 | 336 | # The concurrent cage treatment function cannot delete the entire entry for animals which have only one entry in the dataframe. 337 | # This is because the row may still contain unique animal treatment information, which would otherwise be lost. 338 | # As the `concurrent_cagetreatment()` function is unaware of this context, we perform this filtering here. 339 | if cage_treatments: 340 | df = df.loc[df['Cage_TreatmentProtocol_code'].isin(cage_treatments)] 341 | 342 | return df 343 | 344 | 345 | def by_animals(db_path, select, animals): 346 | """Select a dataframe of animals and all related tables through to the "select" table based on animal filters. 347 | 348 | Parameters 349 | ---------- 350 | 351 | db_path : str 352 | Path to a LabbookDB formatted database. 353 | 354 | select : str 355 | For which kind of evaluation to select dataframe. 356 | 357 | animals : list of str 358 | Animal.id values based on which to filter dataframe 359 | """ 360 | 361 | accepted_select_values = ["sucrose preference","timetable"] 362 | 363 | if select == "sucrose preference": 364 | join_types = ["inner","inner","inner"] 365 | col_entries=[ 366 | ("Animal","id"), 367 | ("Cage","id"), 368 | ("SucrosePreferenceMeasurement",), 369 | ] 370 | join_entries=[ 371 | ("Animal.cage_stays",), 372 | ("CageStay.cage",), 373 | ("SucrosePreferenceMeasurement",), 374 | ] 375 | elif select == "timetable": 376 | col_entries=[ 377 | ("Animal","id"), 378 | ("Treatment",), 379 | ("FMRIMeasurement","date"), 380 | ("OpenFieldTestMeasurement","date"), 381 | ("ForcedSwimTestMeasurement","date"), 382 | ("TreatmentProtocol","code"), 383 | ("Cage","id"), 384 | ("Cage","Treatment",""), 385 | ("Cage","TreatmentProtocol","code"), 386 | ("SucrosePreferenceMeasurement","date"), 387 | ] 388 | join_entries=[ 389 | ("Animal.treatments",), 390 | ("FMRIMeasurement",), 391 | ("OpenFieldTestMeasurement","Animal.measurements"), 392 | ("ForcedSwimTestMeasurement","Animal.measurements"), 393 | ("Treatment.protocol",), 394 | ("Animal.cage_stays",), 395 | ("CageStay.cage",), 396 | ("Cage_Treatment","Cage.treatments"), 397 | ("Cage_TreatmentProtocol","Cage_Treatment.protocol"), 398 | ("SucrosePreferenceMeasurement","Cage.measurements"), 399 | ] 400 | else: 401 | raise ValueError("The value for select needs to be one of {}".format(accepted_select_values)) 402 | 403 | animals = [str(i) for i in animals] #for some reason the Animal.id values need to be string :-/ 404 | my_filter = ["Animal","id"] 405 | my_filter.extend(animals) 406 | 407 | df = query.get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=[my_filter], join_types=join_types) 408 | 409 | return df 410 | 411 | 412 | def cage_drinking_measurements(db_path, 413 | treatments=[] 414 | ): 415 | """""" 416 | filters = [] 417 | join_type = 'inner' 418 | col_entries=[ 419 | ("Cage","id"), 420 | ("Treatment",), 421 | ("TreatmentProtocol",'code'), 422 | ('DrinkingMeasurement',) 423 | ] 424 | join_entries=[ 425 | ("Cage.treatments",), 426 | ("Treatment.protocol",), 427 | ('DrinkingMeasurement',) 428 | ] 429 | my_filter = [] 430 | if treatments: 431 | my_filter = ["TreatmentProtocol","code"] 432 | my_filter.extend(treatments) 433 | filters.append(my_filter) 434 | 435 | df = query.get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=filters, default_join=join_type) 436 | 437 | return df 438 | 439 | 440 | def animals_by_genotype(db_path, genotypes, 441 | attribute='code', 442 | ): 443 | """Return `pandas.Dataframe` object containing ID and genotype table columns of animals as matched by selected values on a selected Genotype attribute field. 444 | 445 | Parameters 446 | ---------- 447 | db_path : string 448 | Path to database file to query. 449 | genotypes : list 450 | List of strings containing values to be matched for the selected Genotype attribute. 451 | attribute : str 452 | Genotype attribute to match on. 453 | """ 454 | filters = [] 455 | join_type = 'inner' 456 | col_entries=[ 457 | ("Animal","id"), 458 | ("Genotype",), 459 | ] 460 | join_entries=[ 461 | ("Animal.genotypes",), 462 | ] 463 | my_filter = ["Genotype", attribute] 464 | my_filter.extend(genotypes) 465 | filters.append(my_filter) 466 | 467 | df = query.get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=filters, default_join=join_type) 468 | 469 | return df 470 | 471 | 472 | def cage_periods(db_path, 473 | animal_filter=[], 474 | cage_filter=[], 475 | ): 476 | """ 477 | Return a `pandas.DataFrame` object containing the periods which animals spent in which cages. 478 | 479 | Parameters 480 | ---------- 481 | db_path : string 482 | Path to database file to query. 483 | animal_filter : list, optional 484 | A list of `Animal.id` attribute values for which to specifically filter the query. 485 | 486 | Notes 487 | ----- 488 | Operations on `pandas.DataFrame` objects should be performed in `labbookdb.report.tracking`, however, the cagestay end date is not explicitly recordes, so to select it or select animals by it, we calculate it here. 489 | """ 490 | df = parameterized(db_path, animal_filter=animal_filter, cage_filter=cage_filter, data_type='cage list') 491 | df['CageStay_end_date'] = '' 492 | for subject in df['Animal_id'].unique(): 493 | for start_date in df[df['Animal_id']==subject]['CageStay_start_date'].tolist(): 494 | possible_end_dates = df[(df['Animal_id']==subject)&(df['CageStay_start_date']>start_date)]['CageStay_start_date'].tolist() 495 | try: 496 | end_date = min(possible_end_dates) 497 | except ValueError: 498 | end_date = None 499 | if not end_date: 500 | end_date = df[df['Animal_id']==subject]['Animal_death_date'].tolist()[0] 501 | df.loc[(df['Animal_id']==subject)&(df['CageStay_start_date']==start_date),'CageStay_end_date'] = end_date 502 | return df 503 | 504 | def timetable(db_path, filters, 505 | default_join="outer", 506 | join_types=[], 507 | ): 508 | """Select a dataframe with animals as rown and all timetable-relevant events as columns. 509 | 510 | Parameters 511 | ---------- 512 | 513 | db_path : str 514 | Path to a LabbookDB formatted database. 515 | 516 | filters : list of list 517 | A list of lists giving filters for the query. It is passed to `..query.get_df()`. 518 | 519 | outerjoin_all : bool 520 | Pased as outerjoin_all to `..query.get_df()` 521 | """ 522 | 523 | col_entries=[ 524 | ("Animal","id"), 525 | ("Animal","death_date"), 526 | ("Treatment",), 527 | ("FMRIMeasurement","date"), 528 | ("OpenFieldTestMeasurement","date"), 529 | ("ForcedSwimTestMeasurement","date"), 530 | ("TreatmentProtocol","code"), 531 | ("CageStay","start_date"), 532 | ("Cage","id"), 533 | ("Cage","Treatment",""), 534 | ("Cage","TreatmentProtocol","code"), 535 | ("SucrosePreferenceMeasurement","date"), 536 | ] 537 | join_entries=[ 538 | ("Animal.treatments",), 539 | ("FMRIMeasurement",), 540 | ("OpenFieldTestMeasurement","Animal.measurements"), 541 | ("ForcedSwimTestMeasurement","Animal.measurements"), 542 | ("Treatment.protocol",), 543 | ("Animal.cage_stays",), 544 | ("CageStay.cage",), 545 | ("Cage_Treatment","Cage.treatments"), 546 | ("Cage_TreatmentProtocol","Cage_Treatment.protocol"), 547 | ("SucrosePreferenceMeasurement","Cage.measurements"), 548 | ] 549 | 550 | # setting outerjoin to true will indirectly include controls 551 | df = query.get_df(db_path, col_entries=col_entries, join_entries=join_entries, filters=filters, default_join=default_join, join_types=join_types) 552 | 553 | return df 554 | 555 | 556 | def parameterized(db_path, data_type, 557 | animal_filter=[], 558 | cage_filter=[], 559 | treatment_start_dates=[], 560 | ): 561 | """Select dataframe from a LabbookDB style database. 562 | 563 | Parameters 564 | ---------- 565 | 566 | db_path : string 567 | Path to a LabbookDB formatted database. 568 | data_type : {"animals id", "animals info", "animals measurements", "animals measurements irregularities", "cage list", "forced swim"} 569 | What type of data should be selected values can be: 570 | animal_filter : list, optional 571 | A list of animal identifiers (`Animal.id` attributes) for which to limit the query. 572 | treatment_start_dates : list, optional 573 | A list containing the treatment start date or dates by which to filter the cages for the sucrose preference measurements. 574 | Items should be strings in datetime format, e.g. "2016,4,25,19,30". 575 | """ 576 | 577 | default_join = "inner" 578 | my_filter = [] 579 | 580 | allowed_data_types = [ 581 | "animals base", 582 | "animals id", 583 | "animals info", 584 | "animals measurements", 585 | "animals measurements irregularities", 586 | "cage list", 587 | "forced swim", 588 | ] 589 | 590 | if data_type == "animals base": 591 | col_entries=[ 592 | ("Animal","id"), 593 | ("Animal","birth_date"), 594 | ("Animal","sex"), 595 | ] 596 | join_entries=[] 597 | elif data_type == "animals id": 598 | col_entries=[ 599 | ("Animal","id"), 600 | ("AnimalExternalIdentifier",), 601 | ] 602 | join_entries=[ 603 | ("Animal.external_ids",), 604 | ] 605 | elif data_type == "animals info": 606 | col_entries=[ 607 | ("Animal","death_date"), 608 | ("Animal","birth_date"), 609 | ("Animal","sex"), 610 | ("AnimalExternalIdentifier",), 611 | ("Genotype",), 612 | ] 613 | join_entries=[ 614 | ("Animal.external_ids",), 615 | ("Animal.genotypes",), 616 | ] 617 | if animal_filter: 618 | my_filter = ['Animal','id'] 619 | # for some reason this needs to be str 620 | my_filter.extend([str(i) for i in animal_filter]) 621 | elif data_type == "animals measurements": 622 | col_entries=[ 623 | ("Animal","id"), 624 | ("Measurement","id"), 625 | ("StimulationProtocol","code"), 626 | ] 627 | join_entries=[ 628 | ("Animal.measurements",), 629 | ("FMRIMeasurement.stimulations",), 630 | ] 631 | elif data_type == "animals measurements irregularities": 632 | col_entries=[ 633 | ("Animal","id"), 634 | ("Measurement","id"), 635 | ("Measurement","date"), 636 | ("Irregularity","description"), 637 | ] 638 | join_entries=[ 639 | ("Animal.measurements",), 640 | ("FMRIMeasurement.irregularities",), 641 | ] 642 | if animal_filter: 643 | my_filter = ['Animal','id'] 644 | # for some reason this needs to be str 645 | my_filter.extend([str(i) for i in animal_filter]) 646 | elif data_type == "animals weights": 647 | col_entries=[ 648 | ("Animal","id"), 649 | ("AnimalExternalIdentifier",), 650 | ("WeightMeasurement",), 651 | ] 652 | join_entries=[ 653 | ("Animal.external_ids",), 654 | ("WeightMeasurement",), 655 | ] 656 | my_filter = ["Measurement","type","weight"] 657 | elif data_type == "cage list": 658 | col_entries=[ 659 | ("Animal",), 660 | ("CageStay",), 661 | ("Cage","id"), 662 | ] 663 | join_entries=[ 664 | ("Animal.cage_stays",), 665 | ("CageStay.cage",), 666 | ] 667 | elif data_type == "forced swim": 668 | col_entries=[ 669 | ("Animal","id"), 670 | ("Cage","id"), 671 | ("Treatment",), 672 | ("TreatmentProtocol","code"), 673 | ("ForcedSwimTestMeasurement",), 674 | ("Evaluation",), 675 | ] 676 | join_entries=[ 677 | ("Animal.cage_stays",), 678 | ("ForcedSwimTestMeasurement",), 679 | ("Evaluation",), 680 | ("CageStay.cage",), 681 | ("Cage.treatments",), 682 | ("Treatment.protocol",), 683 | ] 684 | else: 685 | raise ValueError("The `data_type` value needs to be one of: {}. You specified \"{}\"".format(", ".join(allowed_data_types), data_type)) 686 | 687 | if animal_filter: 688 | my_filter = ['Animal','id'] 689 | # for some reason this needs to be str 690 | my_filter.extend([str(i) for i in animal_filter]) 691 | if cage_filter: 692 | my_filter = ['Cage','id'] 693 | # for some reason this needs to be str 694 | my_filter.extend([str(i) for i in cage_filter]) 695 | if treatment_start_dates: 696 | my_filter = ["Treatment","start_date"] 697 | my_filter.extend(treatment_start_dates) 698 | df = query.get_df(db_path,col_entries=col_entries, join_entries=join_entries, filters=[my_filter], default_join=default_join) 699 | return df 700 | -------------------------------------------------------------------------------- /labbookdb/db/common_classes.py: -------------------------------------------------------------------------------- 1 | from .base_classes import * 2 | from .utils import * 3 | 4 | cage_stay_association = Table('cage_stay_associations', Base.metadata, 5 | Column('cage_stays_id', Integer, ForeignKey('cage_stays.id')), 6 | Column('animals_id', Integer, ForeignKey('animals.id')) 7 | ) 8 | genotype_association = Table('genotype_associations', Base.metadata, 9 | Column('genotypes_id', Integer, ForeignKey('genotypes.id')), 10 | Column('animals_id', Integer, ForeignKey('animals.id')) 11 | ) 12 | ingredients_association = Table('ingredients_associations', Base.metadata, 13 | Column('solutions_id', Integer, ForeignKey('solutions.id')), 14 | Column('ingredients_id', Integer, ForeignKey('ingredients.id')) 15 | ) 16 | stimulation_events_association = Table('stimulation_events_associations', Base.metadata, 17 | Column('stimulation_events_id', Integer, ForeignKey('stimulation_events.id')), 18 | Column('stimulation_protocols_id', Integer, ForeignKey('stimulation_protocols.id')) 19 | ) 20 | stimulations_association = Table('stimulations_associations', Base.metadata, 21 | Column('fmri_measurements_id', Integer, ForeignKey('fmri_measurements.id')), 22 | Column('stimulation_protocols_id', Integer, ForeignKey('stimulation_protocols.id')) 23 | ) 24 | anesthesia_association = Table('anesthesia_associations', Base.metadata, 25 | Column('anesthesia_protocols_id', Integer, ForeignKey('anesthesia_protocols.id')), 26 | Column('treatment_protocols_id', Integer, ForeignKey('treatment_protocols.id')) 27 | ) 28 | operation_association = Table('operation_associations', Base.metadata, 29 | Column('operations_id', Integer, ForeignKey('operations.id')), 30 | Column('protocols_id', Integer, ForeignKey('protocols.id')) 31 | ) 32 | oprations_irregularities_association = Table('oprations_irregularities_association', Base.metadata, 33 | Column('operations_id', Integer, ForeignKey('operations.id')), 34 | Column('irregularities_id', Integer, ForeignKey('irregularities.id')) 35 | ) 36 | treatment_animal_association = Table('treatment_animal_associations', Base.metadata, 37 | Column('treatments_id', Integer, ForeignKey('treatments.id')), 38 | Column('animals_id', Integer, ForeignKey('animals.id')) 39 | ) 40 | treatment_cage_association = Table('treatment_cage_associations', Base.metadata, 41 | Column('treatments_id', Integer, ForeignKey('treatments.id')), 42 | Column('cages_id', Integer, ForeignKey('cages.id')) 43 | ) 44 | 45 | 46 | #general classes: 47 | 48 | class AnimalExternalIdentifier(Base): 49 | __tablename__ = "animal_external_identifiers" 50 | id = Column(Integer, primary_key=True) 51 | database = Column(String) 52 | identifier = Column(String) 53 | animal_id = Column(Integer, ForeignKey('animals.id')) 54 | 55 | class Evaluation(Base): 56 | __tablename__ = "evaluations" 57 | id = Column(Integer, primary_key=True) 58 | 59 | path = Column(String) #path to file contining the data from evaluation 60 | author_id = Column(Integer, ForeignKey('operators.id')) 61 | author = relationship("Operator") 62 | 63 | measurement_id = Column(Integer, ForeignKey('measurements.id')) 64 | 65 | class StimulationProtocol(Base): 66 | __tablename__ = "stimulation_protocols" 67 | id = Column(Integer, primary_key=True) 68 | code = Column(String, unique=True) 69 | name = Column(String, unique=True) 70 | #tme values specified in seconds, frequencies in hertz 71 | events = relationship("StimulationEvent", secondary=stimulation_events_association) 72 | 73 | class Substance(Base): 74 | __tablename__ = "substances" 75 | id = Column(Integer, primary_key=True) 76 | code = Column(String, unique=True) 77 | name = Column(String, unique=True) 78 | long_name = Column(String, unique=True) 79 | concentration = Column(Float) 80 | concentration_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 81 | concentration_unit = relationship("MeasurementUnit") 82 | supplier = Column(String) 83 | supplier_product_code = Column(String) 84 | pubchem_sid = Column(String) 85 | 86 | class Ingredient(Base): 87 | __tablename__ = "ingredients" 88 | id = Column(Integer, primary_key=True) 89 | concentration = Column(Float) 90 | concentration_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 91 | concentration_unit = relationship("MeasurementUnit") 92 | substance_id = Column(Integer, ForeignKey('substances.id')) 93 | substance = relationship("Substance") 94 | 95 | class Solution(Base): 96 | __tablename__ = "solutions" 97 | id = Column(Integer, primary_key=True) 98 | code = Column(String, unique=True) 99 | name = Column(String, unique=True) 100 | supplier = Column(String) 101 | supplier_product_code = Column(String) 102 | contains = relationship("Ingredient", secondary=ingredients_association) 103 | 104 | def __repr__(self): 105 | return ""\ 106 | % (self.id, self.code, self.name, [str(self.contains[i].concentration)+" "+str(self.contains[i].concentration_unit.code)+" "+str(self.contains[i].substance.name) for i in range(len(self.contains))]) 107 | 108 | 109 | #behavioural classes: 110 | 111 | class Arena(Base): 112 | __tablename__ = "arenas" 113 | id = Column(Integer, primary_key=True) 114 | code = Column(String, unique=True) 115 | name = Column(String, unique=True) 116 | shape = Column(String) # e.g. "square" or "round" 117 | x_dim = Column(Float) # in mm 118 | y_dim = Column(Float) # in mm 119 | z_dim = Column(Float) # in mm 120 | wall_color = Column(String) 121 | 122 | measurements = relationship("OpenFieldTestMeasurement") 123 | 124 | class ForcedSwimTestMeasurement(Measurement): 125 | __tablename__ = 'forcedswimtest_measurements' 126 | __mapper_args__ = {'polymorphic_identity': 'forcedswimtest'} 127 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 128 | temperature = Column(Float) #in degrees Centigrade 129 | data_path = Column(String) #path to the recording file 130 | evaluations = relationship("Evaluation") 131 | 132 | # Bracket of recording file representing this measurement: 133 | # Format: "x_start-x_end,y_start-y_end" 134 | # Values should be formatted as integers, and represent percent of the recording with and height 135 | recording_bracket = Column(String) 136 | 137 | class OpenFieldTestMeasurement(Measurement): 138 | __tablename__ = 'openfieldtest_measurements' 139 | __mapper_args__ = {'polymorphic_identity': 'openfieldtest'} 140 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 141 | center_luminostiy = Column(Integer) #in lux 142 | edge_luminostiy = Column(Integer) #in lux 143 | corner_luminostiy = Column(Integer) #in lux (only if `arena_shape == "square"`) 144 | data_path = Column(String) #path to the recording file 145 | 146 | arena_id = Column(Integer, ForeignKey('arenas.id')) 147 | evaluations = relationship("Evaluation") 148 | 149 | class FMRIMeasurement(Measurement): 150 | __tablename__ = 'fmri_measurements' 151 | __mapper_args__ = {'polymorphic_identity': 'fmri'} 152 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 153 | temperature = Column(Float) 154 | anesthesia_id = Column(Integer, ForeignKey('anesthesia_protocols.id')) 155 | anesthesia = relationship("AnesthesiaProtocol") 156 | scanner_setup_id = Column(Integer, ForeignKey('fmri_scanner_setups.id')) 157 | scanner_setup = relationship("FMRIScannerSetup") 158 | data_path = Column(String) #path to the recording file 159 | 160 | stimulations = relationship("StimulationProtocol", secondary=stimulations_association) 161 | 162 | def __str__(self): 163 | template = "fMRI({date}" 164 | if self.temperature: 165 | template += ': temp: {temp}' 166 | if self.stimulations: 167 | template +='; stim: {stimulations}' 168 | if any(["measurement aborted" in self.irregularities[i].description for i in range(len(self.irregularities))]): 169 | template += "; ABORTED" 170 | if any(["failed to indicate response to stimulus" in self.irregularities[i].description for i in range(len(self.irregularities))]): 171 | template += "; NONRESPONDENT" 172 | template += ")" 173 | return template.format( 174 | date=dt_format(self.date), 175 | temp=self.temperature, 176 | stimulations=", ".join([i.code for i in self.stimulations]), 177 | ) 178 | 179 | class AnesthesiaProtocol(Protocol): 180 | __tablename__ = 'anesthesia_protocols' 181 | __mapper_args__ = {'polymorphic_identity': 'anesthesia'} 182 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 183 | 184 | bolus_to_maintenance_delay = Column(Float) #delay from start of bolus delivery to start of maintenance delivery, in seconds 185 | respiration = Column(String) 186 | 187 | induction = relationship("TreatmentProtocol", secondary=anesthesia_association) 188 | bolus = relationship("TreatmentProtocol", secondary=anesthesia_association) 189 | maintenance = relationship("TreatmentProtocol", secondary=anesthesia_association) 190 | recovery_bolus = relationship("TreatmentProtocol", secondary=anesthesia_association) 191 | 192 | class VirusInjectionProtocol(Protocol): 193 | __tablename__ = 'virus_injection_protocols' 194 | __mapper_args__ = {'polymorphic_identity': 'virus_injection'} 195 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 196 | amount = Column(Float) # injected virus amount, in microlitres 197 | virus_diffusion_time = Column(Float) # time to wait for virus to diffuse before retracting injector - in minutes 198 | virus_injection_speed = Column(Float) # speed at which the virus is injected - in nanolitres per minute 199 | 200 | stereotactic_target_id = Column(Integer, ForeignKey('orthogonal_stereotactic_targets.id')) 201 | stereotactic_target = relationship("OrthogonalStereotacticTarget", foreign_keys=[stereotactic_target_id]) 202 | virus_id = Column(Integer, ForeignKey('viruses.id')) 203 | virus = relationship("Virus", foreign_keys=[virus_id]) 204 | 205 | class OpticFiberImplantProtocol(Protocol): 206 | __tablename__ = 'optic_fiber_implant_protocols' 207 | __mapper_args__ = {'polymorphic_identity': 'optic_fiber_implant'} 208 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 209 | 210 | stereotactic_target_id = Column(Integer, ForeignKey('orthogonal_stereotactic_targets.id')) 211 | stereotactic_target = relationship("OrthogonalStereotacticTarget", foreign_keys=[stereotactic_target_id]) 212 | optic_fiber_implant_id = Column(Integer, ForeignKey('implants.id')) 213 | optic_fiber_implant = relationship("OpticFiberImplant", foreign_keys=[optic_fiber_implant_id]) 214 | 215 | 216 | #treatment classes: 217 | 218 | class HandlingHabituation(Base): 219 | __tablename__ = "handling_habituations" 220 | id = Column(Integer, primary_key=True) 221 | date = Column(DateTime) 222 | 223 | cage_id = Column(Integer, ForeignKey('cages.id')) 224 | cage = relationship("Cage", back_populates="handling_habituations") 225 | 226 | protocol_id = Column(Integer, ForeignKey('handling_habituation_protocols.id')) 227 | protocol = relationship("HandlingHabituationProtocol") 228 | 229 | class HandlingHabituationProtocol(Protocol): 230 | __tablename__ = 'handling_habituation_protocols' 231 | __mapper_args__ = {'polymorphic_identity': 'handling_habituation'} 232 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 233 | session_duration = Column(Integer) #handling duration, per animal and per cage (assumed to be equal) in MINUTES 234 | individual_picking_up = Column(Boolean) 235 | group_picking_up = Column(Boolean) 236 | transparent_tube = Column(Boolean) 237 | 238 | class Observation(Base): 239 | __tablename__ = "observations" 240 | id = Column(Integer, primary_key=True) 241 | date = Column(DateTime) 242 | behaviour = Column(String) 243 | physiology = Column(String) 244 | severtity = Column(Integer, default=0) 245 | value = Column(Float) 246 | 247 | animal_id = Column(Integer, ForeignKey('animals.id')) 248 | unit_id = Column(Integer, ForeignKey('measurement_units.id')) 249 | unit = relationship("MeasurementUnit") 250 | operator_id = Column(Integer, ForeignKey('operators.id')) 251 | operator = relationship("Operator") 252 | def __str__(self): 253 | if self.physiology: 254 | out = "{date}: {physiology}"\ 255 | .format(date=dt_format(self.date), physiology=self.physiology) 256 | if self.behaviour: 257 | out = "{date}: {behaviour}"\ 258 | .format(date=dt_format(self.date), behaviour=self.behaviour) 259 | return out 260 | 261 | class TreatmentProtocol(Protocol): 262 | __tablename__ = 'treatment_protocols' 263 | __mapper_args__ = {'polymorphic_identity': 'treatment'} 264 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 265 | frequency = Column(String) 266 | route = Column(String) 267 | rate = Column(Float) 268 | rate_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 269 | rate_unit = relationship("MeasurementUnit", foreign_keys=[rate_unit_id]) 270 | dose = Column(Float) 271 | dose_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 272 | dose_unit = relationship("MeasurementUnit", foreign_keys=[dose_unit_id]) 273 | solution_id = Column(Integer, ForeignKey('solutions.id')) 274 | solution = relationship("Solution") 275 | 276 | class DrinkingMeasurement(Measurement): 277 | __tablename__ = 'drinking_measurements' 278 | __mapper_args__ = {'polymorphic_identity': 'drinking'} 279 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 280 | reference_date = Column(DateTime) 281 | #water consumption, in ml: 282 | consumption = Column(Float) 283 | #volumes in water source, in ml: 284 | start_amount = Column(Float) 285 | end_amount = Column(Float) 286 | 287 | class SucrosePreferenceMeasurement(Measurement): 288 | __tablename__ = 'sucrosepreference_measurements' 289 | __mapper_args__ = {'polymorphic_identity': 'sucrosepreference'} 290 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 291 | reference_date = Column(DateTime) 292 | #volumes in water source, in ml: 293 | water_start_amount = Column(Float) 294 | water_end_amount = Column(Float) 295 | sucrose_start_amount = Column(Float) 296 | sucrose_end_amount = Column(Float) 297 | sucrose_bottle_position = Column(String) 298 | sucrose_concentration = Column(Float) # in percent 299 | concentration_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 300 | concentration_unit = relationship("MeasurementUnit") 301 | 302 | class Treatment(Base): 303 | __tablename__ = "treatments" 304 | id = Column(Integer, primary_key=True) 305 | start_date = Column(DateTime) #date of first occurence 306 | end_date = Column(DateTime) #date of last occurence 307 | protocol_id = Column(Integer, ForeignKey('protocols.id')) 308 | protocol = relationship('Protocol') 309 | 310 | def __str__(self): 311 | if self.end_date: 312 | out = "{start_date} - {end_date}: {protocol_name}"\ 313 | .format(start_date=dt_format(self.start_date), end_date=dt_format(self.end_date), protocol_name=self.protocol.name) 314 | else: 315 | out = "{start_date}: {protocol_name}"\ 316 | .format(start_date=dt_format(self.start_date), protocol_name=self.protocol.name) 317 | return out 318 | 319 | #operation classes: 320 | 321 | class Operation(Base): 322 | __tablename__ = "operations" 323 | id = Column(Integer, primary_key=True) 324 | date = Column(DateTime) 325 | 326 | animal_id = Column(Integer, ForeignKey('animals.id')) 327 | 328 | irregularities = relationship("Irregularity", secondary=operations_irregularities_association) 329 | operator_id = Column(Integer, ForeignKey('operators.id')) 330 | operator = relationship("Operator") 331 | anesthesia_id = Column(Integer, ForeignKey('anesthesia_protocols.id')) 332 | anesthesia = relationship("AnesthesiaProtocol", foreign_keys=[anesthesia_id]) 333 | protocols = relationship("Protocol", secondary=operation_association) 334 | irregularities = relationship("Irregularity", secondary=oprations_irregularities_association) 335 | 336 | def __str__(self): 337 | out = "{date}: {protocols}"\ 338 | .format(date=dt_format(self.date), protocols=", ".join([i.type for i in self.protocols])) 339 | return out 340 | 341 | #animal classes: 342 | 343 | class Animal(Base): 344 | __tablename__ = "animals" 345 | id = Column(Integer, primary_key=True) 346 | birth_date = Column(DateTime) 347 | death_date = Column(DateTime) 348 | death_reason = Column(String) 349 | ear_punches = Column(String) 350 | license = Column(String) 351 | maximal_severtity = Column(Integer, default=0) 352 | sex = Column(String) 353 | 354 | cage_stays = relationship("CageStay", secondary=cage_stay_association, backref="animals") 355 | 356 | external_ids = relationship("AnimalExternalIdentifier") 357 | measurements = relationship("Measurement") 358 | 359 | genotypes = relationship("Genotype", secondary=genotype_association, backref="animals") 360 | treatments = relationship("Treatment", secondary=treatment_animal_association, backref="animals") 361 | 362 | observations = relationship("Observation") 363 | operations = relationship("Operation") 364 | 365 | biopsies = relationship("Biopsy", backref="animal") 366 | 367 | def __repr__(self): 368 | return ""\ 369 | % (self.id, [self.genotypes[i].construct+" "+self.genotypes[i].zygosity for i in range(len(self.genotypes))], self.sex, self.ear_punches,[self.treatments[i].protocol.solution for i in range(len(self.treatments))]) 370 | def __str__(self): 371 | out = "Animal(id: {id}, sex: {sex}, ear_punches: {ep}):\n"\ 372 | "\tlicense:\t{license}\n"\ 373 | "\tbirth:\t{bd}\n"\ 374 | "\tdeath:\t{dd}\t(death_reason: {dr})\n"\ 375 | "\texternal_ids:\t{eids}\n"\ 376 | "\tgenotypes:\t{genotypes}\n"\ 377 | "\tcage_stays:\t{cage_stays}\n"\ 378 | "\tobservations:\t{observations}\n"\ 379 | "\toperations:\t{operations}\n"\ 380 | "\ttreatments:\t{treatments}\n"\ 381 | "\tmeasurements:\t{measurements}\n"\ 382 | .format(id=self.id, sex=self.sex, ep=self.ear_punches, 383 | license=self.license, 384 | bd=dt_format(self.birth_date), 385 | dd=dt_format(self.death_date), dr=self.death_reason, 386 | eids=", ".join([self.external_ids[i].identifier+"("+self.external_ids[i].database+")" for i in range(len(self.external_ids))]), 387 | genotypes=", ".join([self.genotypes[i].construct+"("+self.genotypes[i].zygosity+")" for i in range(len(self.genotypes))]), 388 | operations="\n\t\t\t".join(arange_by_date(self.operations)), 389 | observations="\n\t\t\t".join(arange_by_date(self.observations)), 390 | treatments="\n\t\t\t".join(arange_by_date(self.treatments)), 391 | cage_stays="\n\t\t\t".join([self.cage_stays[i].__str__() for i in range(len(self.cage_stays))]), 392 | measurements="\n\t\t\t".join(arange_by_date(self.measurements)), 393 | ) 394 | return out 395 | 396 | class CageStay(Base): 397 | __tablename__ = "cage_stays" 398 | id = Column(Integer, primary_key=True) 399 | 400 | start_date = Column(DateTime) #date of first occurence 401 | 402 | cage_id = Column(Integer, ForeignKey('cages.id')) 403 | cage = relationship("Cage", back_populates="stays") 404 | 405 | single_caged = Column(String) #if singel caged, state reason 406 | 407 | def __str__(self): 408 | return "cage {cage_id}, starting {start_date}"\ 409 | .format(cage_id=self.cage_id, start_date=dt_format(self.start_date)) 410 | def report_animals(self): 411 | return ["Animal "+str(i.id)+"["+", ".join([j.identifier+"("+j.database+")" for j in i.external_ids])+"] starting "+dt_format(self.start_date) for i in self.animals] 412 | 413 | class Cage(Base): 414 | __tablename__ = "cages" 415 | id = Column(Integer, primary_key=True) 416 | 417 | handling_habituations = relationship("HandlingHabituation", back_populates="cage") 418 | treatments = relationship("Treatment", secondary=treatment_cage_association, backref="cages") 419 | measurements = relationship("Measurement") 420 | id_local = Column(String, unique=True) 421 | location = Column(String) 422 | environmental_enrichment = Column(String) 423 | stays = relationship("CageStay", back_populates="cage") 424 | 425 | def __str__(self): 426 | if self.id_local: 427 | idl = self.id_local 428 | else: 429 | idl = self.id 430 | return "Cage(id: {id}, location: {loc}, id_local: {idl}):\n"\ 431 | "\t{stays}"\ 432 | .format(id=self.id, idl=idl, loc=self.location, stays="\n\t".join(["\n\t".join(i.report_animals()) for i in self.stays])) 433 | 434 | class WeightMeasurement(Measurement): 435 | __tablename__ = 'weight_measurements' 436 | __mapper_args__ = {'polymorphic_identity': 'weight'} 437 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 438 | weight = Column(Float) 439 | weight_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 440 | weight_unit = relationship("MeasurementUnit", foreign_keys=[weight_unit_id]) 441 | 442 | def __str__(self): 443 | return "Weight({date}, weight: {w}{wu})"\ 444 | .format(date=self.date, w=self.weight, wu=self.weight_unit.code) 445 | 446 | class BrainBiopsy(Biopsy): 447 | __tablename__ = "brain_biopsies" 448 | __mapper_args__ = {'polymorphic_identity': 'brain'} 449 | id = Column(Integer, ForeignKey('biopsies.id'), primary_key=True) 450 | sectioning_protocol_id = Column(Integer, ForeignKey('protocols.id')) 451 | sectioning_protocol = relationship("Protocol", foreign_keys=[sectioning_protocol_id]) 452 | data_path = Column(String) # path to corresponding microscopy data 453 | 454 | # DNA classes: 455 | 456 | class Incubation(Base): 457 | __tablename__ = "incubations" 458 | id = Column(Integer, primary_key=True) 459 | revolutions_per_minute = Column(Float) 460 | duration = Column(Float) 461 | temperature = Column(Float) 462 | movement = Column(String) # "centrifuge" or "shake" 463 | 464 | duration_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 465 | duration_unit = relationship("MeasurementUnit", foreign_keys=[duration_unit_id]) 466 | #temperature - usually in degrees Centigrade 467 | temperature_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 468 | temperature_unit = relationship("MeasurementUnit", foreign_keys=[temperature_unit_id]) 469 | 470 | class DNAExtraction(Base): 471 | __tablename__ = "dna_extractions" 472 | id = Column(Integer, primary_key=True) 473 | code = Column(String, unique=True) 474 | protocol_id = Column(Integer, ForeignKey('dna_extraction_protocols.id')) 475 | protocol = relationship('DNAExtractionProtocol') 476 | source_id = Column(Integer, ForeignKey('biopsies.id')) 477 | source = relationship('Biopsy') 478 | 479 | class DNAExtractionProtocol(Protocol): 480 | __tablename__ = 'dna_extraction_protocols' 481 | __mapper_args__ = {'polymorphic_identity': 'dna_extraction'} 482 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 483 | sample_mass = Column(Float) 484 | mass_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 485 | mass_unit = relationship("MeasurementUnit", foreign_keys=[mass_unit_id]) 486 | digestion_buffer_id = Column(Integer, ForeignKey("solutions.id")) 487 | digestion_buffer = relationship("Solution", foreign_keys=[digestion_buffer_id]) 488 | digestion_buffer_volume = Column(Float) 489 | digestion_id = Column(Integer, ForeignKey("incubations.id")) 490 | digestion = relationship("Incubation", foreign_keys=[digestion_id]) 491 | lysis_buffer_id = Column(Integer, ForeignKey("solutions.id")) 492 | lysis_buffer = relationship("Solution", foreign_keys=[lysis_buffer_id]) 493 | lysis_buffer_volume = Column(Float) 494 | lysis_id = Column(Integer, ForeignKey("incubations.id")) 495 | lysis = relationship("Incubation", foreign_keys=[lysis_id]) 496 | proteinase_id = Column(Integer, ForeignKey("solutions.id")) 497 | proteinase = relationship("Solution", foreign_keys=[proteinase_id]) 498 | proteinase_volume = Column(Float) 499 | inactivation_id = Column(Integer, ForeignKey("incubations.id")) 500 | inactivation = relationship("Incubation", foreign_keys=[inactivation_id]) 501 | cooling_id = Column(Integer, ForeignKey("incubations.id")) 502 | cooling = relationship("Incubation", foreign_keys=[cooling_id]) 503 | centrifugation_id = Column(Integer, ForeignKey("incubations.id")) 504 | centrifugation = relationship("Incubation", foreign_keys=[centrifugation_id]) 505 | 506 | volume_unit_id = Column(Integer, ForeignKey('measurement_units.id')) 507 | volume_unit = relationship("MeasurementUnit", foreign_keys=[volume_unit_id]) 508 | 509 | class BrainExtractionProtocol(Protocol): 510 | __tablename__ = 'brain_extraction_protocols' 511 | __mapper_args__ = {'polymorphic_identity': 'brain_extraction'} 512 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 513 | perfusion_system = Column(String(50)) #e.g. "pump", "peristaltic pump", "syringe" 514 | perfusion_flow = Column(Float) #in ml/min 515 | peristaltic_frequency = Column(Float) #in Hz 516 | flushing_solution_id = Column(Integer, ForeignKey("solutions.id")) 517 | flushing_solution = relationship("Solution", foreign_keys=[flushing_solution_id]) 518 | flushing_solution_volume = Column(Float) #in ml 519 | fixation_solution_id = Column(Integer, ForeignKey("solutions.id")) 520 | fixation_solution = relationship("Solution", foreign_keys=[fixation_solution_id]) 521 | fixation_solution_volume = Column(Float) #in ml 522 | storage_solution_id = Column(Integer, ForeignKey("solutions.id")) 523 | storage_solution = relationship("Solution", foreign_keys=[storage_solution_id]) 524 | storage_solution_volume = Column(Float) #in ml 525 | storage_time = Column(Float) #in days 526 | post_extraction_fixation_time = Column(Float) #in hours 527 | 528 | class SectioningProtocol(Protocol): 529 | __tablename__ = 'sectioning_protocols' 530 | __mapper_args__ = {'polymorphic_identity': 'sectioning'} 531 | id = Column(Integer, ForeignKey('protocols.id'), primary_key=True) 532 | system = Column(String(50)) #e.g. "vibratome", "cryotome", "microtome" 533 | slice_thickness = Column(Float) #in micrometres 534 | blade_frequency = Column(Float) #in Hz 535 | blade_speed = Column(Float) #in mm/s 536 | start_bregma_distance = Column(Float) #positive towards rostral, in mm 537 | start_interaural_distance = Column(Float) #positive towards rostral, in mm 538 | start_lambda_distance = Column(Float) #positive towards rostral, in mm 539 | start_midline_distance = Column(Float) #positive towards right of animal, in mm 540 | start_depth = Column(Float) #in mm 541 | 542 | 543 | # Histological Classes 544 | 545 | class FluorescentMicroscopyMeasurement(Measurement): 546 | __tablename__ = 'fluorescent_microscopy_measurements' 547 | __mapper_args__ = {'polymorphic_identity': 'fluorescent_microscopy'} 548 | id = Column(Integer, ForeignKey('measurements.id'), primary_key=True) 549 | light_source = Column(String) # e.g. "LED", "LASER" 550 | stimulation_wavelength = Column(Float) #in nm 551 | imaged_wavelength = Column(Float) #in nm 552 | exposure = Column(Float) #in s 553 | data = Column(String) #path data folder 554 | biopsy_id = Column(Integer, ForeignKey('biopsies.id')) 555 | --------------------------------------------------------------------------------