├── .Dockerfile ├── .github └── workflows │ ├── release.yaml │ └── test.yaml ├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── bin ├── larcv-config ├── merge_larcv3_files.py └── run_processor.py ├── data_loader_api.py ├── larcv ├── __init__.py ├── _version.py ├── batch_pydata.py ├── config_builder.py ├── data_generator.py ├── distributed_queue_interface.py ├── larcv_io_enums.py ├── larcv_writer.py └── queueloader.py ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── setup.py ├── src ├── CMakeLists.txt └── larcv3 │ ├── CMakeLists.txt │ ├── app │ ├── CMakeLists.txt │ ├── augment │ │ ├── CMakeLists.txt │ │ ├── GaussianBlur.cxx │ │ ├── GaussianBlur.h │ │ ├── Mirror.cxx │ │ ├── Mirror.h │ │ ├── Translate.cxx │ │ ├── Translate.h │ │ ├── augment.cxx │ │ └── augment.h │ ├── filter │ │ ├── CMakeLists.txt │ │ ├── EmptyImageFilter.cxx │ │ ├── EmptyImageFilter.h │ │ ├── EmptyTensorFilter.cxx │ │ └── EmptyTensorFilter.h │ ├── imagemod │ │ ├── BBoxFromCluster.cxx │ │ ├── BBoxFromCluster.h │ │ ├── BBoxFromParticle.cxx │ │ ├── BBoxFromParticle.h │ │ ├── CMakeLists.txt │ │ ├── DenseToSparse.cxx │ │ ├── DenseToSparse.h │ │ ├── Downsample.cxx │ │ ├── Downsample.h │ │ ├── Embed.cxx │ │ ├── Embed.h │ │ ├── Normalize.cxx │ │ ├── Normalize.h │ │ ├── SparseToDense.cxx │ │ ├── SparseToDense.h │ │ ├── TensorFromCluster.cxx │ │ ├── TensorFromCluster.h │ │ ├── Threshold.cxx │ │ ├── Threshold.h │ │ ├── imagemod.cxx │ │ └── imagemod.h │ ├── queueio │ │ ├── BatchData.cxx │ │ ├── BatchData.h │ │ ├── BatchDataQueue.cxx │ │ ├── BatchDataQueue.h │ │ ├── BatchDataQueueFactory.cxx │ │ ├── BatchDataQueueFactory.h │ │ ├── BatchFillerBBox.cxx │ │ ├── BatchFillerBBox.h │ │ ├── BatchFillerPIDLabel.cxx │ │ ├── BatchFillerPIDLabel.h │ │ ├── BatchFillerParticle.cxx │ │ ├── BatchFillerParticle.h │ │ ├── BatchFillerSparseTensor.cxx │ │ ├── BatchFillerSparseTensor.h │ │ ├── BatchFillerTemplate.cxx │ │ ├── BatchFillerTemplate.h │ │ ├── BatchFillerTensor.cxx │ │ ├── BatchFillerTensor.h │ │ ├── BatchHolder.cxx │ │ ├── BatchHolder.h │ │ ├── CMakeLists.txt │ │ ├── QueueIOTypes.cxx │ │ ├── QueueIOTypes.h │ │ ├── QueueProcessor.cxx │ │ ├── QueueProcessor.h │ │ ├── queueio.cxx │ │ └── queueio.h │ └── sbnd_imagemod │ │ ├── CMakeLists.txt │ │ ├── CosmicNeutrinoSegLabel.cxx │ │ ├── CosmicNeutrinoSegLabel.h │ │ ├── ParentParticleSeg.cxx │ │ ├── ParentParticleSeg.h │ │ ├── sbnd_imagemod.cxx │ │ └── sbnd_imagemod.h │ ├── core │ ├── CMakeLists.txt │ ├── base │ │ ├── CMakeLists.txt │ │ ├── LArCVBaseUtilFunc.cxx │ │ ├── LArCVBaseUtilFunc.h │ │ ├── LArCVTypes.h │ │ ├── Watch.cxx │ │ ├── Watch.h │ │ ├── base.cxx │ │ ├── base.h │ │ ├── larbys.cxx │ │ ├── larbys.h │ │ ├── larcv_base.cxx │ │ ├── larcv_base.h │ │ ├── larcv_logger.cxx │ │ ├── larcv_logger.h │ │ └── stack_trace.h │ ├── dataformat │ │ ├── BBox.cxx │ │ ├── BBox.h │ │ ├── CMakeLists.txt │ │ ├── DataFormatTypes.cxx │ │ ├── DataFormatTypes.h │ │ ├── DataProductFactory.cxx │ │ ├── DataProductFactory.h │ │ ├── EventBBox.cxx │ │ ├── EventBBox.h │ │ ├── EventBase.cxx │ │ ├── EventBase.h │ │ ├── EventID.cxx │ │ ├── EventID.h │ │ ├── EventParticle.cxx │ │ ├── EventParticle.h │ │ ├── EventSparseCluster.cxx │ │ ├── EventSparseCluster.h │ │ ├── EventSparseTensor.cxx │ │ ├── EventSparseTensor.h │ │ ├── EventTensor.cxx │ │ ├── EventTensor.h │ │ ├── IOManager.cxx │ │ ├── IOManager.h │ │ ├── ImageMeta.cxx │ │ ├── ImageMeta.h │ │ ├── Particle.cxx │ │ ├── Particle.h │ │ ├── Point.cxx │ │ ├── Point.h │ │ ├── Tensor.cxx │ │ ├── Tensor.h │ │ ├── Vertex.cxx │ │ ├── Vertex.h │ │ ├── Voxel.cxx │ │ ├── Voxel.h │ │ ├── dataformat.cxx │ │ ├── dataformat.h │ │ └── schema.md │ └── processor │ │ ├── CMakeLists.txt │ │ ├── ProcessBase.cxx │ │ ├── ProcessBase.h │ │ ├── ProcessDriver.cxx │ │ ├── ProcessDriver.h │ │ ├── ProcessFactory.cxx │ │ ├── ProcessFactory.h │ │ ├── ProcessorTypes.h │ │ ├── processor.cxx │ │ └── processor.h │ └── larcv.cxx └── tests ├── larcv ├── app │ └── queueio │ │ ├── test_queueio_BBox2D.py │ │ ├── test_queueio_PIDLabel.py │ │ ├── test_queueio_Particle.py │ │ ├── test_queueio_SparseTensor2D.py │ │ ├── test_queueio_SparseTensor3D.py │ │ ├── test_queueio_Tensor2D.py │ │ └── test_queueio_Tensor3D.py └── core │ ├── base │ ├── test_base.py │ ├── test_json.py │ ├── test_larbys.py │ └── test_logger.py │ ├── dataformat │ ├── sparse_tensor.py │ ├── test_bbox.py │ ├── test_dataformat.py │ ├── test_dataformat_binding.py │ ├── test_eventid.py │ ├── test_image_meta.py │ ├── test_io.py │ ├── test_point.py │ ├── test_tensor.py │ ├── test_voxel.py │ ├── test_write_bbox.py │ ├── test_write_image2d.py │ ├── test_write_particle.py │ ├── test_write_sparse_cluster.py │ ├── test_write_sparse_tensor.py │ ├── test_write_tensor.py │ └── todo.txt │ └── processor │ └── test_processor.py └── test_larcv.py /.Dockerfile: -------------------------------------------------------------------------------- 1 | #FROM deeplearnphysics/ubuntu16.04_deps 2 | FROM deeplearnphysics/larcv2:minimal 3 | 4 | MAINTAINER kterao@slac.stanford.edu 5 | 6 | # larcv build 7 | ENV LARCV_BASEDIR=/app/larcv2 8 | ENV LARCV_BUILDDIR="${LARCV_BASEDIR}/build" 9 | ENV LARCV_COREDIR="${LARCV_BASEDIR}/larcv/core" 10 | ENV LARCV_APPDIR="${LARCV_BASEDIR}/larcv/app" 11 | ENV LARCV_LIBDIR="${LARCV_BUILDDIR}/lib" 12 | ENV LARCV_INCDIR="${LARCV_BUILDDIR}/include" 13 | ENV LARCV_BINDIR="${LARCV_BUILDDIR}/bin" 14 | ENV LARCV_ROOT6=1 15 | ENV LARCV_CXX=g++ 16 | 17 | # without numpy 18 | #ENV LARCV_NUMPY=0 19 | #ENV LARCV_INCLUDES="-I${LARCV_INCDIR} " 20 | #ENV LARCV_LIBS="-L${LARCV_LIBDIR} -llarcv" 21 | 22 | # with numpy 23 | ENV LARCV_NUMPY=1 24 | ENV LARCV_INCLUDES="-I${LARCV_INCDIR} -I/usr/include/python2.7 -I/usr/include/x86_64-linux-gnu/python2.7" 25 | ENV LARCV_LIBS="-L/usr/lib/ -L/usr/lib/python2.7/config-x86_64-linux-gnu -L/usr/lib" 26 | ENV LARCV_LIBS="${LARCV_LIBS} -lpthread -ldl -lutil -lm -lpython2.7 -Xlinker -export-dynamic -Wl,-O1 -Wl,-Bsymbolic-functions" 27 | ENV LARCV_LIBS="${LARCV_LIBS} -L${LARCV_LIBDIR} -llarcv" 28 | 29 | # set bin and lib path 30 | ENV PATH=${LARCV_BASEDIR}/bin:${LARCV_BINDIR}:${PATH} 31 | ENV LD_LIBRARY_PATH=${LARCV_LIBDIR}:${LD_LIBRARY_PATH}: 32 | ENV PYTHONPATH=${LARCV_BASEDIR}/python:${PYTHONPATH} 33 | 34 | # larcv 35 | RUN mkdir -p /app && \ 36 | cd /app && \ 37 | git clone https://github.com/DeepLearnPhysics/larcv2 && \ 38 | cd larcv2 && \ 39 | mkdir -p $LARCV_BUILDDIR && \ 40 | mkdir -p $LARCV_LIBDIR && \ 41 | mkdir -p $LARCV_BINDIR && \ 42 | make -j4 43 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-20.04 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v2 14 | - name: Install dependencies 15 | run: | 16 | python -m pip install --upgrade pip 17 | pip install setuptools wheel twine pytest 18 | sudo apt-get install libhdf5-serial-dev 19 | pip install -r requirements.txt 20 | git submodule update --init 21 | # - name: Build manylinux Python wheels 22 | # uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux2014_x86_64 23 | # with: 24 | # python-versions: 'cp36-cp36m' 25 | # build-requirements: 'scikit-build numpy h5py cmake' 26 | # system-packages: 'hdf5-devel' 27 | # # pre-build-command: 'sh pre-build-script.sh' 28 | # # package-path: 'my_project' 29 | # # pip-wheel-args: '-w ./dist --no-deps' 30 | - name: Build and publish Source Dist 31 | env: 32 | TWINE_USERNAME: __token__ 33 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 34 | run: | 35 | python setup.py sdist 36 | twine upload dist/* 37 | 38 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python-version: ["3.7", "3.8", "3.9", "3.10"] 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install setuptools wheel twine pytest 23 | sudo apt-get install libhdf5-serial-dev 24 | pip install -r requirements.txt 25 | git submodule update --init 26 | - name: Build and Install 27 | run: | 28 | python setup.py build -j 1 29 | python setup.py install --user 30 | - name: Test with pytest 31 | run: | 32 | pytest -m "not distributed_test" tests/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | doc/* 2 | !doc/doxygenLArCV.script 3 | *.out 4 | *.depend 5 | *.png 6 | *.jpg 7 | *.jpeg 8 | *.pcm 9 | *.json 10 | *.db 11 | *.log 12 | *.dat 13 | *.bin 14 | *~ 15 | *.pyc 16 | .ipynb_checkpoints 17 | *.ipynb 18 | .DS_Store 19 | *.pdf 20 | *.o 21 | *.so 22 | *.h5 23 | _skbuild 24 | larcv.egg-info 25 | dist 26 | MANIFEST -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/pybind11"] 2 | path = src/pybind11 3 | url = https://github.com/pybind/pybind11.git 4 | [submodule "src/pybind11_json"] 5 | path = src/pybind11_json 6 | url = https://github.com/pybind/pybind11_json 7 | [submodule "src/json"] 8 | path = src/json 9 | url = https://github.com/nlohmann/json.git 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 DeepLearnPhysics 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Tests](https://github.com/DeepLearnPhysics/larcv3/actions/workflows/test.yaml/badge.svg)](https://github.com/DeepLearnPhysics/larcv3/actions/workflows/test.yaml) [![license](https://img.shields.io/github/license/mashape/apistatus.svg)](https://raw.githubusercontent.com/DeepLearnPhysics/larcv2/develop/LICENSE) 2 | 3 | # LArCV (Version 3) 4 | Software framework for image(2D)/volumetric(3D) data processing with APIs to interface deep neural network open-source softwares, written in C++ with extensive Python supports. Originally developed for analyzing data from [time-projection-chamber (TPC)](https://en.wikipedia.org/wiki/Time_projection_chamber). It is now converted to be a generic tool to handle 2D-projected images and 3D-voxelized data. LArCV is particularly suitable for sparse data processing. 5 | 6 | 7 | ## Installation 8 | 9 | You can install larcv through pypi: `pip install larcv` and it should work. You can also build from source: 10 | 11 | ```bash 12 | git clone https://github.com/DeepLearnPhysics/larcv3.git 13 | cd larcv3 14 | git submodule update --init # Pulls pybind11 subpackage 15 | python setup.py build [-j 12] # Optional parallel build for faster compilation 16 | python setup.py install [--user | -prefix ${INSTALLATION_DIR} ] 17 | ``` 18 | 19 | To verify your larcv installation, after install has completed: 20 | ``` 21 | cd larcv3/tests 22 | py.test . 23 | ``` 24 | 25 | 26 | ### Dependencies 27 | 28 | * Python 29 | * Numpy 30 | * HDF5 (for IO) 31 | * cmake (for building) 32 | * scikit-build (for installation) 33 | * pytest (for continuous integration) 34 | 35 | HDF5 and cmake can all be installed by package managers. Conda will also work. 36 | 37 | For compilation, a gcc > 4.8 is required. GCC versions 5 to 8 are all known to work, as is clang on MacOS. 38 | 39 | To install requirements on ubuntu, you can do: 40 | sudo apt-get install cmake libhdf5-serial-dev python-dev 41 | pip install numpy scikit-build pytest 42 | 43 | To install requirements on mac, you can do: 44 | sudo port install cmake hdf5 45 | pip install numpy scikit-build pytest 46 | 47 | To install in a generic system, you can try conda or a virtual environment. It has been shown to work on many linux distributions. 48 | 49 | 50 | 51 | 52 | 53 | 54 | ## Compatibility 55 | 56 | larcv3 works on mac and many flavors of linux. It has never been tested on windows as far as I know. If you try to install and need help, please open an Issue. 57 | 58 | ### Use Cases 59 | 60 | Larcv is predominantly used as an IO framework and data preprocessing tool for machine learning and deep learning. It has run on many systems and in many scenarios. Larcv has a suite of test cases available that test the serialization, read back, threaded IO tools, and distributed IO tools. 61 | 62 | Larcv has run on some of the biggest systems in the world, including Summit (ORNL) and Theta (ANL). It has been used for distributed io of sparse, non-uniform data up to hundreds of CPUs/GPUs, and had good performance. 63 | 64 | If you would like to use larcv for your application and want to benchmark the performance, you are welcome to use the larcv3 open dataset (more info on deeplearnphysics.org) and if you would like help, open an issue or contact the authors directly. 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /bin/larcv-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | out="" 4 | while test $# -gt 0; do 5 | case "$1" in 6 | -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; 7 | *) optarg= ;; 8 | esac 9 | 10 | case $1 in 11 | --incdir) 12 | out="${LARCV_INCDIR} " 13 | ;; 14 | --includes) 15 | out="${LARCV_INCLUDES} " 16 | ;; 17 | --libdir) 18 | out="${LARCV_LIBDIR} " 19 | ;; 20 | --libs) 21 | out="${LARCV_LIBS}" 22 | ;; 23 | --help) 24 | ### Print a help message 25 | echo "Usage: `basename $0` [options]" 26 | echo "" 27 | echo " --includes ... print out includes" 28 | echo " --libs ... print out libraries" 29 | exit 0 30 | ;; 31 | *) 32 | ### Give an error 33 | echo "Unknown argument \"$1\"!" 1>&2 34 | echo "${usage}" 1>&2 35 | exit 1 36 | ;; 37 | esac 38 | shift 39 | done 40 | echo $out 41 | -------------------------------------------------------------------------------- /bin/run_processor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys,os,argparse 3 | import larcv 4 | import json 5 | 6 | # This script is a simplification of run_processordb but without the proddb dependence. 7 | # It incorporates command line arguments but runs stand-alone with just larcv. 8 | 9 | parser = argparse.ArgumentParser(description='LArCV ProcessDriver execution script (use proddb)') 10 | 11 | parser.add_argument('-c','--config', 12 | type=str, dest='cfg', 13 | help='string, Config file or json text',required=True) 14 | 15 | parser.add_argument('-il','--input-larcv',required=True, 16 | dest='larcv_fin',nargs='+', 17 | help='string or list, Input larcv file name[s] (Required)') 18 | 19 | parser.add_argument('-nevents','--num-events', 20 | type=int, dest='nevents', default=0, 21 | help='integer, Number of events to process') 22 | 23 | parser.add_argument('-nskip','--num-skip', 24 | type=int, dest='nskip', default=0, 25 | help='integer, Number of events to skip') 26 | 27 | parser.add_argument('-ol','--output-larcv',default='', 28 | type=str, dest='larcv_fout', 29 | help='string, Output larcv file name (optional)') 30 | 31 | parser.add_argument('-oa','--output-ana',default='', 32 | type=str, dest='ana_fout', 33 | help='string, Output analysis file name (optional)') 34 | 35 | args = parser.parse_args() 36 | 37 | 38 | if len(args.larcv_fin) == 0: 39 | print('No input files found') 40 | sys.exit(1) 41 | 42 | 43 | proc = larcv.ProcessDriver('ProcessDriver') 44 | 45 | 46 | # Read the config, if present. 47 | if "cfg" in args: 48 | # Could be a file: 49 | if os.path.exists(args.cfg): 50 | # Read it in as json: 51 | with open(args.cfg) as config_file: 52 | conf = json.load(config_file) 53 | else: 54 | # REad it directly from string: 55 | conf = joson.loads(args.cfg) 56 | 57 | proc.configure(conf) 58 | 59 | print(json.dumps(proc.config(), indent=4)) 60 | 61 | 62 | if args.larcv_fout != '': 63 | proc.override_output_file(args.larcv_fout) 64 | 65 | 66 | proc.override_input_file(args.larcv_fin) 67 | 68 | proc.initialize() 69 | 70 | proc.batch_process(args.nskip,args.nevents) 71 | 72 | print("Number of entries processed: " + str(proc.io().get_n_entries_out())) 73 | print("Output file name: " + str(proc.io().get_file_out_name())) 74 | 75 | 76 | proc.finalize() 77 | -------------------------------------------------------------------------------- /data_loader_api.py: -------------------------------------------------------------------------------- 1 | import larcv 2 | 3 | 4 | # This script is an example of what a larcv dataloader should look like. 5 | 6 | 7 | dl = larcv.dataloader() 8 | 9 | # Add several input streams. 10 | # Each input stream is from a file (or, maybe, set of files if distributed?) 11 | dl.add_input_stream(key="train", file="example_train_file.h5") 12 | dl.add_input_stream(key="test", file="example_test_file.h5") 13 | 14 | 15 | # Set target outputs for each stream. 16 | dl.add_output_stream( 17 | keys="all", # This is default, but can be configured 18 | input_format="image2d", # This needs to be a larcv3 dataformat 19 | input_label="sbndwire", # This is the larcv key under which this is stored. 20 | output_format="dense", # This can be "dense" for dense images, "sparse" for SCN, "graph" for torch_geometric 21 | framework = "numpy", # This can be "numpy", "torch", "tf"/"tensorflow" 22 | preload = True # If set, it will preload data to the GPU/accelerator 23 | ) 24 | 25 | 26 | dl.configure() # Calling this will configure the C++ code and begin loading data -------------------------------------------------------------------------------- /larcv/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . pylarcv import * 3 | 4 | import os 5 | from . _version import __version__ 6 | 7 | from . config_builder import ConfigBuilder 8 | 9 | def get_includes(): 10 | return os.path.dirname(__file__) + "/include/" 11 | 12 | def get_lib_dir(): 13 | return os.path.dirname(__file__) + "/lib/" 14 | -------------------------------------------------------------------------------- /larcv/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "3.5.1" 2 | -------------------------------------------------------------------------------- /larcv/larcv_io_enums.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | class RandomAccess(Enum): 4 | # Read all events in order, always: 5 | serial_access = 0 6 | # Read chunks of events in order: 7 | random_blocks = 1 8 | # Read every event randomly from file: 9 | random_events = 2 10 | 11 | class ReadOption(Enum): 12 | # Use only the specified root rank to read the data, 13 | # and the distribute to all other ranks: 14 | read_from_single_rank = 0 15 | 16 | # Use all ranks to read the data, no distribution is needed 17 | # This makes a copy of the file on every rank 18 | read_from_all_ranks_copy = 1 19 | 20 | # Use only one *local* rank to read the data, 21 | # and the distribute to all other ranks in subgroup 22 | # This copies the file to the local rank. 23 | read_from_single_local_rank = 2 24 | 25 | # Use every rank to read the data with MPI-IO 26 | # This is best for large files that are 27 | # challenging to duplicate to every node. 28 | read_from_all_ranks_mpi = 3 29 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | 2 | [build-system] 3 | requires = ["numpy", "scikit-build", "h5py", "cmake"] 4 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | distributed_test: only run tests with distributed IO 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | scikit-build 2 | numpy 3 | h5py 4 | cmake 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from skbuild import setup # This line replaces 'from setuptools import setup' 2 | import argparse 3 | 4 | import io 5 | import sys, os 6 | this_directory = os.path.abspath(os.path.dirname(__file__)) 7 | with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: 8 | long_description = f.read() 9 | 10 | import re 11 | VERSIONFILE="larcv/_version.py" 12 | verstrline = open(VERSIONFILE, "rt").read() 13 | VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" 14 | mo = re.search(VSRE, verstrline, re.M) 15 | if mo: 16 | verstr = mo.group(1) 17 | else: 18 | raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) 19 | 20 | 21 | 22 | v = sys.version_info 23 | py_version = ".".join([str(v.major), str(v.minor), str(v.micro)]) 24 | print(py_version) 25 | 26 | 27 | # Several environment variables control special build features: 28 | if 'LARCV_WITH_MPI' in os.environ and os.environ['LARCV_WITH_MPI']: 29 | mpi_value='ON' 30 | else: 31 | mpi_value='OFF' 32 | 33 | if 'LARCV_WITH_OPENMP' in os.environ and os.environ['LARCV_WITH_OPENMP']: 34 | openmp_value='ON' 35 | else: 36 | openmp_value='OFF' 37 | 38 | if 'LARCV_WITHOUT_PYBIND' in os.environ and os.environ['LARCV_WITHOUT_PYBIND']: 39 | pybind_value='OFF' 40 | else: 41 | pybind_value='ON' 42 | 43 | 44 | # Speed up the build if not directly set: 45 | if 'MAKEFLAGS' not in os.environ: 46 | os.environ['MAKEFLAGS'] = "-j 1" # This lets builds on CI work better. 47 | 48 | setup( 49 | name="larcv", 50 | version=verstr, 51 | cmake_source_dir='src/', 52 | include_package_data=True, 53 | cmake_args=[ 54 | '-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.9', 55 | '-DCMAKE_PYTHON_BINDINGS={}'.format(pybind_value), 56 | # '-DMPI_CXX_COMPILER={}'.format(mpicxx), 57 | # '-DMPI_C_COMPILER={}'.format(mpicc), 58 | '-DMPI:BOOL={}'.format(mpi_value), 59 | '-DOPENMP:BOOL={}'.format(openmp_value), 60 | f'-DCMAKE_PYVERSION={py_version}' 61 | ], 62 | author=['Corey Adams', 'Kazuhiro Terao', 'Taritree Wongjirad', 'Marco del Tutto'], 63 | author_email='corey.adams@anl.gov', 64 | description='C++ IO and Preprocessing package for sparse neutrino data, with H5 for IO and python bindings.', 65 | license='MIT', 66 | keywords='larcv larcv3 neutrinos hdf5 h5 deep learning IO sparse', 67 | project_urls={ 68 | 'Source Code': 'https://github.com/DeepLearnPhysics/larcv3' 69 | }, 70 | url='https://github.com/DeepLearnPhysics/larcv3', 71 | scripts=['bin/merge_larcv3_files.py', 'bin/run_processor.py'], 72 | packages=['larcv','src/pybind11'], 73 | install_requires=[ 74 | "numpy", 75 | "scikit-build", 76 | "h5py", 77 | "cmake", 78 | ], 79 | long_description=long_description, 80 | long_description_content_type='text/markdown', 81 | ) 82 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.1 FATAL_ERROR) 2 | 3 | 4 | # general configuration for compilation: 5 | set(CMAKE_CXX_STANDARD 11) 6 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 7 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -g") 8 | 9 | enable_language(C CXX) 10 | 11 | # We set a larcv compiler definition to build all of the python code 12 | # This is obviously not set if you are including headers, which protects 13 | # the need to install pybind11 outside of larcv (since it's header only, 14 | # there is no need to link to it.) 15 | add_definitions(-DLARCV_INTERNAL) 16 | 17 | 18 | if(MPI) 19 | set(MPI_ASSUME_NO_BUILTIN_MPI True) 20 | find_package(MPI REQUIRED COMPONENTS C CXX) 21 | include_directories(MPI_C_INCLUDE_DIRECTORIES) 22 | include_directories(MPI_CXX_INCLUDE_DIRECTORIES) 23 | add_definitions(-DLARCV_MPI) 24 | endif() 25 | 26 | 27 | # set(CMAKE_SKIP_RPATH TRUE) 28 | project(larcv3) 29 | 30 | set(CMAKE_PACKAGE_DIR "larcv") 31 | 32 | 33 | 34 | include_directories("./") 35 | 36 | # IO needs HDF5: 37 | find_package(HDF5 REQUIRED) 38 | include_directories(${HDF5_INCLUDE_DIR}) 39 | 40 | if(OPENMP) 41 | find_package(OpenMP) 42 | if(OpenMP_CXX_FOUND) 43 | MESSAGE(STATUS "OpenMP Include directories ${OpenMP_CXX_INCLUDE_DIRS}") 44 | include_directories(${OpenMP_C_INCLUDE_DIRS}) 45 | include_directories(${OpenMP_CXX_INCLUDE_DIRS}) 46 | add_definitions(-DLARCV_OPENMP) 47 | endif() 48 | endif() 49 | 50 | # This package needs numpy:: 51 | 52 | 53 | 54 | 55 | # Add the json library 56 | set(JSON_BuildTests OFF CACHE INTERNAL "") 57 | add_subdirectory(json) 58 | message("NLOHMANN_JSON_INCLUDE_INSTALL_DIR: ${NLOHMANN_JSON_INCLUDE_INSTALL_DIR}") 59 | include_directories(json/include) 60 | 61 | 62 | # endmacro(generate_python_bindings) 63 | if(CMAKE_PYTHON_BINDINGS) 64 | 65 | # Many packages need python: 66 | find_package(Python ${CMAKE_PYVERSION} REQUIRED COMPONENTS Interpreter Development) 67 | include_directories(${Python_INCLUDE_DIRS}) 68 | 69 | message("Adding pybind11") 70 | add_subdirectory(pybind11) 71 | include_directories(${PYBIND11_INCLUDE_DIR}) 72 | 73 | set(pybind11_DIR pybind11) 74 | 75 | # GIves bindings between json and python 76 | add_subdirectory(pybind11_json) 77 | include_directories(pybind11_json/include) 78 | 79 | else() 80 | add_definitions(-DLARCV_NO_PYBIND) 81 | endif(CMAKE_PYTHON_BINDINGS) 82 | 83 | 84 | 85 | add_subdirectory(larcv3) 86 | 87 | if(DOCS) 88 | set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/docs" ${CMAKE_MODULE_PATH}) 89 | add_subdirectory(docs) 90 | endif() 91 | -------------------------------------------------------------------------------- /src/larcv3/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(core) 2 | add_subdirectory(app) 3 | 4 | 5 | # This is admittedly pretty clunky, but it gets the job done to make one object file: 6 | 7 | add_library(larcv3 SHARED 8 | $ 9 | $ 10 | $ 11 | $ 12 | $ 13 | $ 14 | $ 15 | $ 16 | ) 17 | 18 | 19 | 20 | if (OpenMP_CXX_FOUND) 21 | message("Linking against openmp") 22 | message("OpenMP_CXX_LIBRARY: " ${OpenMP_CXX_LIBRARIES}) 23 | message("OpenMP_C_LIBRARY: " ${OpenMP_C_LIBRARIES}) 24 | target_link_libraries(larcv3 OpenMP::OpenMP_C) 25 | endif() 26 | 27 | if (MPI_FOUND) 28 | message("Linking against MPI") 29 | message("OpenMP_CXX_LIBRARY: " ${MPI_CXX_LIBRARIES}) 30 | message("OpenMP_C_LIBRARY: " ${MPI_C_LIBRARIES}) 31 | target_link_libraries(larcv3 ${MPI_CXX_LIBRARIES} ${MPI_C_LIBRARIES}) 32 | endif() 33 | 34 | set(HDF5_PREFER_PARALLEL TRUE) 35 | message("Using hdf5 parallel: " ${HDF5_IS_PARALLEL}) 36 | target_link_libraries(larcv3 ${HDF5_LIBRARIES}) 37 | 38 | # Link against python: 39 | target_link_libraries(larcv3 ${Python_LIBRARIES}) 40 | 41 | 42 | ############################### 43 | 44 | # Link against the json library: 45 | target_link_libraries(larcv3 nlohmann_json::nlohmann_json) 46 | 47 | 48 | if(CMAKE_PYTHON_BINDINGS) 49 | pybind11_add_module(pylarcv SHARED larcv.cxx) 50 | 51 | # Add a shared library 52 | set_target_properties(pylarcv PROPERTIES SKIP_BUILD_RPATH FALSE) 53 | 54 | # If building with setuptools, CMake will not be performing the install 55 | set_target_properties(pylarcv PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE) 56 | 57 | target_link_libraries(pylarcv PRIVATE larcv3) 58 | 59 | if(UNIX AND NOT APPLE) 60 | set_target_properties(pylarcv PROPERTIES INSTALL_RPATH "$ORIGIN/lib/") 61 | elseif(APPLE) 62 | set_target_properties(pylarcv PROPERTIES INSTALL_RPATH "@loader_path/lib/") 63 | endif() 64 | 65 | set_target_properties(pylarcv PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) 66 | 67 | endif(CMAKE_PYTHON_BINDINGS) 68 | 69 | ############################### 70 | 71 | if (MPI) 72 | target_link_libraries(larcv3 ${MPI_LIBRARIES}) 73 | endif() 74 | 75 | 76 | 77 | # Install larvc shared library 78 | install(TARGETS larcv3 79 | LIBRARY DESTINATION ${CMAKE_PACKAGE_DIR}/lib 80 | ARCHIVE DESTINATION ${CMAKE_PACKAGE_DIR}/lib 81 | RUNTIME DESTINATION ${CMAKE_PACKAGE_DIR}/bin 82 | INCLUDES DESTINATION ${CMAKE_PACKAGE_DIR}/include 83 | ) 84 | 85 | 86 | # Define the location of the header files, for doc generation 87 | set(LARCV3_PUBLIC_HEADER_DIR ${CMAKE_PACKAGE_DIR}/include) 88 | message("Header dir ${CMAKE_PACKAGE_DIR}/include") 89 | # set(LARCV3_PUBLIC_HEADER_DIR ${CMAKE_CURRENT_SOURCE_DIR}) 90 | 91 | 92 | if(CMAKE_PYTHON_BINDINGS) 93 | # Install larvc wrapper library 94 | install(TARGETS pylarcv 95 | LIBRARY DESTINATION ${CMAKE_PACKAGE_DIR} 96 | ARCHIVE DESTINATION ${CMAKE_PACKAGE_DIR} 97 | # RUNTIME DESTINATION ${CMAKE_PACKAGE_DIR}/bin 98 | # INCLUDES DESTINATION ${CMAKE_PACKAGE_DIR}/include 99 | ) 100 | endif(CMAKE_PYTHON_BINDINGS) 101 | -------------------------------------------------------------------------------- /src/larcv3/app/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(filter) 2 | add_subdirectory(queueio) 3 | add_subdirectory(imagemod) 4 | add_subdirectory(sbnd_imagemod) 5 | add_subdirectory(augment) 6 | -------------------------------------------------------------------------------- /src/larcv3/app/augment/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name augment) 2 | 3 | 4 | # Get all the source files: 5 | file(GLOB SOURCES *.cxx) 6 | file(GLOB HEADERS *.h) 7 | 8 | # Add a shared library 9 | add_library(${name} OBJECT ${SOURCES}) 10 | 11 | 12 | install (FILES ${HEADERS} 13 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/app/${name}) 14 | -------------------------------------------------------------------------------- /src/larcv3/app/augment/GaussianBlur.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file GaussianBlur.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class GaussianBlur 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_GAUSSIANBLUR_H__ 15 | #define __LARCV3_GAUSSIANBLUR_H__ 16 | 17 | #include 18 | 19 | #include "larcv3/core/processor/ProcessBase.h" 20 | #include "larcv3/core/processor/ProcessFactory.h" 21 | namespace larcv3 { 22 | 23 | /** 24 | \class ProcessBase 25 | User defined class GaussianBlur ... these comments are used to generate 26 | doxygen documentation! 27 | */ 28 | class GaussianBlur : public ProcessBase { 29 | 30 | public: 31 | 32 | /// Default constructor 33 | GaussianBlur(const std::string name="GaussianBlur"); 34 | 35 | /// Default destructor 36 | ~GaussianBlur(){} 37 | 38 | void configure(const json&); 39 | 40 | void initialize(); 41 | 42 | bool process(IOManager& mgr); 43 | 44 | void finalize(); 45 | 46 | static json default_config(){ 47 | json c = { 48 | {"Producer", std::string()}, 49 | {"Product", std::string()}, 50 | {"OutputProducer", std::string()}, 51 | {"Sigma", float(0.0)}, 52 | }; 53 | return c; 54 | } 55 | 56 | private: 57 | 58 | json config; 59 | 60 | template 61 | bool process_dense_product( 62 | IOManager& mgr, 63 | std::string producer, 64 | std::string output_producer, 65 | std::normal_distribution, 66 | std::default_random_engine generator); 67 | 68 | template 69 | bool process_sparse_product( 70 | IOManager& mgr, 71 | std::string producer, 72 | std::string output_producer, 73 | std::normal_distribution, 74 | std::default_random_engine generator); 75 | 76 | }; 77 | 78 | /** 79 | \class larcv3::GaussianBlurFactory 80 | \brief A concrete factory class for larcv3::GaussianBlur 81 | */ 82 | class GaussianBlurProcessFactory : public ProcessFactoryBase { 83 | public: 84 | /// ctor 85 | GaussianBlurProcessFactory() { ProcessFactory::get().add_factory("GaussianBlur",this); } 86 | /// dtor 87 | ~GaussianBlurProcessFactory() {} 88 | /// creation method 89 | ProcessBase* create(const std::string instance_name) { return new GaussianBlur(instance_name); } 90 | }; 91 | 92 | } 93 | 94 | #ifdef LARCV_INTERNAL 95 | #include 96 | void init_gaussian_blur(pybind11::module m); 97 | #endif 98 | 99 | #endif 100 | /** @} */ // end of doxygen group 101 | 102 | -------------------------------------------------------------------------------- /src/larcv3/app/augment/Mirror.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Mirror.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class Mirror 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_MIRROR_H__ 15 | #define __LARCV3_MIRROR_H__ 16 | 17 | #include 18 | 19 | #include "larcv3/core/processor/ProcessBase.h" 20 | #include "larcv3/core/processor/ProcessFactory.h" 21 | namespace larcv3 { 22 | 23 | /** 24 | \class ProcessBase 25 | User defined class Mirror ... these comments are used to generate 26 | doxygen documentation! 27 | */ 28 | class Mirror : public ProcessBase { 29 | 30 | public: 31 | 32 | /// Default constructor 33 | Mirror(const std::string name="Mirror"); 34 | 35 | /// Default destructor 36 | ~Mirror(){} 37 | 38 | void configure(const json&); 39 | 40 | void initialize(); 41 | 42 | bool process(IOManager& mgr); 43 | 44 | void finalize(); 45 | 46 | static json default_config(){ 47 | json c = { 48 | {"Producer", std::string()}, 49 | {"Product", std::string()}, 50 | {"OutputProducer", std::string()}, 51 | {"Axes", std::vector()}, 52 | }; 53 | return c; 54 | } 55 | 56 | private: 57 | 58 | json config; 59 | 60 | template 61 | bool process_dense_product( 62 | IOManager& mgr, 63 | std::string producer, 64 | std::string output_producer, 65 | std::vector); 66 | 67 | template 68 | bool process_sparse_product( 69 | IOManager& mgr, 70 | std::string producer, 71 | std::string output_producer, 72 | std::vector); 73 | 74 | }; 75 | 76 | /** 77 | \class larcv3::MirrorFactory 78 | \brief A concrete factory class for larcv3::Mirror 79 | */ 80 | class MirrorProcessFactory : public ProcessFactoryBase { 81 | public: 82 | /// ctor 83 | MirrorProcessFactory() { ProcessFactory::get().add_factory("Mirror",this); } 84 | /// dtor 85 | ~MirrorProcessFactory() {} 86 | /// creation method 87 | ProcessBase* create(const std::string instance_name) { return new Mirror(instance_name); } 88 | }; 89 | 90 | } 91 | 92 | #ifdef LARCV_INTERNAL 93 | #include 94 | void init_mirror(pybind11::module m); 95 | #endif 96 | 97 | #endif 98 | /** @} */ // end of doxygen group 99 | 100 | -------------------------------------------------------------------------------- /src/larcv3/app/augment/Translate.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Translate.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class Translate 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_Translate_H__ 15 | #define __LARCV3_Translate_H__ 16 | 17 | #include 18 | 19 | #include "larcv3/core/processor/ProcessBase.h" 20 | #include "larcv3/core/processor/ProcessFactory.h" 21 | namespace larcv3 { 22 | 23 | /** 24 | \class ProcessBase 25 | User defined class Translate ... these comments are used to generate 26 | doxygen documentation! 27 | */ 28 | class Translate : public ProcessBase { 29 | 30 | public: 31 | 32 | /// Default constructor 33 | Translate(const std::string name="Translate"); 34 | 35 | /// Default destructor 36 | ~Translate(){} 37 | 38 | void configure(const json&); 39 | 40 | void initialize(); 41 | 42 | bool process(IOManager& mgr); 43 | 44 | void finalize(); 45 | 46 | static json default_config(){ 47 | json c = { 48 | {"Producer", std::string()}, 49 | {"Product", std::string()}, 50 | {"OutputProducer", std::string()}, 51 | {"MaxShiftPerAxis", std::vector()}, 52 | }; 53 | return c; 54 | } 55 | 56 | private: 57 | 58 | json config; 59 | 60 | template 61 | bool process_dense_product( 62 | IOManager& mgr, 63 | std::string producer, 64 | std::string output_producer, 65 | std::vector); 66 | 67 | template 68 | bool process_sparse_product( 69 | IOManager& mgr, 70 | std::string producer, 71 | std::string output_producer, 72 | std::vector); 73 | 74 | }; 75 | 76 | /** 77 | \class larcv3::TranslateFactory 78 | \brief A concrete factory class for larcv3::Translate 79 | */ 80 | class TranslateProcessFactory : public ProcessFactoryBase { 81 | public: 82 | /// ctor 83 | TranslateProcessFactory() { ProcessFactory::get().add_factory("Translate",this); } 84 | /// dtor 85 | ~TranslateProcessFactory() {} 86 | /// creation method 87 | ProcessBase* create(const std::string instance_name) { return new Translate(instance_name); } 88 | }; 89 | 90 | } 91 | 92 | #ifdef LARCV_INTERNAL 93 | #include 94 | void init_translate(pybind11::module m); 95 | #endif 96 | 97 | #endif 98 | /** @} */ // end of doxygen group 99 | 100 | -------------------------------------------------------------------------------- /src/larcv3/app/augment/augment.cxx: -------------------------------------------------------------------------------- 1 | #include "augment.h" 2 | 3 | #include "GaussianBlur.h" 4 | #include "Translate.h" 5 | #include "Mirror.h" 6 | 7 | void init_augment(pybind11::module m){ 8 | 9 | init_gaussian_blur(m); 10 | init_translate(m); 11 | init_mirror(m); 12 | } 13 | -------------------------------------------------------------------------------- /src/larcv3/app/augment/augment.h: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * \file larbys.h 4 | * 5 | * \ingroup app_augment 6 | * 7 | * \brief Class def header for pybind11 for larcv3 framework 8 | * 9 | * @author cadams 10 | */ 11 | 12 | /** \addtogroup core_Base 13 | 14 | @{*/ 15 | #ifndef __LARCV3AUGMENT_AUGMENT_H__ 16 | #define __LARCV3AUGMENT_AUGMENT_H__ 17 | 18 | 19 | 20 | #ifndef LARCV_NO_PYBIND 21 | #ifdef LARCV_INTERNAL 22 | #include 23 | __attribute__ ((visibility ("default"))) void init_augment(pybind11::module m); 24 | #endif 25 | // bindings 26 | #endif 27 | 28 | 29 | // include guards 30 | #endif 31 | -------------------------------------------------------------------------------- /src/larcv3/app/filter/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name filter) 2 | 3 | 4 | # Get all the source files: 5 | file(GLOB SOURCES *.cxx) 6 | file(GLOB HEADERS *.h) 7 | 8 | # Add a shared library 9 | add_library(${name} OBJECT ${SOURCES}) 10 | 11 | 12 | 13 | 14 | 15 | install (FILES ${HEADERS} 16 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/app/${name}) 17 | -------------------------------------------------------------------------------- /src/larcv3/app/filter/EmptyImageFilter.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3FILTER_EMPTYIMAGEFILTER_CXX__ 2 | #define __LARCV3FILTER_EMPTYIMAGEFILTER_CXX__ 3 | 4 | #include "EmptyImageFilter.h" 5 | #include "larcv3/core/dataformat/EventTensor.h" 6 | namespace larcv3 { 7 | 8 | static EmptyImageFilterProcessFactory __global_EmptyImageFilterProcessFactory__; 9 | 10 | EmptyImageFilter::EmptyImageFilter(const std::string name) 11 | : ProcessBase(name) 12 | {} 13 | 14 | void EmptyImageFilter::configure(const json& cfg) 15 | { 16 | config = this -> default_config(); 17 | config = augment_default_config(config, cfg); 18 | } 19 | 20 | void EmptyImageFilter::initialize() 21 | {} 22 | 23 | bool EmptyImageFilter::process(IOManager& mgr) 24 | { 25 | 26 | auto const& ev_image = mgr.get_data(config["ImageProducer"].get()); 27 | if(ev_image.image2d_array().empty()) return false; 28 | return true; 29 | } 30 | 31 | void EmptyImageFilter::finalize() 32 | {} 33 | 34 | } 35 | #endif 36 | -------------------------------------------------------------------------------- /src/larcv3/app/filter/EmptyImageFilter.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file EmptyImageFilter.h 3 | * 4 | * \ingroup Package_Name 5 | * 6 | * \brief Class def header for a class EmptyImageFilter 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup Package_Name 12 | 13 | @{*/ 14 | #ifndef __LARCV3FILTER_EMPTYIMAGEFILTER_H__ 15 | #define __LARCV3FILTER_EMPTYIMAGEFILTER_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class EmptyImageFilter ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class EmptyImageFilter : public larcv3::ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | EmptyImageFilter(const std::string name="EmptyImageFilter"); 32 | 33 | /// Default destructor 34 | ~EmptyImageFilter(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(larcv3::IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | static json default_config() { 45 | json c = { 46 | {"ImageProducer", ""}, 47 | }; 48 | return c; 49 | } 50 | 51 | private: 52 | 53 | json config; 54 | }; 55 | 56 | /** 57 | \class larcv3::EmptyImageFilterFactory 58 | \brief A concrete factory class for larcv3::EmptyImageFilter 59 | */ 60 | class EmptyImageFilterProcessFactory : public ProcessFactoryBase { 61 | public: 62 | /// ctor 63 | EmptyImageFilterProcessFactory() { ProcessFactory::get().add_factory("EmptyImageFilter",this); } 64 | /// dtor 65 | ~EmptyImageFilterProcessFactory() {} 66 | /// creation method 67 | ProcessBase* create(const std::string instance_name) { return new EmptyImageFilter(instance_name); } 68 | }; 69 | 70 | } 71 | 72 | #endif 73 | /** @} */ // end of doxygen group 74 | 75 | -------------------------------------------------------------------------------- /src/larcv3/app/filter/EmptyTensorFilter.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file EmptyTensorFilter.h 3 | * 4 | * \ingroup Package_Name 5 | * 6 | * \brief Class def header for a class EmptyTensorFilter 7 | * 8 | * @author drinkingkazu 9 | */ 10 | 11 | /** \addtogroup Package_Name 12 | 13 | @{*/ 14 | #ifndef __LARCV3FILTER_EMPTYTENSORFILTER_H__ 15 | #define __LARCV3FILTER_EMPTYTENSORFILTER_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class EmptyTensorFilter ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class EmptyTensorFilter : public larcv3::ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | EmptyTensorFilter(const std::string name="EmptyTensorFilter"); 32 | 33 | /// Default destructor 34 | ~EmptyTensorFilter(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(larcv3::IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | static json default_config() { 45 | json c = { 46 | {"TensorProducer", std::vector()}, // Name of producer 47 | {"TensorType", std::vector()}, // Type of produce, eg, sparse2d 48 | {"MinVoxelCount", std::vector()}, // Min Number of voxels per projection 49 | {"MinVoxelValue", std::vector()}, // Min voxel value per projection 50 | }; 51 | return c; 52 | } 53 | 54 | private: 55 | template 56 | bool process_tensor(IOManager & mgr, std::string producer, int min_voxel_count, float min_voxel_value); 57 | 58 | template 59 | bool process_sparse(IOManager & mgr, std::string producer, int min_voxel_count, float min_voxel_value); 60 | 61 | 62 | json config; 63 | 64 | }; 65 | 66 | /** 67 | \class larcv3::EmptyTensorFilterFactory 68 | \brief A concrete factory class for larcv3::EmptyTensorFilter 69 | */ 70 | class EmptyTensorFilterProcessFactory : public ProcessFactoryBase { 71 | public: 72 | /// ctor 73 | EmptyTensorFilterProcessFactory() { ProcessFactory::get().add_factory("EmptyTensorFilter",this); } 74 | /// dtor 75 | ~EmptyTensorFilterProcessFactory() {} 76 | /// creation method 77 | ProcessBase* create(const std::string instance_name) { return new EmptyTensorFilter(instance_name); } 78 | }; 79 | 80 | } 81 | 82 | #endif 83 | /** @} */ // end of doxygen group 84 | 85 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/BBoxFromCluster.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BBoxFromCluster.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class BBoxFromCluster 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV_BBOXFROMCLUSTER_H__ 15 | #define __LARCV_BBOXFROMCLUSTER_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | 20 | namespace larcv3 { 21 | 22 | /** 23 | \class ProcessBase 24 | User defined class BBoxFromCluster ... these comments are used to generate 25 | doxygen documentation! 26 | */ 27 | class BBoxFromCluster : public ProcessBase { 28 | 29 | public: 30 | 31 | /// Default constructor 32 | BBoxFromCluster(const std::string name="BBoxFromCluster"); 33 | 34 | /// Default destructor 35 | ~BBoxFromCluster(){} 36 | 37 | void configure(const json&); 38 | 39 | void initialize(); 40 | 41 | bool process(IOManager& mgr); 42 | 43 | void finalize(); 44 | 45 | static json default_config(){ 46 | json c = { 47 | {"Producer", std::string()}, 48 | {"Product", std::string()}, 49 | {"OutputProducer", std::string()}, 50 | {"Threshold", 0.0}, 51 | {"MinVoxels", 1}, 52 | }; 53 | return c; 54 | } 55 | 56 | private: 57 | 58 | json config; 59 | 60 | template 61 | bool bbox_from_cluster(IOManager& mgr, std::string producer, std::string output_producer); 62 | 63 | }; 64 | 65 | /** 66 | \class larcv3::BBoxFromClusterProcessFactory 67 | \brief A concrete factory class for larcv3::BBoxFromCluster 68 | */ 69 | 70 | 71 | class BBoxFromClusterProcessFactory : public ProcessFactoryBase { 72 | public: 73 | /// ctor 74 | BBoxFromClusterProcessFactory() { ProcessFactory::get().add_factory("BBoxFromCluster",this); } 75 | /// dtor 76 | ~BBoxFromClusterProcessFactory() {} 77 | /// creation method 78 | ProcessBase* create(const std::string instance_name) { return new BBoxFromCluster(instance_name); } 79 | }; 80 | 81 | 82 | } 83 | 84 | #ifdef LARCV_INTERNAL 85 | #include 86 | void init_bbox_from_cluster(pybind11::module m); 87 | #endif 88 | 89 | #endif 90 | /** @} */ // end of doxygen group 91 | 92 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/BBoxFromParticle.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BBoxFromParticle.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class BBoxFromParticle 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV_BBOXFROMPARTICLE_H__ 15 | #define __LARCV_BBOXFROMPARTICLE_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | 20 | namespace larcv3 { 21 | 22 | /** 23 | \class ProcessBase 24 | User defined class BBoxFromParticle ... these comments are used to generate 25 | doxygen documentation! 26 | */ 27 | class BBoxFromParticle : public ProcessBase { 28 | 29 | public: 30 | 31 | /// Default constructor 32 | BBoxFromParticle(const std::string name="BBoxFromParticle"); 33 | 34 | /// Default destructor 35 | ~BBoxFromParticle(){} 36 | 37 | void configure(const json&); 38 | 39 | void initialize(); 40 | 41 | bool process(IOManager& mgr); 42 | 43 | void finalize(); 44 | 45 | static json default_config(){ 46 | json c = { 47 | {"Producer", std::string()}, 48 | {"OutputProducer", std::string()}, 49 | }; 50 | return c; 51 | } 52 | 53 | private: 54 | 55 | json config; 56 | 57 | template 58 | bool bbox_from_particle(IOManager& mgr, std::string producer, std::string output_producer); 59 | 60 | }; 61 | 62 | /** 63 | \class larcv3::BBoxFromParticleProcessFactory 64 | \brief A concrete factory class for larcv3::BBoxFromParticle 65 | */ 66 | 67 | 68 | class BBoxFromParticleProcessFactory : public ProcessFactoryBase { 69 | public: 70 | /// ctor 71 | BBoxFromParticleProcessFactory() { ProcessFactory::get().add_factory("BBoxFromParticle",this); } 72 | /// dtor 73 | ~BBoxFromParticleProcessFactory() {} 74 | /// creation method 75 | ProcessBase* create(const std::string instance_name) { return new BBoxFromParticle(instance_name); } 76 | }; 77 | 78 | 79 | } 80 | 81 | #ifdef LARCV_INTERNAL 82 | #include 83 | void init_bbox_from_particle(pybind11::module m); 84 | #endif 85 | 86 | #endif 87 | /** @} */ // end of doxygen group 88 | 89 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name imagemod) 2 | 3 | 4 | # Get all the source files: 5 | file(GLOB SOURCES *.cxx) 6 | file(GLOB HEADERS *.h) 7 | 8 | # Add a shared library 9 | add_library(${name} OBJECT ${SOURCES}) 10 | 11 | 12 | install (FILES ${HEADERS} 13 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/app/${name}) 14 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/DenseToSparse.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file DenseToSparse.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class DenseToSparse 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_DENSETOSPARSE_H__ 15 | #define __LARCV3_DENSETOSPARSE_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class DenseToSparse ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class DenseToSparse : public ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | DenseToSparse(const std::string name="DenseToSparse"); 32 | 33 | /// Default destructor 34 | ~DenseToSparse(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | static json default_config(){ 45 | json c = { 46 | {"Producer", std::string()}, 47 | {"Product", std::string()}, 48 | {"OutputProducer", std::string()}, 49 | {"ReferenceProducer", std::string()} 50 | }; 51 | return c; 52 | } 53 | 54 | private: 55 | 56 | json config; 57 | 58 | template 59 | bool process_product( 60 | IOManager& mgr, 61 | std::string producer, 62 | std::string output_producer, 63 | std::string ref_producer); 64 | 65 | }; 66 | 67 | /** 68 | \class larcv3::DenseToSparseFactory 69 | \brief A concrete factory class for larcv3::DenseToSparse 70 | */ 71 | class DenseToSparseProcessFactory : public ProcessFactoryBase { 72 | public: 73 | /// ctor 74 | DenseToSparseProcessFactory() { ProcessFactory::get().add_factory("DenseToSparse",this); } 75 | /// dtor 76 | ~DenseToSparseProcessFactory() {} 77 | /// creation method 78 | ProcessBase* create(const std::string instance_name) { return new DenseToSparse(instance_name); } 79 | }; 80 | 81 | } 82 | 83 | #ifdef LARCV_INTERNAL 84 | #include 85 | void init_dense_to_sparse(pybind11::module m); 86 | #endif 87 | 88 | #endif 89 | /** @} */ // end of doxygen group 90 | 91 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/Downsample.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Downsample.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class Downsample 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_DOWNSAMPLE_H__ 15 | #define __LARCV3_DOWNSAMPLE_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class Downsample ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class Downsample : public ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | Downsample(const std::string name="Downsample"); 32 | 33 | /// Default destructor 34 | ~Downsample(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | 45 | static json default_config(){ 46 | json c = { 47 | {"Producer", std::string()}, 48 | {"Product", std::string()}, 49 | {"OutputProducer", std::string()}, 50 | {"Downsample", 0}, 51 | {"PoolType", kPoolAverage}, 52 | }; 53 | return c; 54 | } 55 | 56 | 57 | private: 58 | 59 | template< class dataproduct> 60 | bool process_data_product( 61 | IOManager& mgr, 62 | std::string producer, 63 | std::string output_producer, 64 | int downsample, 65 | PoolType_t pool); 66 | 67 | template< class dataproduct> 68 | bool process_bbox_product( 69 | IOManager& mgr, 70 | std::string producer, 71 | std::string output_producer, 72 | int downsample); 73 | 74 | json config; 75 | 76 | }; 77 | 78 | 79 | /** 80 | \class larcv3::ThresholdFactory 81 | \brief A concrete factory class for larcv3::Downsample 82 | */ 83 | class DownsampleProcessFactory : public ProcessFactoryBase { 84 | public: 85 | /// ctor 86 | DownsampleProcessFactory() { ProcessFactory::get().add_factory("Downsample",this); } 87 | /// dtor 88 | ~DownsampleProcessFactory() {} 89 | /// creation method 90 | ProcessBase* create(const std::string instance_name) { return new Downsample(instance_name); } 91 | }; 92 | 93 | } 94 | 95 | #ifdef LARCV_INTERNAL 96 | #include 97 | void init_downsample(pybind11::module m); 98 | #endif 99 | 100 | 101 | #endif 102 | /** @} */ // end of doxygen group 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/Embed.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Embed.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class Embed 7 | * 8 | * Embed an object into a bigger image, without downsampling. 9 | * 10 | * @author cadams 11 | */ 12 | 13 | /** \addtogroup ImageMod 14 | 15 | @{*/ 16 | #ifndef __LARCV3_EMBED_H__ 17 | #define __LARCV3_EMBED_H__ 18 | 19 | #include "larcv3/core/processor/ProcessBase.h" 20 | #include "larcv3/core/processor/ProcessFactory.h" 21 | #include "larcv3/core/dataformat/ImageMeta.h" 22 | namespace larcv3 { 23 | 24 | /** 25 | \class ProcessBase 26 | User defined class embed ... these comments are used to generate 27 | doxygen documentation! 28 | */ 29 | class Embed : public ProcessBase { 30 | 31 | public: 32 | 33 | /// Default constructor 34 | Embed(const std::string name="Embed"); 35 | 36 | /// Default destructor 37 | ~Embed(){} 38 | 39 | void configure(const json&); 40 | 41 | void initialize(); 42 | 43 | bool process(IOManager& mgr); 44 | 45 | void finalize(); 46 | 47 | 48 | static json default_config(){ 49 | json c = { 50 | {"Producer", std::string()}, 51 | {"Product", std::string()}, 52 | {"OutputProducer", std::string()}, 53 | {"TargetSize", std::vector()}, 54 | }; 55 | return c; 56 | } 57 | 58 | 59 | private: 60 | 61 | template< size_t dimension> 62 | bool process_dense_product( 63 | IOManager& mgr, 64 | std::string producer, 65 | std::string output_producer, 66 | const std::vector& target_size); 67 | 68 | template< size_t dimension> 69 | bool process_sparse_product( 70 | IOManager& mgr, 71 | std::string producer, 72 | std::string output_producer, 73 | const std::vector& target_size); 74 | 75 | template< size_t dimension> 76 | bool process_bbox_product( 77 | IOManager& mgr, 78 | std::string producer, 79 | std::string output_producer, 80 | const std::vector& target_size); 81 | 82 | template< size_t dimension> 83 | bool process_cluster_product( 84 | IOManager& mgr, 85 | std::string producer, 86 | std::string output_producer, 87 | const std::vector& target_size); 88 | 89 | template< size_t dimension> 90 | std::vector create_new_image_meta_and_offsets( 91 | const ImageMeta & input, 92 | const std::vector & target_size, 93 | ImageMeta & new_meta); 94 | 95 | json config; 96 | 97 | }; 98 | 99 | 100 | /** 101 | \class larcv3::ThresholdFactory 102 | \brief A concrete factory class for larcv3::Embed 103 | */ 104 | class EmbedProcessFactory : public ProcessFactoryBase { 105 | public: 106 | /// ctor 107 | EmbedProcessFactory() { ProcessFactory::get().add_factory("Embed",this); } 108 | /// dtor 109 | ~EmbedProcessFactory() {} 110 | /// creation method 111 | ProcessBase* create(const std::string instance_name) { return new Embed(instance_name); } 112 | }; 113 | 114 | } 115 | 116 | #ifdef LARCV_INTERNAL 117 | #include 118 | void init_embed(pybind11::module m); 119 | #endif 120 | 121 | 122 | #endif 123 | /** @} */ // end of doxygen group 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/Normalize.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Normalize.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class Normalize 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_NORMALIZE_H__ 15 | #define __LARCV3_NORMALIZE_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class Normalize ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class Normalize : public ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | Normalize(const std::string name="Normalize"); 32 | 33 | /// Default destructor 34 | ~Normalize(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | static json default_config(){ 45 | json c = { 46 | {"Producer", std::string()}, 47 | {"Product", std::string()}, 48 | {"OutputProducer", std::string()}, 49 | {"Mean", float(0.0)}, 50 | {"Std", float(1.0)}, 51 | }; 52 | return c; 53 | } 54 | 55 | private: 56 | 57 | json config; 58 | 59 | template 60 | bool process_dense_product( 61 | IOManager& mgr, 62 | std::string producer, 63 | std::string output_producer, 64 | float target_mean, 65 | float target_std); 66 | 67 | template 68 | bool process_sparse_product( 69 | IOManager& mgr, 70 | std::string producer, 71 | std::string output_producer, 72 | float target_mean, 73 | float target_std); 74 | 75 | }; 76 | 77 | /** 78 | \class larcv3::ThresholdFactory 79 | \brief A concrete factory class for larcv3::Normalize 80 | */ 81 | class NormalizeProcessFactory : public ProcessFactoryBase { 82 | public: 83 | /// ctor 84 | NormalizeProcessFactory() { ProcessFactory::get().add_factory("Normalize",this); } 85 | /// dtor 86 | ~NormalizeProcessFactory() {} 87 | /// creation method 88 | ProcessBase* create(const std::string instance_name) { return new Normalize(instance_name); } 89 | }; 90 | 91 | } 92 | 93 | #ifdef LARCV_INTERNAL 94 | #include 95 | void init_normalize(pybind11::module m); 96 | #endif 97 | 98 | #endif 99 | /** @} */ // end of doxygen group 100 | 101 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/SparseToDense.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __SPARSETODENSE_CXX__ 2 | #define __SPARSETODENSE_CXX__ 3 | 4 | #include "SparseToDense.h" 5 | #include "larcv3/core/dataformat/EventTensor.h" 6 | #include "larcv3/core/dataformat/EventSparseTensor.h" 7 | #include "larcv3/core/dataformat/EventSparseCluster.h" 8 | 9 | namespace larcv3 { 10 | 11 | static SparseToDenseProcessFactory __global_SparseToDenseProcessFactory__; 12 | 13 | 14 | SparseToDense::SparseToDense(const std::string name) 15 | : ProcessBase(name) {} 16 | 17 | 18 | 19 | void SparseToDense::configure(const json& cfg) { 20 | 21 | config = this -> default_config(); 22 | config = augment_default_config(config, cfg); 23 | 24 | } 25 | 26 | 27 | void SparseToDense::initialize() {} 28 | 29 | 30 | bool SparseToDense::process(IOManager& mgr) { 31 | 32 | //We apply thresholding for each projection id. 33 | 34 | // std::cout << "Enter SparseToDense::process " << std::endl; 35 | auto const& producer = config["Producer"].get(); 36 | auto const& product = config["Product"].get(); 37 | auto const& output_producer = config["OutputProducer"].get(); 38 | 39 | 40 | if (product == "sparse2d") 41 | process_product(mgr, producer, output_producer); 42 | else if (product == "sparse3d") 43 | process_product(mgr, producer, output_producer); 44 | else{ 45 | LARCV_CRITICAL() << "Can't convert product " << product << " to a dense form " << std::endl; 46 | throw larbys(); 47 | } 48 | 49 | return true; 50 | } 51 | 52 | template 53 | bool SparseToDense::process_product( 54 | IOManager& mgr, 55 | std::string producer, 56 | std::string output_producer){ 57 | 58 | auto const & ev_input = mgr.template get_data(producer); 59 | auto & ev_output = mgr.template get_data(output_producer); 60 | 61 | for (size_t i = 0; i < ev_input.as_vector().size(); i ++ ){ 62 | auto object = ev_input.as_vector().at(i); 63 | auto dense = object.to_tensor(); 64 | ev_output.emplace(std::move(dense)); 65 | } 66 | 67 | return true; 68 | } 69 | 70 | 71 | 72 | void SparseToDense::finalize() {} 73 | 74 | } //namespace larcv3 75 | 76 | 77 | #include 78 | 79 | 80 | 81 | void init_sparse_to_dense(pybind11::module m){ 82 | 83 | 84 | using Class = larcv3::SparseToDense; 85 | pybind11::class_ SparseToDense(m, "SparseToDense"); 86 | // pybind11::class_> ev_sparse_tensor(m, classname.c_str()); 87 | 88 | SparseToDense.doc() = R"pbdoc( 89 | Convert Sparse Products to their corresponding dense products. 90 | )pbdoc"; 91 | 92 | 93 | 94 | SparseToDense.def(pybind11::init(), 95 | pybind11::arg("name") = "SparseToDense"); 96 | 97 | SparseToDense.def("default_config", &Class::default_config); 98 | } 99 | 100 | #endif 101 | 102 | 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/SparseToDense.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file SparseToDense.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class SparseToDense 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_SPARSETODENSE_H__ 15 | #define __LARCV3_SPARSETODENSE_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class SparseToDense ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class SparseToDense : public ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | SparseToDense(const std::string name="SparseToDense"); 32 | 33 | /// Default destructor 34 | ~SparseToDense(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | static json default_config(){ 45 | json c = { 46 | {"Producer", std::string()}, 47 | {"Product", std::string()}, 48 | {"OutputProducer", std::string()}, 49 | }; 50 | return c; 51 | } 52 | 53 | private: 54 | 55 | json config; 56 | 57 | template 58 | bool process_product( 59 | IOManager& mgr, 60 | std::string producer, 61 | std::string output_producer); 62 | 63 | }; 64 | 65 | /** 66 | \class larcv3::SparseToDenseFactory 67 | \brief A concrete factory class for larcv3::SparseToDense 68 | */ 69 | class SparseToDenseProcessFactory : public ProcessFactoryBase { 70 | public: 71 | /// ctor 72 | SparseToDenseProcessFactory() { ProcessFactory::get().add_factory("SparseToDense",this); } 73 | /// dtor 74 | ~SparseToDenseProcessFactory() {} 75 | /// creation method 76 | ProcessBase* create(const std::string instance_name) { return new SparseToDense(instance_name); } 77 | }; 78 | 79 | } 80 | 81 | #ifdef LARCV_INTERNAL 82 | #include 83 | void init_sparse_to_dense(pybind11::module m); 84 | #endif 85 | 86 | #endif 87 | /** @} */ // end of doxygen group 88 | 89 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/TensorFromCluster.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file TensorFromCluster.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class TensorFromCluster 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV_TENSORFROMCLUSTER_H__ 15 | #define __LARCV_TENSORFROMCLUSTER_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | 20 | namespace larcv3 { 21 | 22 | /** 23 | \class ProcessBase 24 | User defined class TensorFromCluster ... these comments are used to generate 25 | doxygen documentation! 26 | */ 27 | class TensorFromCluster : public ProcessBase { 28 | 29 | public: 30 | 31 | /// Default constructor 32 | TensorFromCluster(const std::string name="TensorFromCluster"); 33 | 34 | /// Default destructor 35 | ~TensorFromCluster(){} 36 | 37 | void configure(const json&); 38 | 39 | void initialize(); 40 | 41 | bool process(IOManager& mgr); 42 | 43 | void finalize(); 44 | 45 | static json default_config(){ 46 | json c = { 47 | {"Producer", std::string()}, 48 | {"Product", std::string()}, 49 | {"OutputProducer", std::string()}, 50 | }; 51 | return c; 52 | } 53 | 54 | private: 55 | 56 | json config; 57 | 58 | 59 | template 60 | bool merge_clusters(IOManager& mgr, std::string producer, std::string output_producer); 61 | 62 | }; 63 | 64 | /** 65 | \class larcv3::TensorFromClusterFactory 66 | \brief A concrete factory class for larcv3::TensorFromCluster 67 | */ 68 | 69 | 70 | class TensorFromClusterProcessFactory : public ProcessFactoryBase { 71 | public: 72 | /// ctor 73 | TensorFromClusterProcessFactory() { ProcessFactory::get().add_factory("TensorFromCluster",this); } 74 | /// dtor 75 | ~TensorFromClusterProcessFactory() {} 76 | /// creation method 77 | ProcessBase* create(const std::string instance_name) { return new TensorFromCluster(instance_name); } 78 | }; 79 | 80 | 81 | } 82 | 83 | #ifdef LARCV_INTERNAL 84 | #include 85 | void init_tensor_from_cluster(pybind11::module m); 86 | #endif 87 | 88 | #endif 89 | /** @} */ // end of doxygen group 90 | 91 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/Threshold.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Threshold.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class Threshold 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV3_THRESHOLD_H__ 15 | #define __LARCV3_THRESHOLD_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | namespace larcv3 { 20 | 21 | /** 22 | \class ProcessBase 23 | User defined class Threshold ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | class Threshold : public ProcessBase { 27 | 28 | public: 29 | 30 | /// Default constructor 31 | Threshold(const std::string name="Threshold"); 32 | 33 | /// Default destructor 34 | ~Threshold(){} 35 | 36 | void configure(const json&); 37 | 38 | void initialize(); 39 | 40 | bool process(IOManager& mgr); 41 | 42 | void finalize(); 43 | 44 | static json default_config(){ 45 | json c = { 46 | {"Producer", std::string()}, 47 | {"Product", std::string()}, 48 | {"OutputProducer", std::string()}, 49 | {"Threshold", std::vector() = {0.0}}, 50 | }; 51 | return c; 52 | } 53 | 54 | private: 55 | 56 | json config; 57 | 58 | template 59 | bool process_dense_product( 60 | IOManager& mgr, 61 | std::string producer, 62 | std::string output_producer, 63 | const std::vector& threshold); 64 | 65 | template 66 | bool process_sparse_product( 67 | IOManager& mgr, 68 | std::string producer, 69 | std::string output_producer, 70 | const std::vector& threshold); 71 | 72 | }; 73 | 74 | /** 75 | \class larcv3::ThresholdFactory 76 | \brief A concrete factory class for larcv3::Threshold 77 | */ 78 | class ThresholdProcessFactory : public ProcessFactoryBase { 79 | public: 80 | /// ctor 81 | ThresholdProcessFactory() { ProcessFactory::get().add_factory("Threshold",this); } 82 | /// dtor 83 | ~ThresholdProcessFactory() {} 84 | /// creation method 85 | ProcessBase* create(const std::string instance_name) { return new Threshold(instance_name); } 86 | }; 87 | 88 | } 89 | 90 | #ifdef LARCV_INTERNAL 91 | #include 92 | void init_threshold(pybind11::module m); 93 | #endif 94 | 95 | #endif 96 | /** @} */ // end of doxygen group 97 | 98 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/imagemod.cxx: -------------------------------------------------------------------------------- 1 | #include "imagemod.h" 2 | 3 | #include "Downsample.h" 4 | #include "Threshold.h" 5 | #include "TensorFromCluster.h" 6 | #include "DenseToSparse.h" 7 | #include "SparseToDense.h" 8 | #include "BBoxFromParticle.h" 9 | #include "BBoxFromCluster.h" 10 | #include "Embed.h" 11 | #include "Normalize.h" 12 | 13 | 14 | void init_imagemod(pybind11::module m){ 15 | 16 | init_downsample(m); 17 | init_threshold(m); 18 | init_tensor_from_cluster(m); 19 | init_sparse_to_dense(m); 20 | init_dense_to_sparse(m); 21 | init_bbox_from_particle(m); 22 | init_bbox_from_cluster(m); 23 | init_embed(m); 24 | init_normalize(m); 25 | } 26 | -------------------------------------------------------------------------------- /src/larcv3/app/imagemod/imagemod.h: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * \file larbys.h 4 | * 5 | * \ingroup app_imagemod 6 | * 7 | * \brief Class def header for pybind11 for larcv3 framework 8 | * 9 | * @author cadams 10 | */ 11 | 12 | /** \addtogroup core_Base 13 | 14 | @{*/ 15 | #ifndef __LARCV3IMAGEMOD_IMAGEMOD_H__ 16 | #define __LARCV3IMAGEMOD_IMAGEMOD_H__ 17 | 18 | 19 | 20 | #ifndef LARCV_NO_PYBIND 21 | #ifdef LARCV_INTERNAL 22 | #include 23 | __attribute__ ((visibility ("default"))) void init_imagemod(pybind11::module m); 24 | #endif 25 | // bindings 26 | #endif 27 | 28 | 29 | // include guards 30 | #endif 31 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchData.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchData.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchData 7 | * 8 | * @author kazuhiro cadams 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHDATA_H 15 | #define __LARCV3THREADIO_BATCHDATA_H 16 | 17 | #include 18 | #include 19 | #include "QueueIOTypes.h" 20 | #include "larcv3/core/dataformat/DataFormatTypes.h" 21 | 22 | #ifdef LARCV_INTERNAL 23 | #include 24 | #include 25 | #endif 26 | 27 | namespace larcv3 { 28 | /** 29 | \class BatchData 30 | 31 | */ 32 | template 33 | class BatchData { 34 | 35 | public: 36 | 37 | /// Default constructor 38 | BatchData() 39 | : _current_size(0) 40 | , _state(BatchDataState_t::kBatchStateUnknown) 41 | {} 42 | 43 | /// Default destructor 44 | ~BatchData() {} 45 | 46 | /// Read access to the data: 47 | const std::vector& data() const; 48 | 49 | // Writeable access to data: 50 | std::vector & writeable_data() {return _data;} 51 | 52 | #ifdef LARCV_INTERNAL 53 | pybind11::array_t pydata(); 54 | #endif 55 | 56 | inline const std::vector& dim() const { return _dim; } 57 | inline const std::vector& dense_dim() const { return _dense_dim; } 58 | 59 | // Data size is number of elements regardless of the size of each element 60 | size_t data_size(bool calculate=false) const; 61 | 62 | inline size_t current_data_size() const { return _current_size; } 63 | 64 | size_t entry_data_size() const; 65 | 66 | void set_dim(const std::vector& dim); 67 | void set_dense_dim(const std::vector& dense_dim); 68 | void set_entry_data(const std::vector& entry_data); 69 | 70 | void reset(); 71 | void reset_data(); 72 | 73 | inline bool is_filled() const 74 | { return ( _state == BatchDataState_t::kBatchStateFilled ); } 75 | 76 | inline BatchDataState_t state() const 77 | { return _state; } 78 | 79 | private: 80 | // This holds the data for this instance, and is changed often 81 | std::vector _data; 82 | // This holds the dimensions of the container for readout data (including sparse), 83 | // and is static. _data is flattened and this provides reshaping information 84 | std::vector _dim; 85 | // This holds the dense shape of this data, in the case that the data is sparse 86 | // In the case that the data is dense, this matches _dim. 87 | std::vector _dense_dim; 88 | size_t _current_size; 89 | BatchDataState_t _state; 90 | }; 91 | } 92 | 93 | 94 | #ifdef LARCV_INTERNAL 95 | template 96 | void init_batchdata_(pybind11::module m); 97 | 98 | void init_batchdata(pybind11::module m); 99 | #endif 100 | 101 | #endif 102 | /** @} */ // end of doxygen group 103 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchDataQueue.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_BATCHDATAQUEUE_CXX 2 | #define __LARCV3THREADIO_BATCHDATAQUEUE_CXX 3 | 4 | #include "BatchDataQueue.h" 5 | #include "larcv3/core/base/larcv_logger.h" 6 | #include "larcv3/core/base/larbys.h" 7 | #include "larcv3/core/dataformat/Particle.h" 8 | 9 | namespace larcv3 { 10 | 11 | 12 | 13 | 14 | 15 | 16 | template 17 | BatchDataQueue::BatchDataQueue() 18 | { 19 | } 20 | 21 | template 22 | void BatchDataQueue::reset() 23 | { 24 | 25 | } 26 | 27 | 28 | template 29 | bool BatchDataQueue::is_next_ready () const 30 | { 31 | return _data_next.is_filled(); 32 | } 33 | 34 | 35 | template 36 | const BatchData& BatchDataQueue::get_batch () const 37 | { 38 | return _data_current; 39 | } 40 | 41 | template 42 | BatchData& BatchDataQueue::get_next_writeable() 43 | { 44 | 45 | return _data_next; 46 | } 47 | 48 | template 49 | void BatchDataQueue::set_next_data (const std::vector& source) 50 | { 51 | _data_next.set_entry_data(source); 52 | } 53 | 54 | template 55 | void BatchDataQueue::pop(){ 56 | _data_current = std::move(_data_next); 57 | _data_next.reset_data(); 58 | } 59 | 60 | } 61 | 62 | template class larcv3::BatchDataQueue; 63 | template class larcv3::BatchDataQueue; 64 | template class larcv3::BatchDataQueue; 65 | template class larcv3::BatchDataQueue; 66 | template class larcv3::BatchDataQueue; 67 | // template class larcv3::BatchDataQueue>; 68 | 69 | 70 | template 71 | void init_batchdataqueue(pybind11::module m){ 72 | 73 | using Class = larcv3::BatchDataQueue; 74 | std::string classname = "BatchDataQueue" + larcv3::as_string(); 75 | pybind11::class_ batch_data_queue(m, classname.c_str()); 76 | batch_data_queue.def(pybind11::init<>()); 77 | batch_data_queue.def("reset", &Class::reset); 78 | batch_data_queue.def("next_state", &Class::next_state); 79 | batch_data_queue.def("is_next_ready", &Class::is_next_ready); 80 | batch_data_queue.def("get_batch", &Class::get_batch); 81 | batch_data_queue.def("pop", &Class::pop); 82 | 83 | 84 | } 85 | 86 | void init_batchdataqueue(pybind11::module m){ 87 | init_batchdataqueue(m); 88 | init_batchdataqueue(m); 89 | init_batchdataqueue(m); 90 | init_batchdataqueue(m); 91 | init_batchdataqueue(m); 92 | 93 | 94 | } 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchDataQueue.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchDataQueue.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchDataQueue 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHDATAQUEUE_H 15 | #define __LARCV3THREADIO_BATCHDATAQUEUE_H 16 | 17 | #include 18 | #include "BatchData.h" 19 | 20 | namespace larcv3 { 21 | /** 22 | \class BatchDataQueue 23 | User defined class BatchDataQueue ... these comments are used to generate 24 | doxygen documentation! 25 | */ 26 | template 27 | class BatchDataQueue { 28 | 29 | public: 30 | /// Default constructor 31 | BatchDataQueue(); 32 | 33 | /// Default destructor 34 | ~BatchDataQueue(){} 35 | 36 | void reset(); 37 | 38 | // Return detailed state of the next batch 39 | inline BatchDataState_t next_state() const 40 | { return _data_next.state(); } 41 | 42 | // Return whether the next batch of data is ready to go or not: 43 | bool is_next_ready () const; 44 | 45 | // Access to the current batch of data 46 | // Returns const reference and it is expected that _data_current will persist until 47 | // pop is called. 48 | const BatchData& get_batch () const; 49 | 50 | // Pop moves _data_next into _data_current and resets _data_next. 51 | void pop(); 52 | 53 | 54 | // Writeable access to the next batch of data 55 | BatchData& get_next_writeable(); 56 | 57 | // Set the data for the next batch 58 | void set_next_data (const std::vector& source); 59 | 60 | 61 | private: 62 | larcv3::BatchData _data_current; 63 | larcv3::BatchData _data_next; 64 | 65 | }; 66 | 67 | } 68 | 69 | #ifdef LARCV_INTERNAL 70 | #include 71 | template 72 | void init_batchdataqueue(pybind11::module m); 73 | 74 | void init_batchdataqueue(pybind11::module m); 75 | #endif 76 | 77 | 78 | #endif 79 | /** @} */ // end of doxygen group 80 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchDataQueueFactory.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchDataQueueFactory.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchDataQueueFactory 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHDATAQUEUEFACTORY_H 15 | #define __LARCV3THREADIO_BATCHDATAQUEUEFACTORY_H 16 | 17 | #include 18 | #include 19 | #include "BatchDataQueue.h" 20 | #include "larcv3/core/base/larcv_logger.h" 21 | #include "larcv3/core/base/larbys.h" 22 | 23 | namespace larcv3 { 24 | class QueueProcessor; 25 | /** 26 | \class BatchDataQueueFactory 27 | User defined class BatchDataQueueFactory ... these comments are used to generate 28 | doxygen documentation! 29 | */ 30 | template 31 | class BatchDataQueueFactory { 32 | friend QueueProcessor; 33 | public: 34 | 35 | BatchDataQueueFactory() 36 | {} 37 | 38 | ~BatchDataQueueFactory() 39 | {} 40 | 41 | static const BatchDataQueueFactory& get(); 42 | 43 | inline bool exist_queue(std::string name) const 44 | { 45 | auto iter = _queue_m.find(name); 46 | return iter != _queue_m.end(); 47 | } 48 | 49 | bool is_next_ready() const; 50 | 51 | // Pop all queues to promote next to current 52 | void pop_all(); 53 | 54 | const BatchDataQueue& get_queue(std::string name) const; 55 | 56 | inline bool make_queue(std::string name) 57 | { 58 | if (exist_queue(name)) { 59 | LARCV_SERROR() << "Queue name " << name << " already present..." << std::endl; 60 | return false; 61 | } 62 | _queue_m.emplace(std::make_pair(name, BatchDataQueue())); 63 | return true; 64 | } 65 | 66 | BatchDataQueue& get_queue_writeable(std::string name); 67 | 68 | private: 69 | static BatchDataQueueFactory& get_writeable(); 70 | 71 | private: 72 | static BatchDataQueueFactory* _me; 73 | std::map > _queue_m; 74 | }; 75 | 76 | } 77 | 78 | #ifdef LARCV_INTERNAL 79 | #include 80 | template 81 | void init_batchdataqueuefactory(pybind11::module m); 82 | 83 | void init_batchdataqueuefactory(pybind11::module m); 84 | #endif 85 | 86 | #endif 87 | /** @} */ // end of doxygen group 88 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerBBox.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchFillerBBox.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchFillerBBox2D 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHFILLERBBOX_H__ 15 | #define __LARCV3THREADIO_BATCHFILLERBBOX_H__ 16 | 17 | #include "larcv3/core/processor/ProcessFactory.h" 18 | #include "BatchFillerTemplate.h" 19 | 20 | #include "larcv3/core/dataformat/EventBBox.h" 21 | 22 | namespace larcv3 { 23 | 24 | /** 25 | \class ProcessBase 26 | User defined class BatchFillerTensor ... these comments are used to generate 27 | doxygen documentation! 28 | */ 29 | template 30 | class BatchFillerBBox : public BatchFillerTemplate { 31 | 32 | public: 33 | 34 | /// Default constructor 35 | BatchFillerBBox(const std::string name="BatchFillerBBox"); 36 | 37 | /// Default destructor 38 | ~BatchFillerBBox(){} 39 | 40 | void configure(const json&); 41 | 42 | void initialize(); 43 | 44 | bool process(IOManager& mgr); 45 | 46 | void finalize(); 47 | 48 | static json default_config(){ 49 | json c = { 50 | {"Producer", std::string()}, 51 | {"MaxBoxes", 25}, 52 | {"UnfilledBoxValue", 0.0}, 53 | {"Channels", std::vector()}, 54 | }; 55 | return c; 56 | } 57 | protected: 58 | 59 | void _batch_begin_(); 60 | void _batch_end_(); 61 | 62 | private: 63 | 64 | size_t set_data_size(const EventBBox& image_data); 65 | int _check_projection(const size_t & projection_id); 66 | 67 | // std::string _bbox_producer; 68 | // size_t _max_boxes; 69 | // float _unfilled_box_value; 70 | // std::vector _slice_v; 71 | 72 | json config; 73 | 74 | std::vector _entry_data; 75 | }; 76 | 77 | typedef BatchFillerBBox<2> BatchFillerBBox2D; 78 | typedef BatchFillerBBox<3> BatchFillerBBox3D; 79 | 80 | // Template instantiation for IO 81 | template<> inline std::string product_unique_name() { 82 | return "BatchFillerBBox2D"; 83 | } 84 | template<> inline std::string product_unique_name() { 85 | return "BatchFillerBBox3D"; 86 | } 87 | 88 | /** 89 | \class larcv3::BatchFillerBBoxFactory 90 | \brief A concrete factory class for larcv3::BatchFillerBBox 91 | */ 92 | template 93 | class BatchFillerBBoxProcessFactory : public ProcessFactoryBase { 94 | public: 95 | /// ctor 96 | BatchFillerBBoxProcessFactory() { 97 | ProcessFactory::get().add_factory(product_unique_name>(), 98 | this); 99 | } 100 | /// dtor 101 | ~BatchFillerBBoxProcessFactory() {} 102 | /// creation method 103 | ProcessBase* create(const std::string instance_name) { 104 | return new BatchFillerBBox(instance_name); 105 | } 106 | }; 107 | 108 | } 109 | 110 | #ifdef LARCV_INTERNAL 111 | template 112 | void init_bf_bbox_(pybind11::module m); 113 | 114 | void init_bf_bbox(pybind11::module m); 115 | #endif 116 | 117 | #endif 118 | /** @} */ // end of doxygen group 119 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerPIDLabel.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_BATCHFILLERPIDLABEL_CXX__ 2 | #define __LARCV3THREADIO_BATCHFILLERPIDLABEL_CXX__ 3 | 4 | #include "BatchFillerPIDLabel.h" 5 | #include "larcv3/core/dataformat/EventTensor.h" 6 | #include "larcv3/core/dataformat/EventParticle.h" 7 | #include 8 | 9 | namespace larcv3 { 10 | 11 | static BatchFillerPIDLabelProcessFactory __global_BatchFillerPIDLabelProcessFactory__; 12 | 13 | BatchFillerPIDLabel::BatchFillerPIDLabel(const std::string name) 14 | : BatchFillerTemplate(name) 15 | {} 16 | 17 | void BatchFillerPIDLabel::configure(const json& cfg) 18 | { 19 | config = this -> default_config(); 20 | config = augment_default_config(config, cfg); 21 | 22 | 23 | 24 | auto _pdg_list = cfg["PdgClassList"].get >(); 25 | if (_pdg_list.empty()) { 26 | LARCV_CRITICAL() << "PdgClassList needed to define classes!" << std::endl; 27 | throw larbys(); 28 | } 29 | } 30 | 31 | void BatchFillerPIDLabel::initialize() 32 | {} 33 | 34 | void BatchFillerPIDLabel::_batch_begin_() 35 | { 36 | if (!batch_data().dim().empty() && (int)(batch_size()) != batch_data().dim().front()) { 37 | auto dim = batch_data().dim(); 38 | dim[0] = batch_size(); 39 | this->set_dim(dim); 40 | } 41 | } 42 | 43 | void BatchFillerPIDLabel::_batch_end_() 44 | { 45 | 46 | } 47 | 48 | void BatchFillerPIDLabel::finalize() 49 | {} 50 | 51 | bool BatchFillerPIDLabel::process(IOManager & mgr) 52 | { 53 | 54 | std::string producer = config["Producer"].get(); 55 | auto const& event_part = mgr.get_data(producer); 56 | auto _pdg_list = config["PdgClassList"].get >(); 57 | 58 | // Refresh the dimension: 59 | std::vector dim(2); 60 | dim[0] = batch_size(); 61 | dim[1] = _pdg_list.size(); 62 | set_dim(dim); 63 | // In this case, set the dense dim as the same: 64 | set_dense_dim(dim); 65 | 66 | // labels 67 | auto const& part_v = event_part.as_vector(); 68 | if (part_v.size() != 1) { 69 | LARCV_CRITICAL() << "Only support single particle label now: EventParticle size != 1" << std::endl; 70 | throw larbys(); 71 | } 72 | // class 73 | size_t label = kINVALID_SIZE; 74 | int pdg = 0; 75 | for (auto const& part : part_v) { 76 | for (size_t class_idx = 0; class_idx < _pdg_list.size(); ++class_idx) { 77 | pdg = part.pdg_code(); 78 | if (pdg != _pdg_list[class_idx]) continue; 79 | label = class_idx; 80 | break; 81 | } 82 | if (label != kINVALID_SIZE) break; 83 | } 84 | LARCV_DEBUG() << "Found PDG code " << pdg << " (class=" << label << ")" << std::endl; 85 | _entry_data.resize(_pdg_list.size(), 0); 86 | for (auto& v : _entry_data) v = 0; 87 | _entry_data.at(label) = 1.; 88 | 89 | 90 | set_entry_data(_entry_data); 91 | 92 | return true; 93 | } 94 | 95 | } 96 | 97 | #include 98 | #include 99 | 100 | void init_bf_pid(pybind11::module m){ 101 | 102 | using Class = larcv3::BatchFillerPIDLabel; 103 | std::string classname = "BatchFillerPIDLabel"; 104 | pybind11::class_ batch_filler(m, classname.c_str()); 105 | batch_filler.def(pybind11::init<>()); 106 | 107 | // batch_filler.def("pydata", &Class::pydata); 108 | batch_filler.def("default_config", &Class::default_config); 109 | batch_filler.def("process", &Class::process); 110 | 111 | 112 | } 113 | 114 | #endif 115 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerPIDLabel.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchFillerPIDLabel.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchFillerPIDLabel 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHFILLERPIDLABEL_H__ 15 | #define __LARCV3THREADIO_BATCHFILLERPIDLABEL_H__ 16 | 17 | #include "larcv3/core/processor/ProcessFactory.h" 18 | #include "BatchFillerTemplate.h" 19 | 20 | namespace larcv3 { 21 | 22 | /** 23 | \class ProcessBase 24 | User defined class BatchFillerPIDLabel ... these comments are used to generate 25 | doxygen documentation! 26 | */ 27 | class BatchFillerPIDLabel : public BatchFillerTemplate { 28 | 29 | public: 30 | 31 | /// Default constructor 32 | BatchFillerPIDLabel(const std::string name="BatchFillerPIDLabel"); 33 | 34 | /// Default destructor 35 | ~BatchFillerPIDLabel(){} 36 | 37 | void configure(const json&); 38 | 39 | void initialize(); 40 | 41 | bool process(IOManager& mgr); 42 | 43 | void _batch_begin_(); 44 | 45 | void _batch_end_(); 46 | 47 | void finalize(); 48 | 49 | static json default_config(){ 50 | json c = { 51 | {"Producer", std::string()}, 52 | {"PdgClassList", std::vector()}, 53 | }; 54 | return c; 55 | } 56 | 57 | private: 58 | 59 | json config; 60 | 61 | std::vector _entry_data; 62 | }; 63 | 64 | /** 65 | \class larcv3::BatchFillerPIDLabelFactory 66 | \brief A concrete factory class for larcv3::BatchFillerPIDLabel 67 | */ 68 | class BatchFillerPIDLabelProcessFactory : public ProcessFactoryBase { 69 | public: 70 | /// ctor 71 | BatchFillerPIDLabelProcessFactory() { ProcessFactory::get().add_factory("BatchFillerPIDLabel",this); } 72 | /// dtor 73 | ~BatchFillerPIDLabelProcessFactory() {} 74 | /// creation method 75 | ProcessBase* create(const std::string instance_name) { return new BatchFillerPIDLabel(instance_name); } 76 | }; 77 | 78 | } 79 | 80 | #ifdef LARCV_INTERNAL 81 | void init_bf_pid(pybind11::module m); 82 | #endif 83 | 84 | #endif 85 | /** @} */ // end of doxygen group 86 | 87 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerParticle.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_BATCHFILLERPARTICLE_CXX__ 2 | #define __LARCV3THREADIO_BATCHFILLERPARTICLE_CXX__ 3 | 4 | #include "BatchFillerParticle.h" 5 | #include 6 | 7 | namespace larcv3 { 8 | 9 | static BatchFillerParticleProcessFactory __global_BatchFillerParticleProcessFactory__; 10 | 11 | BatchFillerParticle::BatchFillerParticle(const std::string name) 12 | : BatchFillerTemplate(name) 13 | {} 14 | 15 | void BatchFillerParticle::configure(const json& cfg){ 16 | config = this -> default_config(); 17 | config = augment_default_config(config, cfg); 18 | } 19 | 20 | void BatchFillerParticle::initialize(){} 21 | 22 | bool BatchFillerParticle::process(IOManager& mgr){ 23 | 24 | std::string producer = config["Producer"].get(); 25 | size_t max_particles = config["MaxParticles"].get(); 26 | // Fetch the particles: 27 | auto const& event_part = mgr.get_data(producer); 28 | 29 | // Refresh the dimension: 30 | std::vector dim(2); 31 | dim[0] = batch_size(); 32 | dim[1] = max_particles; 33 | set_dim(dim); 34 | 35 | 36 | // // labels 37 | // auto const& part_v = event_part.as_vector(); 38 | // if (part_v.size() != 1) { 39 | // LARCV_CRITICAL() << "Only support single particle label now: EventParticle size != 1" << std::endl; 40 | // throw larbys(); 41 | // } 42 | // // class 43 | 44 | _entry_data.resize(max_particles); 45 | size_t i = 0; 46 | for (auto const& part : event_part.as_vector()){ 47 | if (i < max_particles){ 48 | _entry_data.at(i) = part._particle_holder; 49 | } 50 | else{ 51 | break; 52 | } 53 | i ++; 54 | } 55 | 56 | 57 | set_entry_data(_entry_data); 58 | 59 | return true; 60 | 61 | 62 | } 63 | 64 | void BatchFillerParticle::_batch_begin_(){ 65 | if (!batch_data().dim().empty() && (int)(batch_size()) != batch_data().dim().front()) { 66 | auto dim = batch_data().dim(); 67 | dim[0] = batch_size(); 68 | this->set_dim(dim); 69 | } 70 | 71 | } 72 | 73 | void BatchFillerParticle::_batch_end_(){} 74 | 75 | void BatchFillerParticle::finalize(){} 76 | 77 | } 78 | 79 | #include 80 | #include 81 | 82 | void init_bf_particle(pybind11::module m){ 83 | 84 | using Class = larcv3::BatchFillerParticle; 85 | std::string classname = "BatchFillerParticle"; 86 | pybind11::class_ batch_filler(m, classname.c_str()); 87 | batch_filler.def(pybind11::init<>()); 88 | 89 | // batch_filler.def("pydata", &Class::pydata); 90 | batch_filler.def("default_config", &Class::default_config); 91 | batch_filler.def("process", &Class::process); 92 | 93 | 94 | } 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerParticle.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchFillerParticle.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchFillerParticle 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHFILLERPARTICLE_H__ 15 | #define __LARCV3THREADIO_BATCHFILLERPARTICLE_H__ 16 | 17 | #include "larcv3/core/processor/ProcessFactory.h" 18 | #include "BatchFillerTemplate.h" 19 | #include "larcv3/core/dataformat/EventParticle.h" 20 | 21 | namespace larcv3 { 22 | 23 | /** 24 | \class ProcessBase 25 | User defined class BatchFillerParticle ... these comments are used to generate 26 | doxygen documentation! 27 | */ 28 | class BatchFillerParticle : public BatchFillerTemplate { 29 | 30 | public: 31 | 32 | /// Default constructor 33 | BatchFillerParticle(const std::string name="BatchFillerParticle"); 34 | 35 | /// Default destructor 36 | ~BatchFillerParticle(){} 37 | 38 | void configure(const json&); 39 | 40 | void initialize(); 41 | 42 | bool process(IOManager& mgr); 43 | 44 | void _batch_begin_(); 45 | 46 | void _batch_end_(); 47 | 48 | void finalize(); 49 | 50 | static json default_config(){ 51 | json c = { 52 | {"Producer", std::string()}, 53 | {"MaxParticles", size_t(30)} 54 | }; 55 | return c; 56 | } 57 | 58 | private: 59 | 60 | json config; 61 | 62 | std::vector _entry_data; 63 | }; 64 | 65 | /** 66 | \class larcv3::BatchFillerParticleFactory 67 | \brief A concrete factory class for larcv3::BatchFillerParticle 68 | */ 69 | class BatchFillerParticleProcessFactory : public ProcessFactoryBase { 70 | public: 71 | /// ctor 72 | BatchFillerParticleProcessFactory() { ProcessFactory::get().add_factory("BatchFillerParticle",this); } 73 | /// dtor 74 | ~BatchFillerParticleProcessFactory() {} 75 | /// creation method 76 | ProcessBase* create(const std::string instance_name) { return new BatchFillerParticle(instance_name); } 77 | }; 78 | 79 | } 80 | 81 | #ifdef LARCV_INTERNAL 82 | 83 | void init_bf_particle(pybind11::module m); 84 | #endif 85 | 86 | #endif 87 | /** @} */ // end of doxygen group 88 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerSparseTensor.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchFillerSparseTensor.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchFillerSparseTensor2D 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHFILLERSPARSETENSOR_H__ 15 | #define __LARCV3THREADIO_BATCHFILLERSPARSETENSOR_H__ 16 | 17 | #include "larcv3/core/processor/ProcessFactory.h" 18 | #include "BatchFillerTemplate.h" 19 | 20 | #include "larcv3/core/dataformat/EventSparseTensor.h" 21 | 22 | namespace larcv3 { 23 | 24 | /** 25 | \class ProcessBase 26 | User defined class BatchFillerTensor ... these comments are used to generate 27 | doxygen documentation! 28 | */ 29 | template 30 | class BatchFillerSparseTensor : public BatchFillerTemplate { 31 | 32 | public: 33 | 34 | /// Default constructor 35 | BatchFillerSparseTensor(const std::string name="BatchFillerSparseTensor"); 36 | 37 | /// Default destructor 38 | ~BatchFillerSparseTensor(){} 39 | 40 | void configure(const json&); 41 | 42 | void initialize(); 43 | 44 | bool process(IOManager& mgr); 45 | 46 | void finalize(); 47 | 48 | static json default_config(){ 49 | json c = { 50 | {"Producer", std::string()}, 51 | {"Augment", true}, 52 | {"MaxVoxels", 0}, 53 | {"UnfilledVoxelValue", -999.}, 54 | {"Channels", std::vector()}, 55 | {"IncludeValues", true}, 56 | }; 57 | return c; 58 | } 59 | 60 | protected: 61 | 62 | void _batch_begin_(); 63 | void _batch_end_(); 64 | 65 | private: 66 | 67 | json config; 68 | 69 | size_t set_data_size(const EventSparseTensor& image_data); 70 | int _check_projection(const int & projection_id, const std::vector & _slice_v); 71 | 72 | // std::string _tensor_producer; 73 | // size_t _max_voxels; 74 | // float _unfilled_voxel_value; 75 | // std::vector _slice_v; 76 | 77 | 78 | 79 | std::vector _entry_data; 80 | size_t _num_channels; 81 | bool _allow_empty; 82 | // bool _include_values; 83 | // bool _augment; 84 | }; 85 | 86 | typedef BatchFillerSparseTensor<2> BatchFillerSparseTensor2D; 87 | typedef BatchFillerSparseTensor<3> BatchFillerSparseTensor3D; 88 | 89 | // Template instantiation for IO 90 | template<> inline std::string product_unique_name() { 91 | return "BatchFillerSparseTensor2D"; 92 | } 93 | template<> inline std::string product_unique_name() { 94 | return "BatchFillerSparseTensor3D"; 95 | } 96 | 97 | /** 98 | \class larcv3::BatchFillerSparseTensorFactory 99 | \brief A concrete factory class for larcv3::BatchFillerSparseTensor 100 | */ 101 | template 102 | class BatchFillerSparseTensorProcessFactory : public ProcessFactoryBase { 103 | public: 104 | /// ctor 105 | BatchFillerSparseTensorProcessFactory() { 106 | ProcessFactory::get().add_factory(product_unique_name>(), 107 | this); 108 | } 109 | /// dtor 110 | ~BatchFillerSparseTensorProcessFactory() {} 111 | /// creation method 112 | ProcessBase* create(const std::string instance_name) { 113 | return new BatchFillerSparseTensor(instance_name); 114 | } 115 | }; 116 | 117 | } 118 | 119 | 120 | #ifdef LARCV_INTERNAL 121 | template 122 | void init_bf_sparse_tensor_(pybind11::module m); 123 | 124 | void init_bf_sparse_tensor(pybind11::module m); 125 | #endif 126 | 127 | 128 | #endif 129 | /** @} */ // end of doxygen group 130 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerTemplate.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_BATCHFILLERTEMPLATE_CXX__ 2 | #define __LARCV3THREADIO_BATCHFILLERTEMPLATE_CXX__ 3 | 4 | #include "BatchFillerTemplate.h" 5 | #include "larcv3/core/dataformat/Particle.h" 6 | #include 7 | 8 | namespace larcv3 { 9 | 10 | template<> BatchDataType_t BatchFillerTemplate< short >::data_type() const { return BatchDataType_t::kBatchDataShort; } 11 | template<> BatchDataType_t BatchFillerTemplate< int >::data_type() const { return BatchDataType_t::kBatchDataInt; } 12 | template<> BatchDataType_t BatchFillerTemplate< float >::data_type() const { return BatchDataType_t::kBatchDataFloat; } 13 | template<> BatchDataType_t BatchFillerTemplate< double >::data_type() const { return BatchDataType_t::kBatchDataDouble; } 14 | template<> BatchDataType_t BatchFillerTemplate< larcv3::ParticleHolder>::data_type() const { return BatchDataType_t::kBatchDataParticle;} 15 | 16 | 17 | 18 | template class BatchFillerTemplate; 19 | template class BatchFillerTemplate; 20 | template class BatchFillerTemplate; 21 | template class BatchFillerTemplate; 22 | template class BatchFillerTemplate; 23 | 24 | 25 | 26 | } 27 | #endif 28 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerTemplate.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchFillerTemplate.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchFillerTemplate 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHFILLERTEMPLATE_H__ 15 | #define __LARCV3THREADIO_BATCHFILLERTEMPLATE_H__ 16 | 17 | #include "BatchHolder.h" 18 | #include "BatchData.h" 19 | 20 | namespace larcv3 { 21 | class ThreadProcessor; 22 | class QueueProcessor; 23 | /** 24 | \class ProcessBase 25 | User defined class BatchFillerTemplate ... these comments are used to generate 26 | doxygen documentation! 27 | */ 28 | template 29 | class BatchFillerTemplate : public BatchHolder { 30 | friend ThreadProcessor; 31 | friend QueueProcessor; 32 | public: 33 | 34 | /// Default constructor 35 | BatchFillerTemplate(const std::string name = "BatchFillerTemplate") 36 | : BatchHolder(name) 37 | {} 38 | 39 | /// Default destructor 40 | virtual ~BatchFillerTemplate() {} 41 | 42 | inline void batch_begin() { 43 | _batch_data_ptr->reset_data(); 44 | _batch_begin_(); 45 | } 46 | 47 | inline void batch_end() 48 | { 49 | if (!_batch_data_ptr->is_filled()) { 50 | LARCV_CRITICAL() << "Batch data is not filled @ end-of-batch (" << _batch_data_ptr->current_data_size() 51 | << "/" << _batch_data_ptr->data_size() << ")!" << std::endl; 52 | throw larbys(); 53 | } 54 | _batch_end_(); 55 | } 56 | 57 | inline const BatchData& batch_data() const { return *(_batch_data_ptr); } 58 | 59 | BatchDataType_t data_type() const; 60 | 61 | inline size_t type_size() const { return sizeof(T); } 62 | 63 | protected: 64 | 65 | inline void set_dim(std::vector dim) {_batch_data_ptr->set_dim(dim);} 66 | inline void set_dense_dim(std::vector dense_dim) {_batch_data_ptr->set_dense_dim(dense_dim);} 67 | inline void set_entry_data(const std::vector& data) 68 | { _batch_data_ptr->set_entry_data(data); } 69 | 70 | virtual void _batch_begin_() =0; 71 | virtual void _batch_end_() =0; 72 | 73 | private: 74 | 75 | BatchData* _batch_data_ptr; 76 | 77 | }; 78 | 79 | } 80 | 81 | #endif 82 | /** @} */ // end of doxygen group 83 | 84 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchFillerTensor.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchFillerTensor.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchFillerTensor2D 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHFILLERTENSOR_H__ 15 | #define __LARCV3THREADIO_BATCHFILLERTENSOR_H__ 16 | 17 | #include "larcv3/core/processor/ProcessFactory.h" 18 | #include "BatchFillerTemplate.h" 19 | 20 | #include "larcv3/core/dataformat/EventSparseTensor.h" 21 | #include "larcv3/core/dataformat/EventTensor.h" 22 | 23 | namespace larcv3 { 24 | 25 | /** 26 | \class ProcessBase 27 | User defined class BatchFillerTensor ... these comments are used to generate 28 | doxygen documentation! 29 | */ 30 | template 31 | class BatchFillerTensor : public BatchFillerTemplate { 32 | 33 | public: 34 | 35 | /// Default constructor 36 | BatchFillerTensor(const std::string name="BatchFillerTensor"); 37 | 38 | /// Default destructor 39 | ~BatchFillerTensor(){} 40 | 41 | void configure(const json&); 42 | 43 | void initialize(); 44 | 45 | bool process(IOManager& mgr); 46 | 47 | void finalize(); 48 | 49 | static json default_config(){ 50 | json c = { 51 | {"Producer", std::string()}, 52 | {"TensorType", "sparse"}, 53 | {"Augment", true}, 54 | {"EmptyVoxelValue", 0.0}, 55 | {"Channels", std::vector()}, 56 | {"AllowEmpty", true} 57 | }; 58 | return c; 59 | } 60 | 61 | protected: 62 | 63 | void _batch_begin_(); 64 | void _batch_end_(); 65 | 66 | private: 67 | 68 | json config; 69 | 70 | bool _process_sparse(IOManager& mgr); 71 | bool _process_dense(IOManager& mgr); 72 | int _check_projection(const size_t & projection_id); 73 | size_t _set_image_size(const EventTensor& image_data); 74 | void _assert_dimension(const EventTensor& image_data, const std::vector &) const; 75 | 76 | // std::string _tensor_producer; 77 | // std::string _tensor_type; 78 | // size_t _rows; 79 | // size_t _cols; 80 | size_t _dims[dimension]; ///< Total number of voxels in each dimension 81 | // size_t _num_channels; 82 | // std::vector _slice_v; 83 | size_t _max_ch; 84 | 85 | std::vector _entry_data; 86 | }; 87 | 88 | typedef BatchFillerTensor<2> BatchFillerTensor2D; 89 | typedef BatchFillerTensor<3> BatchFillerTensor3D; 90 | 91 | // Template instantiation for IO 92 | template<> inline std::string product_unique_name() { 93 | return "BatchFillerTensor2D"; 94 | } 95 | template<> inline std::string product_unique_name() { 96 | return "BatchFillerTensor3D"; 97 | } 98 | 99 | /** 100 | \class larcv3::BatchFillerTensorFactory 101 | \brief A concrete factory class for larcv3::BatchFillerTensor 102 | */ 103 | template 104 | class BatchFillerTensorProcessFactory : public ProcessFactoryBase { 105 | public: 106 | /// ctor 107 | BatchFillerTensorProcessFactory() { 108 | ProcessFactory::get().add_factory(product_unique_name>(), 109 | this); 110 | } 111 | /// dtor 112 | ~BatchFillerTensorProcessFactory() {} 113 | /// creation method 114 | ProcessBase* create(const std::string instance_name) { 115 | return new BatchFillerTensor(instance_name); 116 | } 117 | }; 118 | 119 | 120 | } 121 | 122 | #ifdef LARCV_INTERNAL 123 | template 124 | void init_bf_tensor_(pybind11::module m); 125 | 126 | void init_bf_tensor(pybind11::module m); 127 | #endif 128 | 129 | 130 | #endif 131 | 132 | 133 | /** @} */ // end of doxygen group 134 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchHolder.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_BATCHHOLDER_CXX__ 2 | #define __LARCV3THREADIO_BATCHHOLDER_CXX__ 3 | 4 | #include "BatchHolder.h" 5 | #include 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/BatchHolder.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file BatchHolder.h 3 | * 4 | * \ingroup ThreadIO 5 | * 6 | * \brief Class def header for a class BatchHolder 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup ThreadIO 12 | 13 | @{*/ 14 | #ifndef __LARCV3THREADIO_BATCHHOLDER_H__ 15 | #define __LARCV3THREADIO_BATCHHOLDER_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "QueueIOTypes.h" 19 | namespace larcv3 { 20 | class QueueProcessor; 21 | 22 | /** 23 | \class ProcessBase 24 | User defined class BatchHolder ... these comments are used to generate 25 | doxygen documentation! 26 | */ 27 | class BatchHolder : public ProcessBase{ 28 | friend class QueueProcessor; 29 | public: 30 | 31 | /// Default constructor 32 | BatchHolder(const std::string name="BatchFiller") 33 | : ProcessBase(name) 34 | , _batch_size(0) 35 | {} 36 | 37 | /// Default destructor 38 | virtual ~BatchHolder(){} 39 | 40 | inline size_t batch_size() const { return _batch_size; } 41 | 42 | virtual BatchDataType_t data_type() const = 0; 43 | 44 | inline bool is(const std::string question) const 45 | { return (question == "BatchFiller"); } 46 | 47 | private: 48 | size_t _batch_size; 49 | }; 50 | 51 | } 52 | 53 | #endif 54 | /** @} */ // end of doxygen group 55 | 56 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name queueio) 2 | 3 | # Get all the source files: 4 | file(GLOB SOURCES *.cxx) 5 | file(GLOB HEADERS *.h) 6 | 7 | # Add a shared library 8 | add_library(${name} OBJECT ${SOURCES}) 9 | 10 | 11 | 12 | install (FILES ${HEADERS} 13 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/app/${name}) 14 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/QueueIOTypes.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_THREADIOTYPES_CXX 2 | #define __LARCV3THREADIO_THREADIOTYPES_CXX 3 | 4 | #include "QueueIOTypes.h" 5 | 6 | namespace larcv3{ 7 | 8 | std::string BatchDataTypeName(BatchDataType_t type) { 9 | switch(type) { 10 | case BatchDataType_t::kBatchDataChar: 11 | return std::string("char"); 12 | case BatchDataType_t::kBatchDataShort: 13 | return std::string("short"); 14 | case BatchDataType_t::kBatchDataInt: 15 | return std::string("int"); 16 | case BatchDataType_t::kBatchDataFloat: 17 | return std::string("float32"); 18 | case BatchDataType_t::kBatchDataDouble: 19 | return std::string("float64"); 20 | case BatchDataType_t::kBatchDataString: 21 | return std::string("string"); 22 | case BatchDataType_t::kBatchDataParticle: 23 | return std::string("particle"); 24 | default: 25 | return std::string(""); 26 | } 27 | } 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/QueueIOTypes.h: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3THREADIO_THREADIOTYPES_H 2 | #define __LARCV3THREADIO_THREADIOTYPES_H 3 | 4 | #include 5 | 6 | namespace larcv3 { 7 | 8 | // enum ThreadFillerState_t : int{ 9 | // kThreadStateIdle, 10 | // kThreadStateStarting, 11 | // kThreadStateRunning, 12 | // kThreadStateUnknown 13 | // }; 14 | 15 | enum class BatchDataType_t : int { 16 | kBatchDataUnknown, 17 | kBatchDataChar, 18 | kBatchDataShort, 19 | kBatchDataInt, 20 | kBatchDataFloat, 21 | kBatchDataDouble, 22 | kBatchDataString, 23 | kBatchDataParticle, 24 | }; 25 | 26 | std::string BatchDataTypeName(BatchDataType_t type); 27 | 28 | enum class BatchDataState_t :int { 29 | kBatchStateUnknown, 30 | kBatchStateEmpty, 31 | kBatchStateFilling, 32 | kBatchStateFilled, 33 | kBatchStateReleased 34 | }; 35 | } 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/queueio.cxx: -------------------------------------------------------------------------------- 1 | #include "queueio.h" 2 | #include "BatchFillerPIDLabel.h" 3 | #include "BatchFillerBBox.h" 4 | #include "BatchFillerParticle.h" 5 | #include "BatchFillerSparseTensor.h" 6 | #include "BatchFillerTensor.h" 7 | 8 | 9 | void init_queueio(pybind11::module m){ 10 | 11 | 12 | 13 | pybind11::enum_ batchdatatype_t(m,"BatchDataType_t"); 14 | batchdatatype_t.value("kBatchDataUnknown", larcv3::BatchDataType_t::kBatchDataUnknown); 15 | batchdatatype_t.value("kBatchDataChar", larcv3::BatchDataType_t::kBatchDataChar); 16 | batchdatatype_t.value("kBatchDataShort", larcv3::BatchDataType_t::kBatchDataShort); 17 | batchdatatype_t.value("kBatchDataInt", larcv3::BatchDataType_t::kBatchDataInt); 18 | batchdatatype_t.value("kBatchDataFloat", larcv3::BatchDataType_t::kBatchDataFloat); 19 | batchdatatype_t.value("kBatchDataDouble", larcv3::BatchDataType_t::kBatchDataDouble); 20 | batchdatatype_t.value("kBatchDataString", larcv3::BatchDataType_t::kBatchDataString); 21 | batchdatatype_t.export_values(); 22 | 23 | m.def("BatchDataTypeName", &larcv3::BatchDataTypeName); 24 | 25 | 26 | pybind11::enum_ batchdatastate_t(m,"BatchDataState_t"); 27 | batchdatastate_t.value("kBatchStateUnknown", larcv3::BatchDataState_t::kBatchStateUnknown); 28 | batchdatastate_t.value("kBatchStateEmpty", larcv3::BatchDataState_t::kBatchStateEmpty); 29 | batchdatastate_t.value("kBatchStateFilling", larcv3::BatchDataState_t::kBatchStateFilling); 30 | batchdatastate_t.value("kBatchStateFilled", larcv3::BatchDataState_t::kBatchStateFilled); 31 | batchdatastate_t.value("kBatchStateReleased", larcv3::BatchDataState_t::kBatchStateReleased); 32 | batchdatastate_t.export_values(); 33 | 34 | 35 | init_batchdata(m); 36 | init_batchdataqueue(m); 37 | init_batchdataqueuefactory(m); 38 | init_queueprocessor(m); 39 | 40 | init_bf_pid(m); 41 | init_bf_bbox(m); 42 | init_bf_particle(m); 43 | init_bf_sparse_tensor(m); 44 | init_bf_tensor(m); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/larcv3/app/queueio/queueio.h: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * \file larbys.h 4 | * 5 | * \ingroup core_Base 6 | * 7 | * \brief Class def header for exception classes for larcv3 framework 8 | * 9 | * @author cadams 10 | */ 11 | 12 | /** \addtogroup core_Base 13 | 14 | @{*/ 15 | #ifndef __LARCV3QUEUEIO_QUEUEIO_H__ 16 | #define __LARCV3QUEUEIO_QUEUEIO_H__ 17 | 18 | 19 | #include "BatchData.h" 20 | #include "BatchDataQueue.h" 21 | #include "BatchDataQueueFactory.h" 22 | #include "QueueIOTypes.h" 23 | #include "QueueProcessor.h" 24 | 25 | #ifndef LARCV_NO_PYBIND 26 | #ifdef LARCV_INTERNAL 27 | #include 28 | __attribute__ ((visibility ("default"))) void init_queueio(pybind11::module m); 29 | #endif 30 | // bindings 31 | #endif 32 | 33 | 34 | // include guards 35 | #endif 36 | -------------------------------------------------------------------------------- /src/larcv3/app/sbnd_imagemod/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name sbnd_imagemod) 2 | 3 | 4 | # Get all the source files: 5 | file(GLOB SOURCES *.cxx) 6 | file(GLOB HEADERS *.h) 7 | 8 | # Add a shared library 9 | add_library(${name} OBJECT ${SOURCES}) 10 | 11 | 12 | 13 | 14 | 15 | install (FILES ${HEADERS} 16 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/app/${name}) 17 | -------------------------------------------------------------------------------- /src/larcv3/app/sbnd_imagemod/CosmicNeutrinoSegLabel.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file CosmicNeutrinoSegLabel.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class CosmicNeutrinoSegLabel 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup ImageMod 12 | 13 | @{*/ 14 | #ifndef __LARCV_COSMICNEUTRINOSEGLABEL_H__ 15 | #define __LARCV_COSMICNEUTRINOSEGLABEL_H__ 16 | 17 | #include "larcv3/core/processor/ProcessBase.h" 18 | #include "larcv3/core/processor/ProcessFactory.h" 19 | #include "larcv3/core/dataformat/Tensor.h" 20 | #include "larcv3/core/dataformat/Particle.h" 21 | #include "larcv3/core/dataformat/EventSparseCluster.h" 22 | 23 | namespace larcv3 { 24 | 25 | /** 26 | \class ProcessBase 27 | User defined class CosmicNeutrinoSegLabel ... these comments are 28 | used to generate 29 | doxygen documentation! 30 | */ 31 | class CosmicNeutrinoSegLabel : public ProcessBase { 32 | public: 33 | 34 | enum particleLabel {kBackground, kCosmic, kNeutrino}; 35 | 36 | /// Default constructor 37 | CosmicNeutrinoSegLabel( 38 | const std::string name = "CosmicNeutrinoSegLabel"); 39 | 40 | /// Default destructor 41 | ~CosmicNeutrinoSegLabel() {} 42 | 43 | void configure(const json&); 44 | 45 | void initialize(); 46 | 47 | bool process(IOManager& mgr); 48 | 49 | void finalize(); 50 | 51 | template 52 | SparseTensor seg_image_creator(const std::vector & particles, 53 | const SparseCluster & clusters, 54 | const ImageMeta & meta, 55 | const int neutrino_label, 56 | const int cosmic_label); 57 | static json default_config(){ 58 | json c = { 59 | {"Cluster2dProducer", ""}, 60 | {"Cluster3dProducer", ""}, 61 | {"OutputProducer", ""}, 62 | {"ParticleProducer", ""}, 63 | {"NeutrinoLabel", 1}, 64 | {"CosmicLabel", 2} 65 | }; 66 | return c; 67 | } 68 | 69 | private: 70 | 71 | json config; 72 | 73 | 74 | }; 75 | 76 | /** 77 | \class larcv3::CosmicNeutrinoSegLabelFactory 78 | \brief A concrete factory class for larcv3::CosmicNeutrinoSegLabel 79 | */ 80 | class CosmicNeutrinoSegLabelProcessFactory 81 | : public ProcessFactoryBase { 82 | public: 83 | /// ctor 84 | CosmicNeutrinoSegLabelProcessFactory() { 85 | ProcessFactory::get().add_factory("CosmicNeutrinoSegLabel", 86 | this); 87 | } 88 | /// dtor 89 | ~CosmicNeutrinoSegLabelProcessFactory() {} 90 | /// creation method 91 | ProcessBase* create(const std::string instance_name) { 92 | return new CosmicNeutrinoSegLabel(instance_name); 93 | } 94 | }; 95 | } 96 | 97 | #endif 98 | /** @} */ // end of doxygen group 99 | -------------------------------------------------------------------------------- /src/larcv3/app/sbnd_imagemod/ParentParticleSeg.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file ParentParticleSeg.h 3 | * 4 | * \ingroup ImageMod 5 | * 6 | * \brief Class def header for a class ParentParticleSeg 7 | * 8 | * Merge all cluster2d voxel sets into a single set if they have a common ancestor 9 | * @author cadams 10 | */ 11 | 12 | /** \addtogroup ImageMod 13 | 14 | @{*/ 15 | #ifndef __PARENTPARTICLESEG_H__ 16 | #define __PARENTPARTICLESEG_H__ 17 | 18 | #include "larcv3/core/processor/ProcessBase.h" 19 | #include "larcv3/core/processor/ProcessFactory.h" 20 | #include "larcv3/core/dataformat/Tensor.h" 21 | #include "larcv3/core/dataformat/Particle.h" 22 | #include "larcv3/core/dataformat/Voxel.h" 23 | 24 | namespace larcv3 { 25 | 26 | /** 27 | \class ProcessBase 28 | User defined class ParentParticleSeg ... these comments are 29 | used to generate 30 | doxygen documentation! 31 | */ 32 | 33 | struct particle_node{ 34 | int trackID; 35 | int parentID; 36 | int ancestorID; 37 | bool primary; 38 | bool is_virtual; 39 | const Particle * reference; 40 | particle_node * parent; 41 | std::vector daughters; 42 | }; 43 | 44 | class ParentParticleSeg : public ProcessBase { 45 | public: 46 | /// Default constructor 47 | ParentParticleSeg( 48 | const std::string name = "ParentParticleSeg"); 49 | 50 | /// Default destructor 51 | ~ParentParticleSeg() {} 52 | 53 | void configure(const json&); 54 | 55 | void initialize(); 56 | 57 | bool process(IOManager& mgr); 58 | 59 | void finalize(); 60 | 61 | larcv3::VoxelSet cluster_merger(const larcv3::SparseCluster2D & clusters, 62 | particle_node * primary_node); 63 | 64 | larcv3::VoxelSet cluster_merger(const larcv3::SparseCluster3D & clusters, 65 | particle_node * primary_node); 66 | 67 | static json default_config(){ 68 | json c = { 69 | {"Cluster2dProducer", ""}, 70 | {"Cluster3dProducer", ""}, 71 | {"OutputProducer", ""}, 72 | {"ParticleProducer", ""}, 73 | }; 74 | return c; 75 | } 76 | 77 | private: 78 | 79 | json config; 80 | 81 | void get_all_daughter_ids(std::vector & ids, const particle_node * node); 82 | 83 | 84 | // std::string _cluster3d_producer; 85 | // std::string _cluster2d_producer; 86 | // std::string _output_producer; 87 | // std::string _particle_producer; 88 | 89 | }; 90 | 91 | /** 92 | \class larcv3::ParentParticleSegFactory 93 | \brief A concrete factory class for larcv3::ParentParticleSeg 94 | */ 95 | class ParentParticleSegProcessFactory 96 | : public ProcessFactoryBase { 97 | public: 98 | /// ctor 99 | ParentParticleSegProcessFactory() { 100 | ProcessFactory::get().add_factory("ParentParticleSeg", 101 | this); 102 | } 103 | /// dtor 104 | ~ParentParticleSegProcessFactory() {} 105 | /// creation method 106 | ProcessBase* create(const std::string instance_name) { 107 | return new ParentParticleSeg(instance_name); 108 | } 109 | }; 110 | } 111 | 112 | #ifdef LARCV_INTERNAL 113 | #include 114 | void init_parent_particle_seg(pybind11::module m); 115 | #endif 116 | 117 | 118 | #endif 119 | /** @} */ // end of doxygen group 120 | -------------------------------------------------------------------------------- /src/larcv3/app/sbnd_imagemod/sbnd_imagemod.cxx: -------------------------------------------------------------------------------- 1 | #include "sbnd_imagemod.h" 2 | 3 | #include "ParentParticleSeg.h" 4 | 5 | void init_sbnd_imagemod(pybind11::module m){ 6 | init_parent_particle_seg(m); 7 | } 8 | -------------------------------------------------------------------------------- /src/larcv3/app/sbnd_imagemod/sbnd_imagemod.h: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * \file larbys.h 4 | * 5 | * \ingroup app_imagemod 6 | * 7 | * \brief Class def header for pybind11 for larcv3 framework 8 | * 9 | * @author cadams 10 | */ 11 | 12 | /** \addtogroup core_Base 13 | 14 | @{*/ 15 | #ifndef __LARCV3SBND_IMAGEMOD_IMAGEMOD_H__ 16 | #define __LARCV3SBND_IMAGEMOD_IMAGEMOD_H__ 17 | 18 | 19 | 20 | #ifndef LARCV_NO_PYBIND 21 | #ifdef LARCV_INTERNAL 22 | #include 23 | __attribute__ ((visibility ("default"))) void init_sbnd_imagemod(pybind11::module m); 24 | #endif 25 | // bindings 26 | #endif 27 | 28 | 29 | // include guards 30 | #endif 31 | -------------------------------------------------------------------------------- /src/larcv3/core/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(base) 2 | add_subdirectory(dataformat) 3 | add_subdirectory(processor) 4 | -------------------------------------------------------------------------------- /src/larcv3/core/base/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name base) 2 | 3 | 4 | # Get all the source files: 5 | file(GLOB SOURCES *.cxx) 6 | file(GLOB HEADERS *.h) 7 | 8 | # Add a shared library 9 | add_library(${name} OBJECT ${SOURCES}) 10 | 11 | 12 | install (FILES ${HEADERS} 13 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/core/${name}) 14 | -------------------------------------------------------------------------------- /src/larcv3/core/base/LArCVBaseUtilFunc.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3BASE_UTILFUNC_CXX__ 2 | #define __LARCV3BASE_UTILFUNC_CXX__ 3 | 4 | #include "larcv3/core/base/LArCVBaseUtilFunc.h" 5 | #include 6 | #include 7 | namespace larcv3 { 8 | 9 | 10 | bool mpi_enabled(){ 11 | #ifdef LARCV_MPI 12 | return true; 13 | #else 14 | return false; 15 | #endif 16 | } 17 | 18 | } 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /src/larcv3/core/base/LArCVBaseUtilFunc.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file LArCVBaseUtilFunc.h 3 | * 4 | * \ingroup core_Base 5 | * 6 | * \brief Utility functions in core_Base 7 | * 8 | * @author Kazu - Nevis 2015 9 | */ 10 | 11 | /** \addtogroup core_Base 12 | 13 | @{*/ 14 | 15 | #ifndef __LARCV3BASE_UTILFUNC_H__ 16 | #define __LARCV3BASE_UTILFUNC_H__ 17 | 18 | namespace larcv3 { 19 | 20 | bool mpi_enabled(); 21 | 22 | } 23 | 24 | #endif 25 | /** @} */ // end of doxygen group 26 | -------------------------------------------------------------------------------- /src/larcv3/core/base/LArCVTypes.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file LArCVTypes.h 3 | * 4 | * \ingroup core_Base 5 | * 6 | * \brief basic typedefs and enums (larcv3::Point2D, larcv3::msg, etc.) 7 | * 8 | * @author Kazu - Nevis 2015 9 | */ 10 | 11 | /** \addtogroup core_Base 12 | 13 | @{*/ 14 | 15 | #ifndef __LARCV3BASE_TYPES_H__ 16 | #define __LARCV3BASE_TYPES_H__ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | 30 | /** 31 | \namespace larcv3 32 | C++ namespace for developping LArTPC software interface to computer vision software (LArCV3) 33 | */ 34 | namespace larcv3 { 35 | 36 | /// Used as an invalid value identifier for long long 37 | const long long kINVALID_LONGLONG = std::numeric_limits< long long >::max(); 38 | /// Used as an invalid value identifier for unsigned long long 39 | const unsigned long long kINVALID_ULONGLONG = std::numeric_limits< unsigned long long >::max(); 40 | /// Used as an invalid value identifier for size_t 41 | const size_t kINVALID_SIZE = std::numeric_limits< size_t >::max(); 42 | /// Used as an invalid value identifier for int 43 | const int kINVALID_INT = std::numeric_limits< int >::max(); 44 | /// Used as an invalid value identifier for unsigned int 45 | const unsigned int kINVALID_UINT = std::numeric_limits< unsigned int >::max(); 46 | /// Used as an invalid value identifier for unsigned short 47 | const short kINVALID_SHORT = std::numeric_limits< short >::max(); 48 | /// Used as an invalid value identifier for unsigned unsigned short 49 | const unsigned short kINVALID_USHORT = std::numeric_limits< unsigned short >::max(); 50 | /// Used as an invalid value identifier for single-point precision 51 | const float kINVALID_FLOAT = std::numeric_limits< float >::max(); 52 | /// Used as an invalid value identifier for double-point precision 53 | const double kINVALID_DOUBLE = std::numeric_limits< double >::max(); 54 | /// Used as an invalid value identifier for signed long precision 55 | const long kINVALID_LONG = std::numeric_limits< long >::max(); 56 | 57 | /// Namespace for larcv3 message related types 58 | namespace msg { 59 | 60 | /** 61 | * @brief Message Level 62 | */ 63 | enum Level_t { kDEBUG, kINFO, kNORMAL, kWARNING, kERROR, kCRITICAL, kMSG_TYPE_MAX }; 64 | 65 | /// Formatted message prefix per message level 66 | const std::string kStringPrefix[kMSG_TYPE_MAX] = 67 | { 68 | " \033[94m[DEBUG]\033[00m ", ///< kDEBUG message prefix 69 | " \033[92m[INFO]\033[00m ", ///< kINFO message prefix 70 | " \033[95m[NORMAL]\033[00m ", ///< kNORMAL message prefix 71 | " \033[93m[WARNING]\033[00m ", ///< kWARNING message prefix 72 | " \033[91m[ERROR]\033[00m ", ///< kERROR message prefix 73 | " \033[5;1;33;41m[CRITICAL]\033[00m " ///< kCRITICAL message prefix 74 | }; 75 | ///< Prefix of message 76 | } 77 | 78 | } 79 | #endif 80 | /** @} */ // end of doxygen group 81 | -------------------------------------------------------------------------------- /src/larcv3/core/base/Watch.cxx: -------------------------------------------------------------------------------- 1 | #include "Watch.h" 2 | 3 | void init_Watch(pybind11::module m){ 4 | pybind11::class_ watch(m, "Watch"); 5 | 6 | watch.doc() = R"pbdoc( 7 | Basic C++ timer object, used to track algorithm performance. 8 | )pbdoc"; 9 | 10 | watch.def(pybind11::init<>(), 11 | R"pbdoc( 12 | Construct a C++ timer object for measuring algorithm performance in C++ code. 13 | )pbdoc"); 14 | watch.def("Start", &larcv3::Watch::Start, 15 | R"pbdoc( 16 | Start the timer. 17 | )pbdoc"); 18 | watch.def("WallTime", &larcv3::Watch::WallTime, 19 | R"pbdoc( 20 | Get the current walltime. 21 | )pbdoc"); 22 | watch.def("CPUTime", &larcv3::Watch::CPUTime, 23 | R"pbdoc( 24 | Report CPU Time since Start was called. 25 | )pbdoc"); 26 | 27 | ///TODO Finish the implementation of the functions here 28 | } -------------------------------------------------------------------------------- /src/larcv3/core/base/Watch.h: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3_BASE_WATCH_H__ 2 | #define __LARCV3_BASE_WATCH_H__ 3 | /** 4 | * \file Watch.h 5 | * 6 | * \ingroup core_Base 7 | * 8 | * \brief Class def header for a class larcv3::Watch 9 | * 10 | * @author kazuhiro 11 | */ 12 | 13 | /** \addtogroup core_Base 14 | 15 | @{*/ 16 | 17 | #include 18 | #include 19 | 20 | 21 | namespace larcv3 { 22 | 23 | /** 24 | \class larcv3::Watch 25 | @brief a _very_ simple stopwatch class for simple time measurement 26 | */ 27 | class Watch { 28 | public: 29 | Watch(){} 30 | ~Watch(){} 31 | 32 | /// Start stop watch 33 | void Start() { 34 | // Get current wall time 35 | struct timeval current_time; 36 | gettimeofday(¤t_time,NULL); 37 | _wall_time_start = (double)current_time.tv_sec + (double)current_time.tv_usec * 1.e-6; 38 | // Get current cpu time 39 | _cpu_time_start = (double)(clock()); 40 | } 41 | /// Report wall-clock time [s] since Start() call 42 | double WallTime() { 43 | // Get current wall time 44 | struct timeval current_time; 45 | gettimeofday(¤t_time,NULL); 46 | double now = (double)current_time.tv_sec + (double)current_time.tv_usec * 1.e-6; 47 | // Return diff 48 | return (now - _wall_time_start); 49 | } 50 | /// Report cpu-clock time [s] since Start() call 51 | double CPUTime() { 52 | // Get cpu time 53 | double now = (double)(clock()); 54 | // Return diff 55 | return (now - _cpu_time_start)/CLOCKS_PER_SEC; 56 | } 57 | private: 58 | double _cpu_time_start; 59 | double _wall_time_start; 60 | }; 61 | } 62 | 63 | #ifdef LARCV_INTERNAL 64 | #include 65 | void init_Watch(pybind11::module m); 66 | #endif 67 | 68 | 69 | #endif 70 | /** @} */ // end of doxygen group 71 | -------------------------------------------------------------------------------- /src/larcv3/core/base/base.cxx: -------------------------------------------------------------------------------- 1 | #include "base.h" 2 | 3 | 4 | void init_base(pybind11::module m){ 5 | 6 | init_larbys(m); 7 | init_larcv_base(m); 8 | init_logger(m); 9 | init_Watch(m); 10 | 11 | m.attr("kINVALID_LONGLONG") = larcv3::kINVALID_LONGLONG; 12 | m.attr("kINVALID_ULONGLONG") = larcv3::kINVALID_ULONGLONG; 13 | m.attr("kINVALID_SIZE") = larcv3::kINVALID_SIZE; 14 | m.attr("kINVALID_INT") = larcv3::kINVALID_INT; 15 | m.attr("kINVALID_UINT") = larcv3::kINVALID_UINT; 16 | m.attr("kINVALID_SHORT") = larcv3::kINVALID_SHORT; 17 | m.attr("kINVALID_USHORT") = larcv3::kINVALID_USHORT; 18 | m.attr("kINVALID_FLOAT") = larcv3::kINVALID_FLOAT; 19 | m.attr("kINVALID_DOUBLE") = larcv3::kINVALID_DOUBLE; 20 | m.attr("kINVALID_LONG") = larcv3::kINVALID_LONG; 21 | } 22 | -------------------------------------------------------------------------------- /src/larcv3/core/base/base.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file larbys.h 3 | * 4 | * \ingroup core_Base 5 | * 6 | * \brief Class def header for exception classes for larcv3 framework 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup core_Base 12 | 13 | @{*/ 14 | #ifndef __LARCV3BASE_BASE_H__ 15 | #define __LARCV3BASE_BASE_H__ 16 | 17 | 18 | #include "larbys.h" 19 | #include "larcv_base.h" 20 | #include "Watch.h" 21 | 22 | #ifndef LARCV_NO_PYBIND 23 | #ifdef LARCV_INTERNAL 24 | #include 25 | __attribute__ ((visibility ("default"))) void init_base(pybind11::module m); 26 | #endif 27 | // bindings 28 | #endif 29 | 30 | 31 | // include guards 32 | #endif 33 | -------------------------------------------------------------------------------- /src/larcv3/core/base/larbys.cxx: -------------------------------------------------------------------------------- 1 | #include "larbys.h" 2 | 3 | void init_larbys(pybind11::module m){ 4 | pybind11::class_ larbys(m, "larbys"); 5 | 6 | larbys.doc() = R"pbdoc( 7 | 8 | Base expection for larcv. 9 | )pbdoc"; 10 | 11 | larbys.def(pybind11::init(), pybind11::arg("msg")="", 12 | R"pbdoc( 13 | Construction an exception - not expected to be used from Python. 14 | )pbdoc"); 15 | larbys.def("what", &larcv3::larbys::what, 16 | R"pbdoc( 17 | Get the details of this exception. 18 | )pbdoc"); 19 | } 20 | -------------------------------------------------------------------------------- /src/larcv3/core/base/larbys.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file larbys.h 3 | * 4 | * \ingroup core_Base 5 | * 6 | * \brief Class def header for exception classes for larcv3 framework 7 | * 8 | * @author kazuhiro tmw 9 | */ 10 | 11 | /** \addtogroup core_Base 12 | 13 | @{*/ 14 | #ifndef __LARCV3BASE_LARBYS_H__ 15 | #define __LARCV3BASE_LARBYS_H__ 16 | 17 | #include 18 | #include 19 | 20 | 21 | 22 | 23 | namespace larcv3 { 24 | 25 | /** 26 | \class larbys 27 | Throw insignificant larbys when you find nonesense 28 | */ 29 | class larbys : public std::exception { 30 | 31 | public: 32 | 33 | /** 34 | * @brief Constructs a new instance. 35 | * 36 | * @param[in] msg The message 37 | */ 38 | larbys(std::string msg="") : std::exception() 39 | { 40 | _msg = "\033[93m"; 41 | _msg += msg; 42 | _msg += "\033[00m"; 43 | } 44 | 45 | virtual ~larbys() throw() {} 46 | virtual const char* what() const throw() 47 | { return _msg.c_str(); } 48 | 49 | private: 50 | 51 | std::string _msg; 52 | }; 53 | } 54 | 55 | #ifdef LARCV_INTERNAL 56 | #include 57 | void init_larbys(pybind11::module m); 58 | #endif 59 | 60 | // PYBIND11_MODULE(larcv, m) { 61 | // init_larbys(m); 62 | // } 63 | 64 | #endif 65 | /** @} */ // end of doxygen group 66 | -------------------------------------------------------------------------------- /src/larcv3/core/base/larcv_base.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file larcv_base.h 3 | * 4 | * \ingroup core_Base 5 | * 6 | * \brief Class definition file of larcv3::larcv_base 7 | * 8 | * @author Kazu - Nevis 2015 9 | * @author Corey - ANL 2019 10 | */ 11 | 12 | /** \addtogroup core_Base 13 | 14 | @{*/ 15 | 16 | #ifndef __LARCV3BASE_LARCV_BASE_H__ 17 | #define __LARCV3BASE_LARCV_BASE_H__ 18 | 19 | #include 20 | #include "larcv3/core/base/larcv_logger.h" 21 | #include "pybind11_json/pybind11_json.hpp" 22 | #include 23 | 24 | // for convenience 25 | using json = nlohmann::json; 26 | 27 | 28 | 29 | namespace larcv3 { 30 | 31 | /** 32 | \class larcv_base 33 | Framework base class equipped with a logger class 34 | */ 35 | class larcv_base { 36 | 37 | public: 38 | 39 | /** 40 | * @brief Constructs a new instance. 41 | * 42 | * @param[in] logger_name The logger name 43 | */ 44 | larcv_base(const std::string logger_name="larcv_base") 45 | : _logger(nullptr) 46 | { _logger = &(::larcv3::logger::get(logger_name)); } 47 | 48 | /** 49 | * @brief Constructs a new instance. 50 | * 51 | * @param[in] original The original 52 | */ 53 | larcv_base(const larcv_base &original) : _logger(original._logger) {} 54 | 55 | /** 56 | * @brief Destroys the object. 57 | */ 58 | virtual ~larcv_base(){}; 59 | 60 | /** 61 | * @brief Get const reference to logger 62 | * 63 | * @return { description_of_the_return_value } 64 | */ 65 | inline const larcv3::logger& logger() const 66 | { return *_logger; } 67 | 68 | /** 69 | * @brief Sets the verbosity. 70 | * 71 | * @param[in] level The level 72 | */ 73 | void set_verbosity(::larcv3::msg::Level_t level) 74 | { _logger->set(level); } 75 | 76 | /** 77 | * @brief Name getter, defined in a logger instance attribute 78 | * 79 | * @return name of the logger 80 | */ 81 | const std::string& name() const 82 | { return logger().name(); } 83 | 84 | /** 85 | * @brief Base config getter 86 | * 87 | * @return json config 88 | */ 89 | static json default_config(){ 90 | json j = {{ "larcv_base" , 91 | {"", ""} 92 | }}; 93 | return j; 94 | }; 95 | 96 | protected: 97 | /** 98 | * @brief Augments the default configuration. 99 | * 100 | * @param[in] default_config The default configuration 101 | * @param[in] user_config The user configuration 102 | * 103 | * @return Recursively updates the default config (first argument) 104 | * with a user config (second argument). ANy items in the user_config are either 105 | * added to the default config or replace the existing keys. 106 | */ 107 | json augment_default_config(const json& default_config, const json& user_config); 108 | 109 | private: 110 | 111 | 112 | larcv3::logger *_logger; ///< logger 113 | 114 | }; 115 | } 116 | 117 | #ifdef LARCV_INTERNAL 118 | #include 119 | void init_larcv_base(pybind11::module m); 120 | #endif 121 | 122 | #endif 123 | 124 | /** @} */ // end of doxygen group 125 | -------------------------------------------------------------------------------- /src/larcv3/core/base/larcv_logger.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3BASE_LOGGER_CXX__ 2 | #define __LARCV3BASE_LOGGER_CXX__ 3 | 4 | #include "larcv3/core/base/larcv_logger.h" 5 | #include 6 | 7 | 8 | #include 9 | std::mutex __logger_mtx; 10 | namespace larcv3 { 11 | 12 | logger *logger::_shared_logger = nullptr; 13 | 14 | std::map *logger::_logger_m = nullptr; 15 | 16 | msg::Level_t logger::_level_default = msg::kNORMAL; 17 | 18 | std::ostream& logger::send(const msg::Level_t level) const 19 | { 20 | __logger_mtx.lock(); 21 | (*_ostrm) << msg::kStringPrefix[level].c_str() 22 | << "\033[0m "; 23 | __logger_mtx.unlock(); 24 | return (*_ostrm); 25 | } 26 | 27 | std::ostream& logger::send(const msg::Level_t level, 28 | const std::string& function ) const 29 | { 30 | auto& strm(send(level)); 31 | strm << "\033[94m<" << _name << "::" << function.c_str() << ">\033[00m "; 32 | return strm; 33 | } 34 | 35 | std::ostream& logger::send(const msg::Level_t level, 36 | const std::string& function, 37 | const unsigned int line_num ) const 38 | { 39 | auto& strm(send(level)); 40 | strm << "\033[94m<" << _name << "::" << function.c_str() << "::L" << line_num << ">\033[00m "; 41 | return strm; 42 | } 43 | 44 | std::ostream& logger::send(const msg::Level_t level, 45 | const std::string& function, 46 | const unsigned int line_num, 47 | const std::string& file_name) const 48 | { 49 | auto& strm(send(level,function)); 50 | // FIXME temporary operation to fetch file name from full path 51 | strm << file_name.substr(file_name.rfind("/")+1,file_name.size()).c_str() << "::L" << line_num << " "; 52 | return strm; 53 | } 54 | 55 | logger& logger::get_shared() 56 | { 57 | __logger_mtx.lock(); 58 | if(!_shared_logger) _shared_logger = new logger("GLOBAL"); 59 | __logger_mtx.unlock(); 60 | return *_shared_logger; 61 | 62 | } 63 | } 64 | 65 | 66 | void init_logger(pybind11::module m){ 67 | pybind11::class_ logger(m, "logger"); 68 | 69 | logger.doc() = 70 | R"pbdoc( 71 | C++ native logger utility, used to manage log messages from the C++ application code. 72 | 73 | For python logging, use the standard `logging` utilities. 74 | )pbdoc"; 75 | 76 | logger.def(pybind11::init(),pybind11::arg("name")="no_name", 77 | R"pbdoc( 78 | Construct a logger with a specified name. 79 | )pbdoc"); 80 | logger.def("name", &larcv3::logger::name, 81 | R"pbdoc( 82 | Get the name associated with this logger. 83 | )pbdoc"); 84 | logger.def("set", &larcv3::logger::set, pybind11::arg("level"), 85 | R"pbdoc( 86 | Set the level of this logger. 87 | )pbdoc"); 88 | logger.def("level", &larcv3::logger::level, 89 | R"pbdoc( 90 | Get the level of this logger. 91 | )pbdoc"); 92 | 93 | } 94 | 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /src/larcv3/core/base/stack_trace.h: -------------------------------------------------------------------------------- 1 | // stacktrace.h (c) 2008, Timo Bingmann from http://idlebox.net/ 2 | // published under the WTFPL v2.0 3 | 4 | #ifndef __LARCV3BASE_STACKTRACE_H_ 5 | #define __LARCV3BASE_STACKTRACE_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | /** Print a demangled stack backtrace of the caller function to FILE* out. */ 13 | static inline void print_stacktrace(FILE *out = stderr, unsigned int max_frames = 63) 14 | { 15 | fprintf(out, "stack trace:\n"); 16 | 17 | // storage array for stack trace address data 18 | void* addrlist[max_frames+1]; 19 | 20 | // retrieve current stack addresses 21 | int addrlen = backtrace(addrlist, sizeof(addrlist) / sizeof(void*)); 22 | 23 | if (addrlen == 0) { 24 | fprintf(out, " \n"); 25 | return; 26 | } 27 | 28 | // resolve addresses into strings containing "filename(function+address)", 29 | // this array must be free()-ed 30 | char** symbollist = backtrace_symbols(addrlist, addrlen); 31 | 32 | // allocate string which will be filled with the demangled function name 33 | size_t funcnamesize = 256; 34 | char* funcname = (char*)malloc(funcnamesize); 35 | 36 | // iterate over the returned symbol lines. skip the first, it is the 37 | // address of this function. 38 | for (int i = 1; i < addrlen; i++) 39 | { 40 | char *begin_name = 0, *begin_offset = 0, *end_offset = 0; 41 | 42 | // find parentheses and +address offset surrounding the mangled name: 43 | // ./module(function+0x15c) [0x8048a6d] 44 | for (char *p = symbollist[i]; *p; ++p) 45 | { 46 | if (*p == '(') 47 | begin_name = p; 48 | else if (*p == '+') 49 | begin_offset = p; 50 | else if (*p == ')' && begin_offset) { 51 | end_offset = p; 52 | break; 53 | } 54 | } 55 | 56 | if (begin_name && begin_offset && end_offset 57 | && begin_name < begin_offset) 58 | { 59 | *begin_name++ = '\0'; 60 | *begin_offset++ = '\0'; 61 | *end_offset = '\0'; 62 | 63 | // mangled name is now in [begin_name, begin_offset) and caller 64 | // offset in [begin_offset, end_offset). now apply 65 | // __cxa_demangle(): 66 | 67 | int status; 68 | char* ret = abi::__cxa_demangle(begin_name, 69 | funcname, &funcnamesize, &status); 70 | if (status == 0) { 71 | funcname = ret; // use possibly realloc()-ed string 72 | fprintf(out, " %s : %s+%s\n", 73 | symbollist[i], funcname, begin_offset); 74 | } 75 | else { 76 | // demangling failed. Output function name as a C function with 77 | // no arguments. 78 | fprintf(out, " %s : %s()+%s\n", 79 | symbollist[i], begin_name, begin_offset); 80 | } 81 | } 82 | else 83 | { 84 | // couldn't parse the line? print the whole line. 85 | fprintf(out, " %s\n", symbollist[i]); 86 | } 87 | } 88 | 89 | free(funcname); 90 | free(symbollist); 91 | } 92 | 93 | #endif // _STACKTRACE_H_ 94 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name dataformat) 2 | 3 | # Get all the source files: 4 | file(GLOB SOURCES *.cxx) 5 | file(GLOB HEADERS *.h) 6 | 7 | # Add a shared library 8 | add_library(${name} OBJECT ${SOURCES}) 9 | 10 | 11 | install (FILES ${HEADERS} 12 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/core/${name}) 13 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/DataProductFactory.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3DATAFORMAT_DATAPRODUCTFACTORY_CXX__ 2 | #define __LARCV3DATAFORMAT_DATAPRODUCTFACTORY_CXX__ 3 | 4 | #include "larcv3/core/dataformat/DataProductFactory.h" 5 | 6 | larcv3::DataProductFactory* larcv3::DataProductFactory::_me = nullptr; 7 | 8 | namespace larcv3 { 9 | 10 | void DataProductFactory::add_factory(std::string type, larcv3::DataProductFactoryBase* factory) 11 | { 12 | if(type.empty()) { 13 | LARCV_CRITICAL() << "Attempted to register a data product type with empty name!" << std::endl 14 | << "Maybe a new data product source code was made by gen_class_product script?" << std::endl 15 | << "This error happens when that + forgotten implementation of product_unique_name function (see the header!)" << std::endl 16 | << std::endl; 17 | throw larbys(); 18 | } 19 | 20 | LARCV_INFO() << "Registering a factory " << factory << " of type " << type << std::endl; 21 | 22 | auto iter = _factory_map.find(type); 23 | 24 | if(iter != _factory_map.end()) { 25 | LARCV_CRITICAL() << "Attempted a duplicate registration of Data product " 26 | << type << " to a factory!" << std::endl; 27 | throw larbys(); 28 | } 29 | 30 | _factory_map[type] = factory; 31 | _id_to_type.push_back(type); 32 | } 33 | 34 | EventBase* DataProductFactory::create(const ProducerName_t& id) { 35 | auto iter = _factory_map.find(id.first); 36 | if(iter == _factory_map.end() || !((*iter).second)) { 37 | LARCV_ERROR() << "Found no registered class " << id.first << std::endl; 38 | return nullptr; 39 | } 40 | auto ptr = (*iter).second->create(); 41 | // ptr->_producer = id.second; 42 | return ptr; 43 | } 44 | 45 | void DataProductFactory::list() const { 46 | std::stringstream ss; 47 | ss << " Listing registered products:" << std::endl; 48 | for(size_t id=0; id<_id_to_type.size(); ++id) { 49 | ss << " Name: " << _id_to_type[id] 50 | << " ... ID=" << id 51 | << " ... Factory @ " << _factory_map.find(_id_to_type[id])->second 52 | << std::endl; 53 | } 54 | ss << std::endl; 55 | LARCV_NORMAL() << ss.str() << std::endl; 56 | } 57 | 58 | } 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/DataProductFactory.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file DataProductFactory.h 3 | * 4 | * \ingroup DataFormat 5 | * 6 | * \brief Class def header for a class DataProductFactory 7 | * 8 | * @author kazuhiro, cadams 9 | */ 10 | 11 | /** \addtogroup DataFormat 12 | 13 | @{*/ 14 | #ifndef __LARCV3DATAFORMAT_DATAPRODUCTFACTORY_H__ 15 | #define __LARCV3DATAFORMAT_DATAPRODUCTFACTORY_H__ 16 | 17 | #include 18 | #include 19 | #include "larcv3/core/base/larcv_base.h" 20 | #include "larcv3/core/base/larbys.h" 21 | #include "larcv3/core/dataformat/EventBase.h" 22 | #include "larcv3/core/dataformat/DataFormatTypes.h" 23 | #include 24 | 25 | namespace larcv3 { 26 | 27 | class EventBase; 28 | /** 29 | \class DataProductFactoryBase 30 | \brief Abstract base class for factory (to be implemented per data product) 31 | */ 32 | class DataProductFactoryBase { 33 | public: 34 | /// Default ctor 35 | DataProductFactoryBase(){} 36 | /// Default dtor (virtual) 37 | virtual ~DataProductFactoryBase(){} 38 | /// Abstract constructor method 39 | virtual EventBase* create() = 0; 40 | }; 41 | 42 | /** 43 | \class ClusterAlgoFactory 44 | \brief Factory class for instantiating event product instance by larcv3::IOManager 45 | This factory class can instantiate a specified product instance w/ provided producer name. \n 46 | The actual factory core (to which each product class must register creation factory instance) is \n 47 | a static std::map. Use static method to get a static instance (larcv3::DataProductFactory::get) \n 48 | to access a factory. 49 | */ 50 | class DataProductFactory : public larcv_base { 51 | 52 | public: 53 | /// Default ctor, shouldn't be used 54 | DataProductFactory() : larcv_base("DataProductFactory") 55 | {} 56 | /// Default dtor 57 | ~DataProductFactory() {_factory_map.clear();} 58 | /// Static sharable instance getter 59 | static inline DataProductFactory& get() 60 | { if(!_me) _me = new DataProductFactory; return *_me; } 61 | 62 | /// Factory registration method (should be called by global factory instance in algorithm header) 63 | void add_factory(std::string type, larcv3::DataProductFactoryBase* factory); 64 | 65 | /// Factory creation method (should åbe called by clients, possibly you!) 66 | inline EventBase* create(const std::string& type, const std::string& producer) { 67 | return create(ProducerName_t(type,producer)); 68 | } 69 | 70 | /// Factory creation method (should be called by clients, possibly you!) 71 | EventBase* create(const ProducerName_t& id); 72 | 73 | /// List registered products 74 | void list() const; 75 | 76 | inline size_t unique_product_count() const 77 | { return _id_to_type.size(); } 78 | 79 | inline const std::vector& product_names() const 80 | { return _id_to_type; } 81 | 82 | private: 83 | /// Factory container 84 | std::map _factory_map; 85 | /// Unique product type ID 86 | std::vector _id_to_type; 87 | /// Static self 88 | static DataProductFactory* _me; 89 | }; 90 | } 91 | #endif 92 | /** @} */ // end of doxygen group 93 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/EventBase.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV_EVENTBASE_CXX 2 | #define __LARCV_EVENTBASE_CXX 3 | 4 | #include "EventBase.h" 5 | // #include 6 | // #include 7 | 8 | namespace larcv3{ 9 | EventBase::~EventBase(){ 10 | // for (auto & p : _data_types){ 11 | // delete p; 12 | // } 13 | } 14 | 15 | int EventBase::get_num_objects(hid_t group){ 16 | hsize_t num_objects[1] = {0}; 17 | H5Gget_num_objs(group, num_objects); 18 | return num_objects[0]; 19 | } 20 | } 21 | 22 | void init_eventbase(pybind11::module m){ 23 | using Class = larcv3::EventBase; 24 | pybind11::class_> base(m, "EventBase"); 25 | 26 | // We don't ever want to end up with an event base in python, 27 | // So nothing is wrapped, not even the constructor. 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/EventBase.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file EventBase.h 3 | * 4 | * \ingroup DataFormat 5 | * 6 | * \brief Class def header for a class EventBase 7 | * 8 | * @author cadams, kazuhiro 9 | */ 10 | 11 | /** \addtogroup DataFormat 12 | 13 | @{*/ 14 | #ifndef __LARCV3DATAFORMAT_EVENTBASE_H 15 | #define __LARCV3DATAFORMAT_EVENTBASE_H 16 | 17 | #include 18 | #include "larcv3/core/base/larcv_base.h" 19 | #include "larcv3/core/dataformat/DataFormatTypes.h" 20 | 21 | // There is no need to generate an explicit wrapper for EventBase, since it's virtual 22 | 23 | namespace larcv3 { 24 | // class IOManager; 25 | class DataProductFactory; 26 | /** 27 | \class EventBase 28 | Base class for an event data product (what is stored in output file), holding run/subrun/event ID + producer name. 29 | */ 30 | class EventBase{ 31 | friend class IOManager; 32 | friend class DataProductFactory; 33 | public: 34 | 35 | /// Destructor 36 | virtual ~EventBase() = 0; 37 | 38 | /** 39 | * @brief Clears the object. 40 | */ 41 | virtual void clear() = 0; 42 | 43 | /** 44 | * @brief Initialization 45 | * 46 | * @param[in] group The group 47 | * @param[in] compression The compression 48 | */ 49 | virtual void initialize (hid_t group, uint compression) = 0; 50 | 51 | /** 52 | * @brief Serialize this object 53 | * 54 | * @param[in] group The group 55 | */ 56 | virtual void serialize (hid_t group) = 0; 57 | 58 | /** 59 | * @brief Deserialize this object at specified entry 60 | * 61 | * @param[in] group The group 62 | * @param[in] entry The entry 63 | * @param[in] reopen_groups For re-opening groups, if file changed for example 64 | */ 65 | virtual void deserialize(hid_t group, size_t entry, bool reopen_groups) = 0; 66 | 67 | /** 68 | * @brief Finalize this series of objects by closing HDF5 objects 69 | */ 70 | virtual void finalize() = 0; 71 | 72 | /** 73 | * @brief Opens in datasets. 74 | * 75 | * @param[in] group The group 76 | */ 77 | virtual void open_in_datasets(hid_t group ) = 0; 78 | 79 | /** 80 | * @brief Opens out datasets. 81 | * 82 | * @param[in] group The group 83 | */ 84 | virtual void open_out_datasets(hid_t group ) = 0; 85 | 86 | std::vector _open_in_datasets; 87 | std::vector _open_in_dataspaces; 88 | std::vector _open_out_datasets; 89 | std::vector _open_out_dataspaces; 90 | std::vector _data_types; 91 | 92 | /** 93 | * @brief Gets the number objects. 94 | * 95 | * @param[in] group The group 96 | * 97 | * @return The number objects. 98 | */ 99 | int get_num_objects(hid_t group); 100 | 101 | 102 | 103 | // #endif 104 | }; 105 | 106 | } 107 | 108 | #ifdef LARCV_INTERNAL 109 | #include 110 | void init_eventbase(pybind11::module m); 111 | #endif 112 | 113 | #endif //inc guard 114 | /** @} */ // end of doxygen group 115 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/EventID.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3DATAFORMAT_EVENTID_CXX 2 | #define __LARCV3DATAFORMAT_EVENTID_CXX 3 | 4 | #include "larcv3/core/dataformat/EventID.h" 5 | #include 6 | #include 7 | namespace larcv3 { 8 | 9 | void EventID::clear() 10 | { _run = _subrun = _event = kINVALID_LONG; } 11 | 12 | 13 | std::string EventID::event_key() const 14 | { 15 | std::stringstream ss; 16 | ss << std::setw( 7 ) << std::setfill( '0' ) << _run << "_" 17 | << std::setw( 5 ) << std::setfill( '0' ) << _subrun << "_" 18 | << std::setw( 6 ) << std::setfill( '0' ) << _event; 19 | return ss.str(); 20 | } 21 | 22 | template<> std::string as_string() {return "EventID";} 23 | 24 | 25 | } 26 | 27 | #include 28 | 29 | void init_eventid(pybind11::module m){ 30 | 31 | using Class = larcv3::EventID; 32 | pybind11::class_ eventid(m, "EventID"); 33 | 34 | 35 | eventid.doc() = R"pbdoc( 36 | An object to track run / subrun / event identification of a single event. 37 | Used for information / data management only; all events are stored with a unique 38 | index and do not need a unique EventID. 39 | )pbdoc"; 40 | 41 | eventid.def(pybind11::init<>()); 42 | eventid.def(pybind11::init ()); 43 | eventid.def(pybind11::self == pybind11::self); 44 | eventid.def(pybind11::self != pybind11::self); 45 | eventid.def(pybind11::self < pybind11::self); 46 | 47 | eventid.def("run", (long (Class::*)( ) const)(&Class::run), 48 | "Get the ``run`` value."); 49 | eventid.def("run", (void (Class::*)(long))(&Class::run), 50 | pybind11::arg("run"), 51 | "Set the ``run`` value."); 52 | eventid.def("subrun", (long (Class::*)( ) const)(&Class::subrun), 53 | "Get the ``subrun`` value."); 54 | eventid.def("subrun", (void (Class::*)(long))(&Class::subrun), 55 | pybind11::arg("subrun"), 56 | "Set the ``subrun`` value."); 57 | eventid.def("event", (long (Class::*)( ) const)(&Class::event), 58 | "Get the ``event`` value."); 59 | eventid.def("event", (void (Class::*)(long))(&Class::event), 60 | pybind11::arg("event"), 61 | "Set the ``event`` value."); 62 | 63 | eventid.def("clear", &Class::clear); 64 | eventid.def("valid", &Class::valid); 65 | eventid.def("set_id", &Class::set_id); 66 | eventid.def("event_key", &Class::event_key); 67 | eventid.def("__repr__", &Class::event_key); 68 | 69 | /* 70 | 71 | 72 | /// Formatted string key getter (a string key consists of 0-padded run, 73 | /// subrun, and event id) 74 | std::string event_key() const; 75 | 76 | 77 | */ 78 | 79 | } 80 | 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/EventParticle.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file EventParticle.h 3 | * 4 | * \ingroup DataFormat 5 | * 6 | * \brief Class def header for a class EventParticle 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup DataFormat 12 | 13 | @{*/ 14 | #ifndef __LARCV3DATAFORMAT_EVENTPARTICLE_H 15 | #define __LARCV3DATAFORMAT_EVENTPARTICLE_H 16 | 17 | #include "larcv3/core/dataformat/EventBase.h" 18 | #include "larcv3/core/dataformat/Particle.h" 19 | #include "larcv3/core/dataformat/DataProductFactory.h" 20 | 21 | 22 | namespace larcv3 { 23 | /** 24 | \class EventParticle 25 | User-defined data product class (please comment!) 26 | */ 27 | class EventParticle : public EventBase { 28 | 29 | public: 30 | 31 | /// Default constructor 32 | EventParticle(); 33 | 34 | /// Default destructor 35 | ~EventParticle(){} 36 | 37 | inline larcv3::Particle at(size_t index) {return _part_v.at(index);} 38 | 39 | 40 | 41 | void set(const std::vector& part_v); 42 | void append(const larcv3::Particle& part); 43 | void emplace_back(larcv3::Particle&& part); 44 | void emplace(std::vector&& part_v); 45 | 46 | inline const std::vector& as_vector() const 47 | { return _part_v; } 48 | 49 | // inline const larcv3::Particle& at(size_t index) const {return _part_v.at(index);} 50 | 51 | inline size_t size() const {return _part_v.size();} 52 | 53 | /// Data clear method 54 | void clear (); 55 | void initialize (hid_t group, uint compression); 56 | void serialize (hid_t group); 57 | void deserialize(hid_t group, size_t entry, bool reopen_groups=false); 58 | void finalize (); 59 | 60 | // static EventParticle * to_particle(EventBase * e){ 61 | // return (EventParticle *) e; 62 | // } 63 | 64 | private: 65 | 66 | void open_in_datasets(hid_t group); 67 | void open_out_datasets(hid_t group); 68 | 69 | std::vector _part_v; ///< a collection of particles (index maintained) 70 | 71 | }; 72 | } 73 | 74 | 75 | #include "IOManager.h" 76 | namespace larcv3 { 77 | 78 | // Template instantiation for IO 79 | template<> 80 | inline std::string product_unique_name() { return "particle"; } 81 | // template<> 82 | // inline EventParticle& IOManager::get_data(const std::string&); 83 | // template<> 84 | // inline EventParticle& IOManager::get_data(const ProducerID_t); 85 | 86 | /** 87 | \class larcv3::EventParticle 88 | \brief A concrete factory class for larcv3::EventParticle 89 | */ 90 | class EventParticleFactory : public DataProductFactoryBase { 91 | public: 92 | /// ctor 93 | EventParticleFactory() 94 | { DataProductFactory::get().add_factory(product_unique_name(),this); } 95 | /// dtor 96 | ~EventParticleFactory() {} 97 | /// create method 98 | EventBase* create() { return new EventParticle; } 99 | }; 100 | 101 | 102 | } 103 | 104 | #ifdef LARCV_INTERNAL 105 | #include 106 | void init_eventparticle(pybind11::module m); 107 | #endif 108 | 109 | #endif 110 | /** @} */ // end of doxygen group 111 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/Point.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3DATAFORMAT_POINT_CXX 2 | #define __LARCV3DATAFORMAT_POINT_CXX 3 | 4 | #include 5 | #include 6 | 7 | 8 | #include "larcv3/core/dataformat/Point.h" 9 | 10 | 11 | namespace larcv3{ 12 | template class Point<2>; 13 | template class Point<3>; 14 | } 15 | 16 | template 17 | void init_point_base(pybind11::module m){ 18 | std::string classname = "Point" + std::to_string(dimension) + "D"; 19 | pybind11::class_> point(m, classname.c_str()); 20 | 21 | point.doc() = R"pbdoc( 22 | 23 | A point is a not-very-novel implementation of a multi-value vector 24 | object, implemented again here to enable storage and ease-of-use 25 | in larcv specific classes. 26 | 27 | )pbdoc"; 28 | 29 | 30 | point.def(pybind11::init<>()); 31 | point.def(pybind11::init > ()); 32 | point.def(pybind11::init > ()); 33 | point.def(pybind11::self == pybind11::self); 34 | point.def(pybind11::self != pybind11::self); 35 | point.def(pybind11::self *= double()); 36 | point.def(pybind11::self /= double()); 37 | point.def(pybind11::self -= pybind11::self); 38 | point.def(pybind11::self += pybind11::self); 39 | point.def(pybind11::self * double()); 40 | point.def(pybind11::self / double()); 41 | point.def(pybind11::self + pybind11::self); 42 | point.def(pybind11::self - pybind11::self); 43 | point.def("squared_distance", &larcv3::Point::squared_distance); 44 | point.def("distance", &larcv3::Point::distance); 45 | point.def("direction", &larcv3::Point::direction); 46 | point.def_readwrite("x", &larcv3::Point::x, 47 | "Read/write access to the underlying data ``x``."); 48 | } 49 | 50 | void init_point(pybind11::module m){ 51 | // init_point2D(m); 52 | init_point_base<2>(m); 53 | init_point_base<3>(m); 54 | } 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/Vertex.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Vertex.h 3 | * 4 | * \ingroup DataFormat 5 | * 6 | * \brief Class def header for a class larcv3::Vertex 7 | * 8 | * @author cadams, 9 | */ 10 | 11 | /** \addtogroup DataFormat 12 | 13 | @{*/ 14 | 15 | #ifndef __LARCV3DATAFORMAT_VERTEX_H__ 16 | #define __LARCV3DATAFORMAT_VERTEX_H__ 17 | 18 | #include "larcv3/core/dataformat/DataFormatTypes.h" 19 | #include "larcv3/core/dataformat/Point.h" 20 | 21 | namespace larcv3 { 22 | 23 | class Vertex { 24 | public: 25 | /// Particle ID default constructor 26 | Vertex() = default; 27 | Vertex(double x, double y, double z, double t); 28 | 29 | /// Reset function 30 | void reset(); 31 | 32 | /// Reset function for x, y, z, t 33 | void reset(double x, double y, double z, double t); 34 | 35 | /// Convert to point 36 | const larcv3::Point2D as_point2d(larcv3::PointType_t point_type) const; 37 | const larcv3::Point3D as_point3d() const; 38 | void as_point(larcv3::PointType_t point_type, double * x, double * y, double * z); 39 | 40 | inline double x() const { return _x; } 41 | inline double y() const { return _y; } 42 | inline double z() const { return _z; } 43 | inline double t() const { return _t; } 44 | 45 | /// Default destructor 46 | // The virtual destructor must be removed to make the data layout standard in C++ specifications 47 | // virtual ~Vertex(){}; 48 | 49 | inline bool operator== (const Vertex& rhs) const 50 | { 51 | return ( _x == rhs._x && _y == rhs._y && _z == rhs._z && _t == rhs._t ); 52 | } 53 | 54 | inline bool operator!= (const Vertex& rhs) const 55 | { 56 | return !((*this) == rhs); 57 | } 58 | 59 | inline bool operator< (const Vertex& rhs) const 60 | { 61 | if( _x < rhs._x ) return true; 62 | if( rhs._x < _x ) return false; 63 | if( _y < rhs._y ) return true; 64 | if( rhs._y < _y ) return false; 65 | if( _z < rhs._z ) return true; 66 | if( rhs._z < _z ) return false; 67 | if( _t < rhs._t ) return true; 68 | if( rhs._t < _t ) return false; 69 | 70 | return false; 71 | } 72 | 73 | std::string dump() const; 74 | 75 | public: 76 | 77 | double _x, _y, _z, _t; 78 | 79 | // What does this function do? 80 | // void approx(); 81 | 82 | 83 | public: 84 | static hid_t get_datatype() { 85 | hid_t datatype; 86 | datatype = H5Tcreate (H5T_COMPOUND, sizeof (Vertex)); 87 | H5Tinsert (datatype, "x", 88 | HOFFSET (Vertex, _x), larcv3::get_datatype()); 89 | H5Tinsert (datatype, "y", 90 | HOFFSET (Vertex, _y), larcv3::get_datatype()); 91 | H5Tinsert (datatype, "z", 92 | HOFFSET (Vertex, _z), larcv3::get_datatype()); 93 | H5Tinsert (datatype, "t", 94 | HOFFSET (Vertex, _t), larcv3::get_datatype()); 95 | return datatype; 96 | } 97 | 98 | 99 | 100 | }; 101 | } 102 | #ifdef LARCV_INTERNAL 103 | #include 104 | void init_vertex(pybind11::module m); 105 | #endif 106 | 107 | #endif 108 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/dataformat.cxx: -------------------------------------------------------------------------------- 1 | #include "dataformat.h" 2 | 3 | #include 4 | 5 | 6 | template 7 | void wrap_vector(pybind11::module m){ 8 | std::string name = "VectorOf"+larcv3::as_string(); 9 | using Class = std::vector; 10 | pybind11::class_ vect(m, name.c_str()); 11 | 12 | vect.doc() = "Wrapper around std::vector<" + larcv3::as_string() +">." + "\n Please see the C++ std::vector documentation for more details."; 13 | 14 | vect.def(pybind11::init<>(), 15 | R"pbdoc( 16 | Construct an empty vector of this type. 17 | )pbdoc"); 18 | vect.def("clear", &Class::clear, 19 | R"pbdoc( 20 | Clear the vector. 21 | )pbdoc"); 22 | vect.def("pop_back", &Class::pop_back, 23 | R"pbdoc( 24 | Pop back the vector. 25 | )pbdoc"); 26 | vect.def("__len__", [](const Class &v) { return v.size(); }); 27 | vect.def("__iter__", [](Class &v) { 28 | return pybind11::make_iterator(v.begin(), v.end()); 29 | }, pybind11::keep_alive<0, 1>(), pybind11::return_value_policy::reference); /* Keep vector alive while iterator is used */ 30 | 31 | 32 | // std::string vecname = "VectorOf" + larcv3::as_string(); 33 | // pybind11::bind_vector >(m, vecname.c_str()); 34 | 35 | 36 | } 37 | 38 | void init_dataformat(pybind11::module m){ 39 | 40 | // Wrap vectors of various products: 41 | 42 | wrap_vector(m); 43 | 44 | wrap_vector>(m); 45 | wrap_vector>(m); 46 | 47 | wrap_vector>(m); 48 | wrap_vector>(m); 49 | wrap_vector>(m); 50 | wrap_vector>(m); 51 | 52 | wrap_vector>(m); 53 | wrap_vector>(m); 54 | wrap_vector>(m); 55 | wrap_vector>(m); 56 | 57 | wrap_vector>(m); 58 | wrap_vector>(m); 59 | 60 | wrap_vector (m); 61 | 62 | init_dataformattypes(m); 63 | init_point(m); 64 | init_vertex(m); 65 | init_particle(m); 66 | init_imagemeta(m); 67 | init_bbox(m); 68 | init_tensor(m); 69 | init_voxel(m); 70 | init_eventid(m); 71 | init_eventbase(m); 72 | init_eventparticle(m); 73 | init_eventsparsecluster(m); 74 | init_eventsparsetensor(m); 75 | init_eventtensor(m); 76 | init_eventbbox(m); 77 | init_iomanager(m); 78 | } 79 | -------------------------------------------------------------------------------- /src/larcv3/core/dataformat/dataformat.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file larbys.h 3 | * 4 | * \ingroup DataFormat 5 | * 6 | * \brief Class def header for exception classes for larcv3 framework 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup DataFormat 12 | 13 | @{*/ 14 | #ifndef __LARCV3DATAFORMAT_DATAFORMAT_H__ 15 | #define __LARCV3DATAFORMAT_DATAFORMAT_H__ 16 | 17 | 18 | #include "DataFormatTypes.h" 19 | #include "BBox.h" 20 | // #include "DataProductFactory.h" 21 | #include "EventBase.h" 22 | #include "EventID.h" 23 | #include "EventParticle.h" 24 | #include "EventSparseCluster.h" 25 | #include "EventSparseTensor.h" 26 | #include "EventTensor.h" 27 | #include "EventBBox.h" 28 | #include "ImageMeta.h" 29 | #include "IOManager.h" 30 | #include "Particle.h" 31 | #include "Point.h" 32 | #include "Tensor.h" 33 | #include "Vertex.h" 34 | #include "Voxel.h" 35 | 36 | #ifndef LARCV_NO_PYBIND 37 | #ifdef LARCV_INTERNAL 38 | #include 39 | __attribute__ ((visibility ("default"))) void init_dataformat(pybind11::module m); 40 | #endif 41 | // bindings 42 | #endif 43 | 44 | 45 | // include guards 46 | #endif 47 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(name processor) 2 | 3 | # Get all the source files: 4 | file(GLOB SOURCES *.cxx) 5 | file(GLOB HEADERS *.h) 6 | 7 | # Add a shared library 8 | add_library(${name} OBJECT ${SOURCES}) 9 | 10 | 11 | 12 | 13 | install (FILES ${HEADERS} 14 | DESTINATION ${CMAKE_PACKAGE_DIR}/include/larcv3/core/${name}) 15 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/ProcessBase.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3PROCESSOR_PROCESSBASE_CXX 2 | #define __LARCV3PROCESSOR_PROCESSBASE_CXX 3 | 4 | #include "larcv3/core/processor/ProcessBase.h" 5 | 6 | namespace larcv3 { 7 | 8 | ProcessBase::ProcessBase(const std::string name) 9 | : larcv_base ( name ) 10 | , _proc_time ( 0. ) 11 | , _proc_count ( 0 ) 12 | , _id ( kINVALID_SIZE ) 13 | , _profile ( false ) 14 | {} 15 | 16 | bool ProcessBase::is(const std::string question) const 17 | { return false; } 18 | 19 | void ProcessBase::_configure_(const json& cfg) 20 | { 21 | LARCV_DEBUG() << "config is " << cfg << std::endl; 22 | _profile = false; 23 | if (cfg.contains("Profile")){ 24 | _profile = cfg["Profile"].get(); 25 | } 26 | configure(cfg); 27 | } 28 | 29 | bool ProcessBase::_process_(IOManager& mgr) 30 | { 31 | bool status=false; 32 | if(_profile) { 33 | _watch.Start(); 34 | status = this->process(mgr); 35 | _proc_time += _watch.WallTime(); 36 | ++_proc_count; 37 | }else status = this->process(mgr); 38 | return status; 39 | } 40 | 41 | 42 | } 43 | 44 | void init_processbase(pybind11::module m){ 45 | using Class = larcv3::ProcessBase; 46 | pybind11::class_ processbase(m, "ProcessBase"); 47 | 48 | // We don't actually want to construct any instances of ProcessBase 49 | // processbase.def(pybind11::init(), 50 | // pybind11::arg("name")="ProcessBase"); 51 | 52 | } 53 | 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/ProcessBase.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file ProcessBase.h 3 | * 4 | * \ingroup core_Processor 5 | * 6 | * \brief Class def header for a class larcv3::ProcessBase 7 | * 8 | * @author drinkingkazu 9 | */ 10 | 11 | /** \addtogroup core_Processor 12 | 13 | @{*/ 14 | #ifndef __LARCV3PROCESSOR_PROCESSBASE_H 15 | #define __LARCV3PROCESSOR_PROCESSBASE_H 16 | 17 | #include "larcv3/core/base/Watch.h" 18 | #include "larcv3/core/dataformat/IOManager.h" 19 | #include "larcv3/core/processor/ProcessorTypes.h" 20 | 21 | namespace larcv3 { 22 | 23 | class ProcessDriver; 24 | class ProcessFactory; 25 | /** 26 | \class ProcessBase 27 | @brief A base class for "process module" to be run by larcv3::ProcessDriver 28 | @detail Inherited class must implment 4 pure virtual functions that are called by larcv3::ProcessDriver instance.\n 29 | ProcessBase::configure(const larcv3::PSet) is called first with the argument passing the configuration parameters.\n 30 | ProcessBase::initialize() is called after configure. This is where you may want to initialize variables.\n 31 | ProcessBase::process(larcv3::IOManager&) is called for every event. The argument provides an access to event data.\n 32 | ProcessBase::finalize() is called after larcv3::ProcessDriver finished looping over all events.\n 33 | */ 34 | class ProcessBase : public larcv_base { 35 | friend class ProcessDriver; 36 | friend class ProcessFactory; 37 | 38 | public: 39 | 40 | /// Default constructor 41 | ProcessBase(const std::string name="ProcessBase"); 42 | 43 | /// Default destructor 44 | virtual ~ProcessBase(){} 45 | 46 | // 47 | // Four pure virtual functions that larcv3::ProcessDriver calls and need implementation. 48 | // 49 | /// Called first with the argument passing the configuration parameters. 50 | virtual void configure(const json&) = 0; 51 | /// Called after configure, this is where you should initialize variables to be stored in an output analysis file. 52 | virtual void initialize() = 0; 53 | /// Called per-event, this is where you should implement your per-event action/analysis. 54 | virtual bool process(IOManager& mgr) = 0; 55 | /// Called after event loop is over. This is where you can store your histograms etc. to an output analysis file. 56 | virtual void finalize() = 0; 57 | 58 | // 59 | // Following functions are 60 | // 61 | /// Only for experts: allows a loose grouping for a set of ProcessBase inherit classes via true/false return to a "question". 62 | virtual bool is(const std::string question) const; 63 | 64 | /** 65 | * @brief Default configuration 66 | * 67 | * @return JSON default config 68 | */ 69 | static json default_config(){ 70 | json c = { 71 | {"ProcessName", ""}, 72 | {"ProcessType", ""}, 73 | }; 74 | return c; 75 | } 76 | 77 | 78 | private: 79 | 80 | void _configure_(const json&); 81 | 82 | bool _process_(IOManager& mgr); 83 | 84 | bool _event_creator; ///< special flag to mark this algorithm an event creator 85 | larcv3::Watch _watch; ///< algorithm profile stopwatch 86 | double _proc_time; ///< algorithm execution time record (cumulative) 87 | size_t _proc_count; ///< algorithm execution counter (cumulative) 88 | 89 | larcv3::ProcessID_t _id; ///< unique algorithm identifier 90 | bool _profile; ///< measure process time if profile flag is on 91 | std::string _typename; ///< process type from factory 92 | }; 93 | } 94 | #ifdef LARCV_INTERNAL 95 | #include 96 | void init_processbase(pybind11::module m); 97 | #endif 98 | 99 | #endif 100 | /** @} */ // end of doxygen group 101 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/ProcessFactory.cxx: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3PROCESSOR_PROCESSFACTORY_CXX__ 2 | #define __LARCV3PROCESSOR_PROCESSFACTORY_CXX__ 3 | 4 | #include "larcv3/core/processor/ProcessFactory.h" 5 | 6 | larcv3::ProcessFactory* larcv3::ProcessFactory::_me = nullptr; 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/ProcessFactory.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file ProcessFactory.h 3 | * 4 | * \ingroup core_Processor 5 | * 6 | * \brief Class def header for a class larcv3::ProcessFactory 7 | * 8 | * @author kazuhiro 9 | */ 10 | 11 | /** \addtogroup core_Processor 12 | 13 | @{*/ 14 | #ifndef __LARCV3PROCESSOR_PROCESSFACTORY_H 15 | #define __LARCV3PROCESSOR_PROCESSFACTORY_H 16 | 17 | #include 18 | #include 19 | #include "larcv3/core/base/larcv_base.h" 20 | #include "larcv3/core/processor/ProcessBase.h" 21 | 22 | #include 23 | 24 | static std::mutex __procfactory_mtx; 25 | 26 | namespace larcv3 { 27 | 28 | /** 29 | \class ProcessFactoryBase 30 | \brief Abstract base class for factory (to be implemented per process) 31 | */ 32 | class ProcessFactoryBase { 33 | public: 34 | /// Default ctor 35 | ProcessFactoryBase(){} 36 | /// Default dtor (virtual) 37 | virtual ~ProcessFactoryBase(){} 38 | /// Abstract constructor method 39 | virtual ProcessBase* create(const std::string instance_name) = 0; 40 | }; 41 | 42 | /** 43 | \class ProcessFactory 44 | \brief Factory class for instantiating process instance 45 | This factory class can instantiate a specified process instance w/ provided instance name. \n 46 | The actual factory core (to which each algorithm must register creation factory instance) is \n 47 | a static std::map. Use static method to get a static instance (larcv3::ProcessFactory::get) \n 48 | to access a factory. 49 | */ 50 | class ProcessFactory : public larcv_base { 51 | 52 | public: 53 | /// Default ctor, shouldn't be used 54 | ProcessFactory() : larcv_base("ProcessFactory") 55 | {} 56 | /// Default dtor 57 | ~ProcessFactory() {_factory_map.clear();} 58 | /// Static sharable instance getter 59 | inline static ProcessFactory& get() 60 | { 61 | __procfactory_mtx.lock(); 62 | if(!_me) _me = new ProcessFactory; 63 | __procfactory_mtx.unlock(); 64 | return *_me; 65 | } 66 | /// Factory registration method (should be called by global factory instance in algorithm header) 67 | void add_factory(const std::string name, larcv3::ProcessFactoryBase* factory) 68 | { _factory_map[name] = factory; } 69 | /// Factory creation method (should be called by clients, possibly you!) 70 | ProcessBase* create(const std::string name, const std::string instance_name) { 71 | auto iter = _factory_map.find(name); 72 | if(iter == _factory_map.end() || !((*iter).second)) { 73 | LARCV_ERROR() << "Found no registered class " << name << std::endl; 74 | return nullptr; 75 | } 76 | auto ptr = (*iter).second->create(instance_name); 77 | ptr->_typename = name; 78 | return ptr; 79 | } 80 | 81 | private: 82 | /// Static factory container 83 | std::map _factory_map; 84 | /// Static self 85 | static ProcessFactory* _me; 86 | }; 87 | } 88 | #endif 89 | /** @} */ // end of doxygen group 90 | 91 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/ProcessorTypes.h: -------------------------------------------------------------------------------- 1 | #ifndef __LARCV3PROCESSOR_PROCESSORTYPES_H__ 2 | #define __LARCV3PROCESSOR_PROCESSORTYPES_H__ 3 | 4 | namespace larcv3 { 5 | 6 | typedef size_t ProcessID_t; ///< A unique identifier for an instantiated processes 7 | 8 | } 9 | #endif 10 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/processor.cxx: -------------------------------------------------------------------------------- 1 | #include "processor.h" 2 | 3 | 4 | 5 | void init_processor(pybind11::module m){ 6 | init_processbase(m); 7 | init_processdriver(m); 8 | } 9 | 10 | -------------------------------------------------------------------------------- /src/larcv3/core/processor/processor.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file larbys.h 3 | * 4 | * \ingroup core_Base 5 | * 6 | * \brief Class def header for exception classes for larcv3 framework 7 | * 8 | * @author cadams 9 | */ 10 | 11 | /** \addtogroup core_Base 12 | 13 | @{*/ 14 | #ifndef __LARCV3PROCESSOR_PROCESSOR_H__ 15 | #define __LARCV3PROCESSOR_PROCESSOR_H__ 16 | 17 | 18 | #include "ProcessBase.h" 19 | #include "ProcessDriver.h" 20 | 21 | 22 | #ifndef LARCV_NO_PYBIND 23 | #ifdef LARCV_INTERNAL 24 | #include 25 | __attribute__ ((visibility ("default"))) void init_processor(pybind11::module m); 26 | #endif 27 | 28 | // bindings 29 | #endif 30 | 31 | 32 | // include guards 33 | #endif 34 | -------------------------------------------------------------------------------- /src/larcv3/larcv.cxx: -------------------------------------------------------------------------------- 1 | #include "larcv3/core/base/base.h" 2 | #include "larcv3/core/dataformat/dataformat.h" 3 | #include "larcv3/core/processor/processor.h" 4 | #include "larcv3/app/queueio/queueio.h" 5 | #include "larcv3/app/imagemod/imagemod.h" 6 | #include "larcv3/app/sbnd_imagemod/sbnd_imagemod.h" 7 | #include "larcv3/app/augment/augment.h" 8 | 9 | 10 | PYBIND11_MAKE_OPAQUE(std::vector >); 11 | PYBIND11_MAKE_OPAQUE(std::vector >); 12 | PYBIND11_MAKE_OPAQUE(std::vector >); 13 | PYBIND11_MAKE_OPAQUE(std::vector >); 14 | PYBIND11_MAKE_OPAQUE(std::vector >); 15 | PYBIND11_MAKE_OPAQUE(std::vector >); 16 | 17 | PYBIND11_MAKE_OPAQUE(std::vector >); 18 | PYBIND11_MAKE_OPAQUE(std::vector >); 19 | PYBIND11_MAKE_OPAQUE(std::vector >); 20 | PYBIND11_MAKE_OPAQUE(std::vector >); 21 | 22 | 23 | PYBIND11_MODULE(pylarcv, m) { 24 | m.doc() = R"pbdoc( 25 | larcv 26 | ----------------------- 27 | .. currentmodule:: larcv 28 | .. autosummary:: 29 | :toctree: _generate 30 | base 31 | )pbdoc"; 32 | 33 | init_base(m); 34 | init_dataformat(m); 35 | init_processor(m); 36 | init_queueio(m); 37 | init_imagemod(m); 38 | init_sbnd_imagemod(m); 39 | init_augment(m); 40 | } 41 | -------------------------------------------------------------------------------- /tests/larcv/app/queueio/test_queueio_BBox2D.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import uuid 5 | import numpy, time 6 | 7 | import larcv 8 | from larcv import queueloader, data_generator 9 | 10 | from collections import OrderedDict 11 | 12 | # queue_io_bbox2d_cfg_template = ''' 13 | # {name}: {{ 14 | # Verbosity: 2 15 | # EnableFilter: false 16 | # RandomAccess: 0 17 | # RandomSeed: 0 18 | # InputFiles: [{input_files}] 19 | # ProcessType: ["BatchFillerBBox2D"] 20 | # ProcessName: ["test_{name}"] 21 | # 22 | # ProcessList: {{ 23 | # test_{name}: {{ 24 | # BBoxProducer: "{producer}" 25 | # MaxBoxes: {max_boxes} 26 | # UnfilledBoxValue: -999 27 | # Channels: {channels} 28 | # }} 29 | # }} 30 | # }} 31 | # ''' 32 | 33 | 34 | def create_bbox2d_file(file_name, rand_num_events, n_projections): 35 | bbox_list, meta_list = data_generator.create_bbox_list(rand_num_events, n_projections = n_projections, dimension = 2) 36 | data_generator.write_bboxes(file_name, bbox_list, meta_list, dimension = 2, n_projections = n_projections) 37 | 38 | 39 | @pytest.mark.parametrize('make_copy', [True, False]) 40 | @pytest.mark.parametrize('batch_size', [2]) 41 | @pytest.mark.parametrize('n_projections', [1,2]) 42 | @pytest.mark.parametrize('from_dense', [False, True]) 43 | def test_bbox2d_queueio(tmpdir, make_copy, batch_size, n_projections, from_dense, n_reads=10): 44 | 45 | 46 | 47 | 48 | 49 | queueio_name = "queueio_{}".format(uuid.uuid4()) 50 | 51 | 52 | # First, create a file to write to: 53 | file_name = str(tmpdir + "/test_queueio_bbox_{}.h5".format(queueio_name)) 54 | 55 | # Next, write some bbox2ds to that file: 56 | create_bbox2d_file(file_name, rand_num_events=25, n_projections=n_projections) 57 | 58 | # Now, let's get the configuration of a queueio object: 59 | 60 | default_config = larcv.QueueProcessor.default_config() 61 | default_config["InputFiles"].append(file_name) 62 | 63 | # Add the batch filler to the default config: 64 | default_config["Verbosity"] = 0 65 | default_config["ProcessDriver"]["ProcessName"].append(f"test_{queueio_name}") 66 | default_config["ProcessDriver"]["ProcessType"].append("BatchFillerBBox2D") 67 | 68 | 69 | if default_config["ProcessDriver"]["ProcessList"] is None: 70 | default_config["ProcessDriver"]["ProcessList"] = { 71 | f"test_{queueio_name}": 72 | { 73 | "Producer": "test", 74 | "Channels": list(range(n_projections)), 75 | } 76 | } 77 | else: 78 | default_config["ProcessDriver"]["ProcessList"].append( 79 | {f"test_{queueio_name}": 80 | { 81 | "Producer": "test", 82 | "Channels": list(range(n_projections)), 83 | } 84 | }) 85 | 86 | 87 | 88 | queue_proc = larcv.QueueProcessor() 89 | queue_proc.configure(default_config) 90 | 91 | 92 | indexes = numpy.arange(batch_size*n_reads*2) % queue_proc.get_n_entries() 93 | 94 | queue_proc.set_next_batch(indexes[0:batch_size]) 95 | queue_proc.prepare_next() 96 | while queue_proc.is_reading(): 97 | print("Sleeping") 98 | time.sleep(0.1) 99 | queue_proc.pop_current_data() 100 | 101 | for i in range(n_reads): 102 | batch=indexes[i*batch_size:(i+1)*batch_size] 103 | queue_proc.set_next_batch(batch) 104 | queue_proc.prepare_next() 105 | 106 | while queue_proc.is_reading(): 107 | print("Sleeping") 108 | time.sleep(0.1) 109 | 110 | 111 | if __name__ == "__main__": 112 | test_bbox2d_queueio("./", make_copy=False, batch_size=2, n_projections=1, from_dense=False, n_reads=10) 113 | test_bbox2d_queueio("./", make_copy=False, batch_size=2, n_projections=2, from_dense=False, n_reads=10) 114 | print("Success") 115 | -------------------------------------------------------------------------------- /tests/larcv/app/queueio/test_queueio_PIDLabel.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import uuid 5 | import numpy 6 | import time 7 | 8 | import larcv 9 | from larcv import data_generator 10 | 11 | from collections import OrderedDict 12 | 13 | 14 | def create_particle_file(file_name, rand_num_events): 15 | data_generator.write_particles(file_name, rand_num_events, particles_per_event=1) 16 | 17 | 18 | @pytest.mark.parametrize('batch_size', [1, 2]) 19 | def test_particle_queueio(tmpdir, batch_size, n_reads=10): 20 | 21 | queueio_name = "queueio_{}".format(uuid.uuid4()) 22 | 23 | 24 | # First, create a file to write to: 25 | file_name = str(tmpdir + "/test_queueio_particles_{}.h5".format(queueio_name)) 26 | 27 | # Next, write some particles to that file: 28 | create_particle_file(file_name, rand_num_events=25) 29 | 30 | # Now, let's get the configuration of a queueio object: 31 | 32 | default_config = larcv.QueueProcessor.default_config() 33 | default_config["InputFiles"].append(file_name) 34 | 35 | # Add the batch filler to the default config: 36 | default_config["Verbosity"] = 0 37 | default_config["ProcessDriver"]["ProcessName"].append(f"test_{queueio_name}") 38 | default_config["ProcessDriver"]["ProcessType"].append("BatchFillerPIDLabel") 39 | 40 | 41 | if default_config["ProcessDriver"]["ProcessList"] is None: 42 | default_config["ProcessDriver"]["ProcessList"] = { 43 | f"test_{queueio_name}": 44 | { 45 | "ParticleProducer": "test", 46 | "PdgClassList": [0], 47 | } 48 | } 49 | else: 50 | default_config["ProcessDriver"]["ProcessList"].append( 51 | {f"test_{queueio_name}": 52 | { 53 | "ParticleProducer": "test", 54 | "PdgClassList": [0], 55 | } 56 | }) 57 | 58 | 59 | queue_proc = larcv.QueueProcessor() 60 | queue_proc.configure(default_config) 61 | 62 | 63 | 64 | indexes = numpy.arange(batch_size*n_reads*2) % queue_proc.get_n_entries() 65 | 66 | queue_proc.set_next_batch(indexes[0:batch_size]) 67 | queue_proc.prepare_next() 68 | while queue_proc.is_reading(): 69 | print("Sleeping") 70 | time.sleep(0.1) 71 | queue_proc.pop_current_data() 72 | 73 | for i in range(n_reads): 74 | batch=indexes[i*batch_size:(i+1)*batch_size] 75 | queue_proc.set_next_batch(batch) 76 | queue_proc.prepare_next() 77 | 78 | while queue_proc.is_reading(): 79 | print("Sleeping") 80 | time.sleep(0.1) 81 | 82 | 83 | 84 | if __name__ == "__main__": 85 | test_particle_queueio("./", batch_size=2, n_reads=10) 86 | test_particle_queueio("./", batch_size=2, n_reads=10) 87 | print("Success") 88 | -------------------------------------------------------------------------------- /tests/larcv/app/queueio/test_queueio_Particle.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import uuid 5 | import numpy 6 | import time 7 | 8 | import larcv 9 | from larcv import data_generator 10 | 11 | from collections import OrderedDict 12 | 13 | 14 | def create_particle_file(file_name, rand_num_events): 15 | data_generator.write_particles(file_name, rand_num_events, particles_per_event=1) 16 | 17 | 18 | @pytest.mark.parametrize('batch_size', [1, 2]) 19 | def test_particle_queueio(tmpdir, batch_size, n_reads=10): 20 | 21 | queueio_name = "queueio_{}".format(uuid.uuid4()) 22 | 23 | 24 | # First, create a file to write to: 25 | file_name = str(tmpdir + "/test_queueio_particles_{}.h5".format(queueio_name)) 26 | 27 | # Next, write some particles to that file: 28 | create_particle_file(file_name, rand_num_events=25) 29 | 30 | # Now, let's get the configuration of a queueio object: 31 | 32 | default_config = larcv.QueueProcessor.default_config() 33 | default_config["InputFiles"].append(file_name) 34 | 35 | # Add the batch filler to the default config: 36 | default_config["Verbosity"] = 0 37 | default_config["ProcessDriver"]["ProcessName"].append(f"test_{queueio_name}") 38 | default_config["ProcessDriver"]["ProcessType"].append("BatchFillerParticle") 39 | 40 | 41 | if default_config["ProcessDriver"]["ProcessList"] is None: 42 | default_config["ProcessDriver"]["ProcessList"] = { 43 | f"test_{queueio_name}": 44 | { 45 | "ParticleProducer": "test", 46 | } 47 | } 48 | else: 49 | default_config["ProcessDriver"]["ProcessList"].append( 50 | {f"test_{queueio_name}": 51 | { 52 | "ParticleProducer": "test", 53 | } 54 | }) 55 | 56 | 57 | queue_proc = larcv.QueueProcessor() 58 | queue_proc.configure(default_config) 59 | 60 | 61 | 62 | indexes = numpy.arange(batch_size*n_reads*2) % queue_proc.get_n_entries() 63 | 64 | queue_proc.set_next_batch(indexes[0:batch_size]) 65 | queue_proc.prepare_next() 66 | while queue_proc.is_reading(): 67 | print("Sleeping") 68 | time.sleep(0.1) 69 | queue_proc.pop_current_data() 70 | 71 | for i in range(n_reads): 72 | batch=indexes[i*batch_size:(i+1)*batch_size] 73 | queue_proc.set_next_batch(batch) 74 | queue_proc.prepare_next() 75 | 76 | while queue_proc.is_reading(): 77 | print("Sleeping") 78 | time.sleep(0.1) 79 | # get the data? 80 | 81 | 82 | 83 | if __name__ == "__main__": 84 | test_particle_queueio("./", batch_size=2, n_reads=10) 85 | test_particle_queueio("./", batch_size=2, n_reads=10) 86 | print("Success") 87 | -------------------------------------------------------------------------------- /tests/larcv/app/queueio/test_queueio_SparseTensor2D.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import uuid 5 | import numpy 6 | import time 7 | 8 | import larcv 9 | from larcv import data_generator 10 | 11 | from collections import OrderedDict 12 | 13 | 14 | 15 | def create_sparsetensor2d_file(file_name, rand_num_events, n_projections): 16 | voxel_set_list = data_generator.build_sparse_tensor(rand_num_events, n_projections = n_projections) 17 | data_generator.write_sparse_tensors(file_name, voxel_set_list, dimension=2, n_projections=n_projections) 18 | 19 | @pytest.mark.parametrize('batch_size', [1, 2]) 20 | @pytest.mark.parametrize('n_projections', [1,2]) 21 | def test_sparsetensor2d_queueio(tmpdir, batch_size, n_projections, n_reads=10): 22 | 23 | 24 | queueio_name = "queueio_{}".format(uuid.uuid4()) 25 | 26 | 27 | # First, create a file to write to: 28 | file_name = str(tmpdir + "/test_queueio_sparsetensor2d_{}.h5".format(queueio_name)) 29 | 30 | # Next, write some sparsetensor2ds to that file: 31 | create_sparsetensor2d_file(file_name, rand_num_events=25, n_projections=n_projections) 32 | 33 | 34 | # Now, let's get the configuration of a queueio object: 35 | default_config = larcv.QueueProcessor.default_config() 36 | default_config["InputFiles"].append(file_name) 37 | 38 | # Add the batch filler to the default config: 39 | default_config["Verbosity"] = 0 40 | default_config["ProcessDriver"]["ProcessName"].append(f"test_{queueio_name}") 41 | default_config["ProcessDriver"]["ProcessType"].append("BatchFillerSparseTensor2D") 42 | 43 | process_list = {f"test_{queueio_name}": 44 | { 45 | "TensorProducer": "test", 46 | "MaxVoxels": 100, 47 | "UnfilledVoxelValue": -999, 48 | "Channels": list(range(n_projections)), 49 | } 50 | } 51 | 52 | 53 | if default_config["ProcessDriver"]["ProcessList"] is None: 54 | default_config["ProcessDriver"]["ProcessList"] = process_list 55 | else: 56 | default_config["ProcessDriver"]["ProcessList"].append(process_list) 57 | 58 | 59 | queue_proc = larcv.QueueProcessor() 60 | queue_proc.configure(default_config) 61 | 62 | 63 | indexes = numpy.arange(batch_size*n_reads*2) % queue_proc.get_n_entries() 64 | 65 | queue_proc.set_next_batch(indexes[0:batch_size]) 66 | queue_proc.prepare_next() 67 | while queue_proc.is_reading(): 68 | print("Sleeping") 69 | time.sleep(0.1) 70 | queue_proc.pop_current_data() 71 | 72 | for i in range(n_reads): 73 | batch=indexes[i*batch_size:(i+1)*batch_size] 74 | queue_proc.set_next_batch(batch) 75 | queue_proc.prepare_next() 76 | 77 | while queue_proc.is_reading(): 78 | print("Sleeping") 79 | time.sleep(0.1) 80 | 81 | 82 | if __name__ == "__main__": 83 | test_sparsetensor2d_queueio("./", batch_size=2, n_projections=1, n_reads=10) 84 | test_sparsetensor2d_queueio("./", batch_size=2, n_projections=2, n_reads=10) 85 | print("Success") 86 | -------------------------------------------------------------------------------- /tests/larcv/app/queueio/test_queueio_Tensor2D.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import uuid 5 | import time 6 | import numpy 7 | 8 | import larcv 9 | from larcv import queueloader, data_generator 10 | 11 | from collections import OrderedDict 12 | 13 | 14 | 15 | 16 | def create_tensor2d_file(file_name, rand_num_events, n_projections): 17 | voxel_set_list = data_generator.build_sparse_tensor(rand_num_events, n_projections = n_projections) 18 | data_generator.write_sparse_tensors(file_name, voxel_set_list, dimension=2, n_projections=n_projections) 19 | 20 | def create_dense_tensor2d_file(file_name, rand_num_events, n_projections): 21 | voxel_set_list = data_generator.build_tensor(rand_num_events, n_projections = n_projections) 22 | data_generator.write_tensor(file_name, voxel_set_list, dimension=2) 23 | 24 | @pytest.mark.parametrize('batch_size', [2]) 25 | @pytest.mark.parametrize('n_projections', [1,2]) 26 | @pytest.mark.parametrize('from_dense', [False, True]) 27 | def test_tensor2d_queueio(tmpdir, batch_size, n_projections, from_dense, n_reads=10): 28 | 29 | 30 | queueio_name = "queueio_{}".format(uuid.uuid4()) 31 | 32 | 33 | # First, create a file to write to: 34 | file_name = str(tmpdir + "/test_queueio_tensor2d_{}.h5".format(queueio_name)) 35 | 36 | # Next, write some tensor2ds to that file: 37 | if from_dense: 38 | create_dense_tensor2d_file(file_name, rand_num_events=25, n_projections=n_projections) 39 | else: 40 | create_tensor2d_file(file_name, rand_num_events=25, n_projections=n_projections) 41 | 42 | 43 | if from_dense: 44 | tensor_type = "dense" 45 | else: 46 | tensor_type = "sparse" 47 | 48 | # Now, let's get the configuration of a queueio object: 49 | default_config = larcv.QueueProcessor.default_config() 50 | default_config["InputFiles"].append(file_name) 51 | 52 | # Add the batch filler to the default config: 53 | default_config["Verbosity"] = 0 54 | default_config["ProcessDriver"]["ProcessName"].append(f"test_{queueio_name}") 55 | default_config["ProcessDriver"]["ProcessType"].append("BatchFillerTensor2D") 56 | 57 | process_list = {f"test_{queueio_name}": 58 | { 59 | "TensorProducer": "test", 60 | "TensorType": tensor_type, 61 | "Channels": list(range(n_projections)), 62 | "EmptyVoxelValue": -999., 63 | "Augment": True, 64 | } 65 | } 66 | 67 | if default_config["ProcessDriver"]["ProcessList"] is None: 68 | default_config["ProcessDriver"]["ProcessList"] = process_list 69 | else: 70 | default_config["ProcessDriver"]["ProcessList"].append(process_list) 71 | 72 | 73 | queue_proc = larcv.QueueProcessor() 74 | queue_proc.configure(default_config) 75 | 76 | 77 | indexes = numpy.arange(batch_size*n_reads*2) % queue_proc.get_n_entries() 78 | 79 | queue_proc.set_next_batch(indexes[0:batch_size]) 80 | print(queue_proc.is_reading()) 81 | queue_proc.prepare_next() 82 | time.sleep(2) 83 | while queue_proc.is_reading(): 84 | print("Sleeping") 85 | time.sleep(0.5) 86 | queue_proc.pop_current_data() 87 | 88 | for i in range(n_reads): 89 | batch=indexes[i*batch_size:(i+1)*batch_size] 90 | queue_proc.set_next_batch(batch) 91 | queue_proc.prepare_next() 92 | 93 | while queue_proc.is_reading(): 94 | print("Sleeping") 95 | time.sleep(0.1) 96 | 97 | 98 | 99 | 100 | if __name__ == "__main__": 101 | test_tensor2d_queueio("./", batch_size=2, n_projections=1, from_dense=False, n_reads=10) 102 | test_tensor2d_queueio("./", batch_size=2, n_projections=2, from_dense=False, n_reads=10) 103 | print("Success") -------------------------------------------------------------------------------- /tests/larcv/app/queueio/test_queueio_Tensor3D.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import uuid 5 | import numpy 6 | import time 7 | 8 | import larcv 9 | from larcv import queueloader, data_generator 10 | 11 | from collections import OrderedDict 12 | 13 | 14 | 15 | 16 | def create_tensor3d_file(file_name, rand_num_events, n_projections=1): 17 | voxel_set_list = data_generator.build_sparse_tensor(rand_num_events, n_projections = n_projections) 18 | data_generator.write_sparse_tensors(file_name, voxel_set_list, dimension=3, n_projections=n_projections) 19 | 20 | def create_dense_tensor3d_file(file_name, rand_num_events, n_projections=1): 21 | voxel_set_list = data_generator.build_tensor(rand_num_events, dimension=3, n_projections = n_projections) 22 | data_generator.write_tensor(file_name, voxel_set_list, dimension=3) 23 | 24 | 25 | 26 | @pytest.mark.parametrize('make_copy', [False]) 27 | @pytest.mark.parametrize('batch_size', [1,2]) 28 | @pytest.mark.parametrize('from_dense', [False, True]) 29 | def test_tensor3d_queueio(tmpdir, make_copy, batch_size, from_dense, n_reads=2): 30 | 31 | 32 | queueio_name = "queueio_{}".format(uuid.uuid4()) 33 | 34 | 35 | # First, create a file to write to: 36 | file_name = str(tmpdir + "/test_queueio_tensor3d_{}.h5".format(queueio_name)) 37 | 38 | # Next, write some tensor3ds to that file: 39 | if from_dense: 40 | create_dense_tensor3d_file(file_name, rand_num_events=25) 41 | else: 42 | create_tensor3d_file(file_name, rand_num_events=25) 43 | 44 | 45 | if from_dense: 46 | tensor_type = "dense" 47 | else: 48 | tensor_type = "sparse" 49 | 50 | # Now, let's get the configuration of a queueio object: 51 | default_config = larcv.QueueProcessor.default_config() 52 | default_config["InputFiles"].append(file_name) 53 | 54 | # Add the batch filler to the default config: 55 | default_config["Verbosity"] = 0 56 | default_config["ProcessDriver"]["ProcessName"].append(f"test_{queueio_name}") 57 | default_config["ProcessDriver"]["ProcessType"].append("BatchFillerTensor3D") 58 | 59 | process_list = {f"test_{queueio_name}": 60 | { 61 | "TensorProducer": "test", 62 | "TensorType": tensor_type, 63 | "Channels": [0,], 64 | "EmptyVoxelValue": -999., 65 | "Augment": True, 66 | } 67 | } 68 | 69 | if default_config["ProcessDriver"]["ProcessList"] is None: 70 | default_config["ProcessDriver"]["ProcessList"] = process_list 71 | else: 72 | default_config["ProcessDriver"]["ProcessList"].append(process_list) 73 | 74 | 75 | queue_proc = larcv.QueueProcessor() 76 | queue_proc.configure(default_config) 77 | 78 | 79 | indexes = numpy.arange(batch_size*n_reads*2) % queue_proc.get_n_entries() 80 | 81 | queue_proc.set_next_batch(indexes[0:batch_size]) 82 | print(queue_proc.is_reading()) 83 | queue_proc.prepare_next() 84 | time.sleep(2) 85 | while queue_proc.is_reading(): 86 | print("Sleeping") 87 | time.sleep(0.5) 88 | queue_proc.pop_current_data() 89 | 90 | for i in range(n_reads): 91 | batch=indexes[i*batch_size:(i+1)*batch_size] 92 | queue_proc.set_next_batch(batch) 93 | queue_proc.prepare_next() 94 | 95 | while queue_proc.is_reading(): 96 | print("Sleeping") 97 | time.sleep(0.1) 98 | 99 | 100 | 101 | if __name__ == "__main__": 102 | test_tensor3d_queueio("./", make_copy=False, batch_size=1, from_dense=False, n_reads=2) 103 | test_tensor3d_queueio("./", make_copy=False, batch_size=1, from_dense=True, n_reads=2) 104 | print("Success") 105 | -------------------------------------------------------------------------------- /tests/larcv/core/base/test_base.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | def test_import_base_full(): 5 | import larcv 6 | lib = larcv.logger() 7 | lib = larcv.larcv_base() 8 | lib = larcv.Watch() 9 | # Config manager is deliberately excluded: 10 | # lib = base.ConfigManager() -------------------------------------------------------------------------------- /tests/larcv/core/base/test_json.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import larcv 3 | 4 | def test_get_json_from_larcv(): 5 | a = larcv.larcv_base.default_config() 6 | print(a) 7 | print(type(a)) 8 | 9 | 10 | # def test_pass_json_to_larcv(): 11 | 12 | # # From larcv base, get the default config: 13 | # assert True -------------------------------------------------------------------------------- /tests/larcv/core/base/test_larbys.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | 5 | def test_larbys(): 6 | import larcv 7 | lib = larcv.larbys() 8 | # Config manager is deliberately excluded: 9 | # lib = base.ConfigManager() 10 | 11 | def test_larbys_exception(): 12 | import larcv 13 | # Creating a PSet object without a name will cause an exception. 14 | # We can catch that and verify we're getting a larbys exception: 15 | try: 16 | pset = larcv.PSet() 17 | return False 18 | except: 19 | return True -------------------------------------------------------------------------------- /tests/larcv/core/base/test_logger.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | # def test_logger(): 4 | # from larcv import base 5 | # # logger = base.logger.get_shared() 6 | # for level in [base.kDEBUG, base.kINFO, base.kNORMAL, base.kWARNING, base.kERROR]: 7 | # base.logger.force_level(level) 8 | 9 | # assert(level == base.logger.get_shared().level()) -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/sparse_tensor.py: -------------------------------------------------------------------------------- 1 | from test_write_sparse_cluster import * 2 | 3 | rand_num_events = 10 4 | dimension = 3 5 | n_projections = 3 6 | 7 | 8 | voxel_set_list = build_sparse_cluster_list(rand_num_events, n_projections=n_projections) 9 | 10 | tempfile = "cluster.h5" 11 | write_sparse_clusters(tempfile, voxel_set_list, dimension, n_projections) 12 | read_voxel_set_list = read_sparse_clusters(tempfile, dimension) 13 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_bbox.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy 3 | import pytest 4 | 5 | import larcv 6 | 7 | from random import Random 8 | random = Random() 9 | 10 | N_CHECKS = 1 11 | 12 | def bbox_factory(dimension): 13 | if dimension == 2: 14 | return larcv.BBox2D() 15 | elif dimension == 3: 16 | return larcv.BBox3D() 17 | else: 18 | raise Exception("Can't do anything with dimension == ", dimension) 19 | 20 | 21 | @pytest.mark.parametrize('dimension', [2,3]) 22 | def test_import(dimension): 23 | 24 | try: 25 | bbox = bbox_factory(dimension) 26 | except: 27 | assert False 28 | 29 | @pytest.mark.parametrize('dimension', [2,3]) 30 | @pytest.mark.parametrize('execution_number', range(N_CHECKS)) 31 | def test_default_constructor(dimension, execution_number): 32 | 33 | bbox = bbox_factory(dimension) 34 | 35 | 36 | assert(bbox.centroid() == [0.0]*dimension) 37 | assert(bbox.half_length() == [1.0]*dimension) 38 | 39 | 40 | 41 | @pytest.mark.parametrize('dimension', [2,3]) 42 | @pytest.mark.parametrize('execution_number', range(N_CHECKS)) 43 | def test_filled_constructor(dimension, execution_number): 44 | 45 | 46 | centroid = [random.uniform(-1e4, 1e4) for d in range(dimension)] 47 | half_length = [random.uniform(-1e4, 1e4) for d in range(dimension)] 48 | 49 | if dimension == 2: 50 | bbox = larcv.BBox2D(centroid, half_length) 51 | elif dimension == 3: 52 | bbox = larcv.BBox3D(centroid, half_length) 53 | 54 | 55 | assert(bbox.centroid() == centroid) 56 | assert(bbox.half_length() == half_length) 57 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_dataformat.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | # def test_import_dataformat_top(): 4 | # from larcv import dataformat 5 | 6 | 7 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_dataformat_binding.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import larcv 4 | 5 | from random import Random 6 | random = Random() 7 | 8 | 9 | # Testing import of Point.h objects: 10 | def test_import_Point_h(): 11 | pt = larcv.Point2D() 12 | pt = larcv.Point3D() 13 | 14 | def test_import_BBox_h(): 15 | bb = larcv.BBox2D() 16 | bb = larcv.BBox3D() 17 | 18 | 19 | # Testing import of Vertex.h objects: 20 | def test_import_Vertex_h(): 21 | vert1 = larcv.Vertex() 22 | vert2 = larcv.Vertex(1,2,3,4) 23 | assert(vert1 != vert2) 24 | assert(vert1 < vert2) 25 | vert2.reset() 26 | assert(vert1 == vert2) 27 | 28 | # Testing import of Particle.h objects: 29 | def test_import_Particle_h(): 30 | part1 = larcv.Particle() 31 | 32 | # Testing import of Voxel.h objects: 33 | def test_import_Voxel_h_Voxel(): 34 | 35 | v = larcv.Voxel() 36 | 37 | def test_import_Voxel_h_VoxelSet(): 38 | 39 | vs = larcv.VoxelSet() 40 | 41 | def test_import_Voxel_h_VoxelSetArray(): 42 | 43 | vsa = larcv.VoxelSetArray() 44 | 45 | def test_import_ImageMeta_h(): 46 | 47 | im = larcv.ImageMeta2D() 48 | im = larcv.ImageMeta3D() 49 | 50 | def test_import_EventID_h(): 51 | 52 | im = larcv.EventID() 53 | 54 | # Event Base is abstract and therefore should NOT import: 55 | def test_import_EventBase_h(): 56 | 57 | try: 58 | eb = larcv.EventBase() 59 | assert(False) 60 | except: 61 | assert(True) 62 | 63 | 64 | # There is no need to wrap data product factory, so no test: 65 | # def test_import_DataProductFactory_h(): 66 | # # dfb = larcv.DataProductFactoryBase() 67 | # df = larcv.DataProductFactory() 68 | 69 | def test_import_IOManager_h(): 70 | io = larcv.IOManager() 71 | 72 | def test_import_EventParticle_h(): 73 | ep = larcv.EventParticle() 74 | 75 | 76 | # def test_vector_double(): 77 | # vec = larcv.VectorOfDouble() 78 | # vec.push_back(random.uniform(-1e5, 1e5)) 79 | 80 | # def test_vector_sizet(): 81 | # vec = larcv.VectorOfSizet() 82 | # vec.push_back(random.randint(1, 2e4)) 83 | 84 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_eventid.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy 3 | import pytest 4 | 5 | 6 | 7 | def test_import_eventID(): 8 | import larcv 9 | 10 | a = larcv.EventID() 11 | 12 | # By Default, it should be invalid on construction: 13 | 14 | assert (a.valid() == False) 15 | 16 | return True 17 | 18 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_io.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | 5 | import larcv 6 | 7 | @pytest.fixture() 8 | def tempfile(tmpdir): 9 | return str(tmpdir.dirpath() + "/test.h5") 10 | 11 | @pytest.fixture() 12 | def rand_num_events(): 13 | return random.randint(1, 10) 14 | 15 | def test_augment_config(): 16 | io_manager = larcv.IOManager() 17 | # Static default config: 18 | default_config = larcv.IOManager.default_config() 19 | 20 | 21 | default_config["Input"]["InputFiles"] = ['temp.h5'] 22 | io_manager.configure(default_config) 23 | 24 | print(io_manager.get_config()) 25 | 26 | def write_temp_file(tempfile, rand_num_events): 27 | 28 | 29 | # This function is purely to test the IO read write capabilities. 30 | io_manager = larcv.IOManager(larcv.IOManager.kWRITE) 31 | io_manager.set_out_file(tempfile) 32 | io_manager.initialize() 33 | for i in range(rand_num_events): 34 | io_manager.set_id(0,0,i) 35 | io_manager.save_entry() 36 | io_manager.finalize() 37 | 38 | def read_temp_file(tempfile, rand_num_events): 39 | 40 | # This function is purely to test the IO read write capabilities. 41 | io_manager = larcv.IOManager(larcv.IOManager.kREAD) 42 | io_manager.add_in_file(tempfile) 43 | io_manager.initialize() 44 | 45 | return io_manager.get_n_entries() 46 | 47 | 48 | def test_file_open(tempfile, rand_num_events): 49 | 50 | write_temp_file(tempfile, rand_num_events) 51 | n_read = read_temp_file(tempfile, rand_num_events) 52 | 53 | assert(n_read == rand_num_events) 54 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_point.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import numpy 3 | import pytest 4 | 5 | from random import Random 6 | random = Random() 7 | 8 | N_CHECKS = 1 9 | 10 | @pytest.mark.parametrize('dimension', [2,3]) 11 | def test_import(dimension): 12 | import larcv 13 | if dimension == 2: 14 | pt = larcv.Point2D() 15 | elif dimension == 3: 16 | pt = larcv.Point3D() 17 | else: 18 | assert False 19 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_tensor.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import larcv 3 | import numpy 4 | 5 | from random import Random 6 | random = Random() 7 | 8 | 9 | @pytest.mark.parametrize('dimension', [1, 2, 3, 4]) 10 | def test_Tensor(dimension): 11 | 12 | # Create a numpy array of the specified dimension with random values: 13 | 14 | 15 | shape = [] 16 | for dim in range(dimension): 17 | if dimension < 3: 18 | shape.append(random.randint(1, 1e3)) 19 | else: 20 | shape.append(random.randint(1, 20)) 21 | 22 | raw_image = numpy.random.random(shape).astype("float32") 23 | 24 | # Create a Tensor from the array: 25 | if dimension == 1: 26 | t = larcv.Tensor1D(raw_image) 27 | elif dimension == 2: 28 | t = larcv.Tensor2D(raw_image) 29 | elif dimension == 3: 30 | t = larcv.Tensor3D(raw_image) 31 | elif dimension == 4: 32 | t = larcv.Tensor4D(raw_image) 33 | 34 | assert t.size() == numpy.prod(shape) 35 | 36 | for i in range(20): 37 | index = random.randint(0,t.size() - 1) 38 | assert t.pixel(index) == raw_image.take(index) 39 | 40 | nd_t = t.as_array() 41 | 42 | assert (nd_t - raw_image).sum() < 1e-6 43 | 44 | # Verify the tensor and array match by checking elements: 45 | return True 46 | 47 | @pytest.mark.parametrize('dimension', [1, 2, 3, 4]) 48 | @pytest.mark.parametrize('pooling', [larcv.kPoolAverage, larcv.kPoolMax, larcv.kPoolSum]) 49 | def test_tensor_compression(dimension, pooling): 50 | 51 | try: 52 | from skimage.measure import block_reduce 53 | except: 54 | pytest.skip("Could not import skimage") 55 | 56 | shape = [] 57 | for dim in range(dimension): 58 | shape.append(64) 59 | 60 | raw_image = numpy.random.random(shape).astype("float32") 61 | # Create a Tensor from the array: 62 | if dimension == 1: 63 | t = larcv.Tensor1D(raw_image) 64 | elif dimension == 2: 65 | t = larcv.Tensor2D(raw_image) 66 | elif dimension == 3: 67 | t = larcv.Tensor3D(raw_image) 68 | elif dimension == 4: 69 | t = larcv.Tensor4D(raw_image) 70 | 71 | # Pooling in numpy is not trivial. So instead of checking every element, 72 | # We will check against random elements. 73 | 74 | compression = 2 75 | 76 | t_compressed = t.compress(compression, pooling) 77 | 78 | compressed_numpy = t_compressed.as_array() 79 | 80 | # Use skimage to compress the original: 81 | kernel = (compression,) * dimension 82 | if pooling == larcv.kPoolAverage: 83 | sk_reduced = block_reduce(raw_image, kernel,func=numpy.mean) 84 | elif pooling == larcv.kPoolMax: 85 | sk_reduced = block_reduce(raw_image, kernel,func=numpy.max) 86 | elif pooling == larcv.kPoolSum: 87 | sk_reduced = block_reduce(raw_image, kernel,func=numpy.sum) 88 | 89 | # Do some checks: 90 | assert numpy.abs( (compressed_numpy.sum() - sk_reduced.sum()) / sk_reduced.sum() )< 1e-6 91 | max_index = numpy.prod(compressed_numpy.shape) 92 | for i in range(50): 93 | index = numpy.random.randint(0, max_index) 94 | assert numpy.abs(compressed_numpy.take(index) - sk_reduced.take(index) )< 1e-4 95 | 96 | 97 | if __name__ == '__main__': 98 | test_Tensor(1) 99 | test_Tensor(2) 100 | test_Tensor(3) 101 | test_Tensor(4) 102 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_write_particle.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import random 4 | import larcv 5 | 6 | from larcv import data_generator 7 | 8 | @pytest.fixture() 9 | def tempfile(tmpdir): 10 | return str(tmpdir.dirpath() + "/test_particle.h5") 11 | 12 | @pytest.fixture() 13 | def rand_num_events(): 14 | return random.randint(5, 100) 15 | 16 | 17 | 18 | 19 | def test_write_particles(tmpdir, rand_num_events): 20 | 21 | tempfile = str(tmpdir + "/test_write_particles.h5") 22 | 23 | data_generator.write_particles(tempfile, rand_num_events) 24 | n_read = data_generator.read_particles(tempfile) 25 | 26 | assert(n_read == rand_num_events) 27 | 28 | # def test_read_particles_coredriver(tmpdir, rand_num_events): 29 | 30 | # tempfile = str(tmpdir + "/test_write_particles.h5") 31 | 32 | # write_particles(tempfile, rand_num_events) 33 | # n_read = read_particles(tempfile, True) 34 | # assert(n_read == rand_num_events) 35 | 36 | if __name__ == '__main__': 37 | tmpdir = "./" 38 | rand_num_events = 5 39 | test_write_particles(tmpdir, rand_num_events) 40 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_write_sparse_cluster.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import larcv 4 | 5 | from random import Random 6 | random = Random(1) 7 | 8 | from larcv import data_generator 9 | 10 | @pytest.fixture() 11 | def tempfile(tmpdir): 12 | return str(tmpdir.dirpath() + "/test_sparse.h5") 13 | 14 | @pytest.fixture() 15 | def rand_num_events(): 16 | return random.randint(5, 100) 17 | 18 | 19 | 20 | # @pytest.mark.parametrize() 21 | @pytest.mark.parametrize('dimension', [2, 3]) 22 | @pytest.mark.parametrize('n_projections', [1, 2, 3]) 23 | def test_write_sparse_clusters(tmpdir, rand_num_events, dimension, n_projections): 24 | 25 | voxel_set_array_list = data_generator.build_sparse_cluster_list(rand_num_events, n_projections) 26 | random_file_name = str(tmpdir + "/test_write_sparse_clusters.h5") 27 | data_generator.write_sparse_clusters(random_file_name, voxel_set_array_list, dimension, n_projections) 28 | 29 | 30 | 31 | @pytest.mark.parametrize('dimension', [2,3]) 32 | @pytest.mark.parametrize('n_projections', [1,2,3]) 33 | def test_read_write_sparse_clusters(tmpdir, rand_num_events, dimension, n_projections): 34 | 35 | import numpy 36 | 37 | 38 | voxel_set_array_list = data_generator.build_sparse_cluster_list(rand_num_events, n_projections) 39 | random_file_name = str(tmpdir + "/test_write_sparse_clusters.h5") 40 | 41 | print(len(voxel_set_array_list)) 42 | for e in voxel_set_array_list: 43 | print(" " + str(len(e))) 44 | for p in e: 45 | print(" " + str(len(p))) 46 | 47 | data_generator.write_sparse_clusters(random_file_name, voxel_set_array_list, dimension, n_projections) 48 | read_voxel_set_array_list = data_generator.read_sparse_clusters(random_file_name, dimension) 49 | 50 | # Check the same number of events came back: 51 | assert(len(read_voxel_set_array_list) == rand_num_events) 52 | for event in range(rand_num_events): 53 | # Check the same number of projections per event: 54 | print 55 | assert(len(read_voxel_set_array_list[event]) == len(voxel_set_array_list[event])) 56 | 57 | for projection in range(n_projections): 58 | # CHeck the number of clusters in this projection: 59 | assert(len(read_voxel_set_array_list[event][projection]) == len(voxel_set_array_list[event][projection])) 60 | 61 | for cluster in range(len(read_voxel_set_array_list[event][projection])): 62 | # Check the same number of voxels: 63 | input_voxelset = voxel_set_array_list[event][projection][cluster] 64 | read_voxelset = read_voxel_set_array_list[event][projection][cluster] 65 | assert(read_voxelset['n_voxels'] == input_voxelset['n_voxels']) 66 | 67 | # Check voxel properties: 68 | # Sum of indexes 69 | # Sum of values 70 | # std of values 71 | if input_voxelset['n_voxels'] == 0: 72 | continue 73 | # print(input_voxelset['values']) 74 | assert(numpy.sum(input_voxelset['indexes']) == numpy.sum(read_voxelset['indexes'])) 75 | assert( abs( numpy.sum(input_voxelset['values']) - numpy.sum(read_voxelset['values']) ) < 1e-3 ) 76 | assert( abs( numpy.std(input_voxelset['values']) - numpy.std(read_voxelset['values']) ) < 1e-3 ) 77 | 78 | 79 | 80 | 81 | if __name__ == '__main__': 82 | tmpdir = "./" 83 | rand_num_events = 5 84 | n_projections = 3 85 | dimension = 2 86 | test_read_write_sparse_clusters(tmpdir, rand_num_events, dimension, n_projections) 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_write_sparse_tensor.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import larcv 4 | 5 | from random import Random 6 | random = Random() 7 | 8 | from larcv import data_generator 9 | 10 | @pytest.fixture() 11 | def rand_num_events(): 12 | return random.randint(5, 100) 13 | 14 | 15 | 16 | @pytest.mark.parametrize('dimension', [2, 3]) 17 | @pytest.mark.parametrize('n_projections', [1, 2, 3]) 18 | def test_write_sparse_tensors(tmpdir, rand_num_events, dimension, n_projections): 19 | 20 | voxel_set_list = data_generator.build_sparse_tensor(rand_num_events, n_projections = n_projections) 21 | 22 | random_file_name = str(tmpdir + "/test_write_sparse_tensors.h5") 23 | 24 | data_generator.write_sparse_tensors(random_file_name, voxel_set_list, dimension, n_projections) 25 | 26 | 27 | @pytest.mark.parametrize('dimension', [2, 3]) 28 | @pytest.mark.parametrize('n_projections', [1, 2, 3]) 29 | def test_read_write_sparse_tensors(tmpdir, rand_num_events, dimension, n_projections): 30 | 31 | import numpy 32 | 33 | random_file_name = str(tmpdir + "/test_write_read_sparse_tensors.h5") 34 | 35 | voxel_set_list = data_generator.build_sparse_tensor(rand_num_events, n_projections = n_projections) 36 | 37 | data_generator.write_sparse_tensors(random_file_name, voxel_set_list, dimension, n_projections) 38 | read_voxel_set_list = data_generator.read_sparse_tensors(random_file_name, dimension) 39 | 40 | # Check the same number of events came back: 41 | assert(len(read_voxel_set_list) == rand_num_events) 42 | for event in range(rand_num_events): 43 | # Check the same number of projections per event: 44 | assert(len(read_voxel_set_list[event]) == len(voxel_set_list[event])) 45 | for projection in range(n_projections): 46 | # Check the same number of voxels: 47 | input_voxelset = voxel_set_list[event][projection] 48 | read_voxelset = read_voxel_set_list[event][projection] 49 | assert(read_voxelset['n_voxels'] == input_voxelset['n_voxels']) 50 | 51 | # Check voxel properties: 52 | # Sum of indexes 53 | # Sum of values 54 | # std of values 55 | if input_voxelset['n_voxels'] == 0: 56 | continue 57 | print(input_voxelset['values']) 58 | assert(numpy.sum(input_voxelset['indexes']) == numpy.sum(read_voxelset['indexes'])) 59 | assert( abs( numpy.sum(input_voxelset['values']) - numpy.sum(read_voxelset['values']) ) < 1e-3 ) 60 | assert( abs( numpy.std(input_voxelset['values']) - numpy.std(read_voxelset['values']) ) < 1e-3 ) 61 | 62 | 63 | 64 | if __name__ == '__main__': 65 | tmpdir = "./" 66 | rand_num_events = 5 67 | n_projections = 3 68 | dimension = 2 69 | test_read_write_sparse_tensors(tmpdir, rand_num_events, dimension, n_projections) 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/test_write_tensor.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import larcv 4 | 5 | from larcv import data_generator 6 | 7 | from random import Random 8 | random = Random() 9 | 10 | @pytest.fixture() 11 | def rand_num_events(): 12 | return random.randint(1, 5) 13 | 14 | 15 | @pytest.mark.parametrize('dimension', [1, 2, 3]) 16 | @pytest.mark.parametrize('n_projections', [1,2,3]) 17 | def test_write_tensor(tmpdir, rand_num_events, dimension, n_projections): 18 | 19 | event_image_list = data_generator.build_tensor(rand_num_events, n_projections=n_projections, dimension=dimension) 20 | 21 | random_file_name = str(tmpdir + "/test_write_tensor.h5") 22 | 23 | data_generator.write_tensor(random_file_name, event_image_list, dimension) 24 | 25 | 26 | @pytest.mark.parametrize('dimension', [1, 2, 3]) 27 | @pytest.mark.parametrize('n_projections', [1,2,3]) 28 | def test_write_read_tensor(tmpdir, rand_num_events, dimension, n_projections): 29 | 30 | import numpy 31 | 32 | random_file_name = str(tmpdir + "/test_write_read_tensor.h5") 33 | 34 | event_image_list = data_generator.build_tensor(rand_num_events, n_projections=n_projections, dimension=dimension) 35 | 36 | data_generator.write_tensor(random_file_name, event_image_list, dimension) 37 | 38 | read_event_image_list = data_generator.read_tensor(random_file_name, dimension) 39 | 40 | # Check the same number of events came back: 41 | assert(len(read_event_image_list) == rand_num_events) 42 | for event in range(rand_num_events): 43 | # Check the same number of projections per event: 44 | assert(len(read_event_image_list[event]) == len(event_image_list[event])) 45 | for projection in range(n_projections): 46 | # Check the same number of voxels: 47 | input_image = event_image_list[event][projection] 48 | read_image = read_event_image_list[event][projection] 49 | 50 | assert(numpy.sum(input_image) - numpy.sum(read_image) < 1e-3) 51 | 52 | assert(numpy.sum(numpy.fabs(input_image)) - numpy.sum(numpy.fabs(read_image)) < 1e-3) 53 | 54 | assert(input_image.shape == read_image.shape) 55 | 56 | assert(numpy.mean(input_image - read_image) < 1e-3) 57 | 58 | 59 | break 60 | break 61 | 62 | 63 | 64 | 65 | 66 | if __name__ == '__main__': 67 | tmpdir = "./" 68 | rand_num_events = 3 69 | dimension = 1 70 | n_projections = 3 71 | # test_write_tensor(tmpdir, rand_num_events, dimension, n_projections) 72 | test_write_read_tensor(tmpdir, rand_num_events, dimension, n_projections) 73 | 74 | -------------------------------------------------------------------------------- /tests/larcv/core/dataformat/todo.txt: -------------------------------------------------------------------------------- 1 | convert between types (casting): 2 | - sparse to dense 3 | - dense to sparse 4 | 5 | transform on types: 6 | - flipping across axes 7 | - rescale 8 | - shifting along axes 9 | 10 | - downsampling (average, max, sum) 11 | 12 | - need these for tensor, sparse cluster, sparse tensor 13 | 14 | 15 | Cast event_* to numpy directly 16 | - event_tensor cast right to numpy ndarray of shape [1,N_views, [x, y, z ...]] 17 | - event_sparse cast right to numpy dense (through event tensor? straight to dense?) 18 | - event cluster need to think more 19 | 20 | QueueIO 21 | - map right to numpy: 22 | - dense to shape [B, n_views, [x,y,z,...]] 23 | - sparse to dense [B, n_views, [x,y,z,...]] 24 | - sparse to scnsparse 25 | - sparse to graph 26 | - always output as numpy that can feed right in to TF/torch tensors 27 | -------------------------------------------------------------------------------- /tests/larcv/core/processor/test_processor.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | 5 | def test_import_processor_top(): 6 | import larcv 7 | 8 | def test_import_processor(): 9 | import larcv 10 | 11 | lib = larcv.ProcessDriver() 12 | # lib = larcv.ProcessFactory() 13 | 14 | def test_import_processor_base(): 15 | import larcv 16 | 17 | # Processes base is virtual and should NOT import: 18 | try: 19 | lib = larcv.ProcessBase("test") 20 | assert(False) 21 | except: 22 | assert(True) 23 | 24 | def test_process_driver(): 25 | import larcv 26 | 27 | lib = larcv.ProcessDriver("test") 28 | -------------------------------------------------------------------------------- /tests/test_larcv.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | def test_larcv(): 4 | import larcv --------------------------------------------------------------------------------