├── katsdpbfingest ├── test │ ├── __init__.py │ └── test_bf_ingest_server.py ├── __init__.py ├── session.h ├── stats_ops.h ├── file_writer.py ├── stats.h ├── common.cpp ├── writer.h ├── receiver.h ├── bf_ingest.cpp ├── session.cpp ├── utils.py ├── common.h ├── writer.cpp ├── stats.cpp ├── units.h ├── receiver.cpp └── bf_ingest_server.py ├── .gitmodules ├── mypy.ini ├── .gitignore ├── test-requirements.txt ├── .dockerignore ├── pyproject.toml ├── .flake8 ├── .pre-commit-config.yaml ├── Jenkinsfile ├── requirements.txt ├── scripts └── bf_ingest.py ├── LICENSE ├── Dockerfile └── setup.py /katsdpbfingest/test/__init__.py: -------------------------------------------------------------------------------- 1 | """Unit testing for katsdpbfingest.""" 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "spead2"] 2 | path = spead2 3 | url = https://github.com/ska-sa/spead2 4 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.6 3 | ignore_missing_imports = True 4 | files = katsdpbfingest, scripts 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /katsdpbfingest/_bf_ingest*.so 2 | *.py[cod] 3 | build 4 | dist 5 | temp 6 | cover 7 | .coverage 8 | .mypy_cache 9 | *.egg-info 10 | pip-wheel-metadata 11 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | -c https://raw.githubusercontent.com/ska-sa/katsdpdockerbase/master/docker-base-build/base-requirements.txt 2 | 3 | asynctest 4 | coverage 5 | nose 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | temp 4 | cover 5 | .coverage 6 | doc/_build 7 | katsdpbfingest/_bf_ingest.so 8 | *.egg-info 9 | *.pyc 10 | __pycache__ 11 | Dockerfile 12 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "katversion", "jinja2==2.11.2", "markupsafe==2.0.1", "pycparser==2.20", "pkgconfig==1.5.1", "pybind11==2.7.1"] 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = spead2 3 | max-line-length = 100 4 | ignore = 5 | # whitespace before ':' - flake8 doesn't handle slices properly 6 | E203 7 | # line break after binary operator 8 | W504 9 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pycqa/flake8 3 | rev: 3.9.2 4 | hooks: 5 | - id: flake8 6 | - repo: https://github.com/pre-commit/mirrors-mypy 7 | rev: v0.780 8 | hooks: 9 | - id: mypy 10 | args: [] 11 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | 3 | @Library('katsdpjenkins') _ 4 | katsdp.killOldJobs() 5 | 6 | katsdp.setDependencies([ 7 | 'ska-sa/katsdpdockerbase/master', 8 | 'ska-sa/katsdpservices/master', 9 | 'ska-sa/katsdptelstate/master']) 10 | katsdp.standardBuild(push_external: true) 11 | katsdp.mail('sdpdev+katsdpbfingest@ska.ac.za') 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -d https://raw.githubusercontent.com/ska-sa/katsdpdockerbase/master/docker-base-build/base-requirements.txt 2 | 3 | aiokatcp 4 | h5py==3.8.0 5 | numpy 6 | spead2 7 | 8 | katsdpservices[argparse,aiomonitor] @ git+https://github.com/ska-sa/katsdpservices 9 | katsdptelstate @ git+https://github.com/ska-sa/katsdptelstate 10 | -------------------------------------------------------------------------------- /katsdpbfingest/__init__.py: -------------------------------------------------------------------------------- 1 | """Katsdpingest library.""" 2 | 3 | # BEGIN VERSION CHECK 4 | # Get package version when locally imported from repo or via -e develop install 5 | try: 6 | import katversion as _katversion 7 | except ImportError: 8 | import time as _time 9 | __version__ = "0.0+unknown.{}".format(_time.strftime('%Y%m%d%H%M')) 10 | else: 11 | __version__ = _katversion.get_version(__path__[0]) # type: ignore 12 | # END VERSION CHECK 13 | -------------------------------------------------------------------------------- /katsdpbfingest/session.h: -------------------------------------------------------------------------------- 1 | #ifndef SESSION_H 2 | #define SESSION_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "common.h" 9 | #include "receiver.h" 10 | #include "session.h" 11 | 12 | class session 13 | { 14 | private: 15 | const session_config config; 16 | receiver recv; 17 | std::future run_future; 18 | 19 | void run_impl(); // internal implementation of run 20 | void run(); // runs in a separate thread 21 | 22 | public: 23 | explicit session(const session_config &config); 24 | ~session(); 25 | 26 | void join(); 27 | void stop_stream(); 28 | 29 | spead2::recv::stream_stats get_counters() const; 30 | std::int64_t get_first_timestamp() const; 31 | 32 | // For unit tests 33 | void add_tcp_reader(const spead2::socket_wrapper &acceptor); 34 | }; 35 | 36 | #endif // SESSION_H 37 | -------------------------------------------------------------------------------- /katsdpbfingest/stats_ops.h: -------------------------------------------------------------------------------- 1 | /* This file is included multiple times from the main code, each time 2 | * with a different value for TARGET. It thus does *not* have the normal 3 | * include guard. 4 | * 5 | * See https://gcc.gnu.org/wiki/FunctionMultiVersioning for more details. 6 | * With GCC 6 it would be possible to avoid this multiple include trick 7 | * and just use the target_clones attribute, but we're targeting GCC 5.4. 8 | */ 9 | 10 | // Python extensions are built with -fwrapv, but it interferes with vectorisation 11 | #pragma GCC push_options 12 | #pragma GCC optimize("no-wrapv") 13 | 14 | #ifdef TARGET 15 | [[gnu::target(TARGET)]] 16 | #endif 17 | // N is the number of sample values 18 | uint32_t power_sum(int N, const int8_t *data) 19 | { 20 | uint32_t accum = 0; 21 | for (int i = 0; i < 2 * N; i++) 22 | { 23 | int16_t v = data[i]; 24 | accum += v * v; 25 | } 26 | return accum; 27 | } 28 | 29 | #ifdef TARGET 30 | [[gnu::target(TARGET)]] 31 | #endif 32 | // N is the number of samples 33 | uint16_t count_saturated(int N, const int8_t *data) 34 | { 35 | uint16_t ans = 0; 36 | for (int i = 0; i < N; i++) 37 | { 38 | int8_t re = data[2 * i]; 39 | int8_t im = data[2 * i + 1]; 40 | // Using | instead of || helps GCC with autovectorisation 41 | ans += (re == 127) | (re == -128) | (im == 127) | (im == -128); 42 | } 43 | return ans; 44 | } 45 | 46 | #pragma GCC pop_options 47 | -------------------------------------------------------------------------------- /scripts/bf_ingest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import signal 4 | import asyncio 5 | import logging 6 | import os 7 | import sys 8 | 9 | import katsdpservices 10 | 11 | from katsdpbfingest.bf_ingest_server import KatcpCaptureServer, parse_args 12 | 13 | 14 | def on_shutdown(server: KatcpCaptureServer) -> None: 15 | logging.info('Shutting down') 16 | loop = asyncio.get_event_loop() 17 | loop.remove_signal_handler(signal.SIGINT) 18 | loop.remove_signal_handler(signal.SIGTERM) 19 | server.halt() 20 | 21 | 22 | def main() -> None: 23 | katsdpservices.setup_logging() 24 | katsdpservices.setup_restart() 25 | 26 | args = parse_args() 27 | if args.log_level is not None: 28 | logging.root.setLevel(args.log_level.upper()) 29 | if args.file_base is None and args.stats is None: 30 | logging.warning('Neither --file-base nor --stats was given; nothing useful will happen') 31 | if args.file_base is not None and not os.access(args.file_base, os.W_OK): 32 | logging.error('Target directory (%s) is not writable', args.file_base) 33 | sys.exit(1) 34 | 35 | loop = asyncio.get_event_loop() 36 | server = KatcpCaptureServer(args, loop) 37 | loop.add_signal_handler(signal.SIGINT, lambda: on_shutdown(server)) 38 | loop.add_signal_handler(signal.SIGTERM, lambda: on_shutdown(server)) 39 | with katsdpservices.start_aiomonitor(loop, args, locals()): 40 | loop.run_until_complete(server.start()) 41 | loop.run_until_complete(server.join()) 42 | loop.close() 43 | 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2019, National Research Foundation (SARAO) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 20 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /katsdpbfingest/file_writer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Writes metadata to an HDF5 file. 3 | """ 4 | 5 | import logging 6 | 7 | import h5py 8 | import numpy as np 9 | import katsdptelstate 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | _TSTATE_DATASET = '/TelescopeState' 14 | 15 | 16 | def set_telescope_state(h5_file: h5py.File, 17 | tstate: katsdptelstate.TelescopeState, 18 | base_path: str = _TSTATE_DATASET, 19 | start_timestamp: float = 0.0) -> None: 20 | """Write raw pickled telescope state to an HDF5 file.""" 21 | tstate_group = h5_file.create_group(base_path) 22 | # include the subarray product id for use by the crawler to identify which 23 | # system the file belongs to. 24 | tstate_group.attrs['subarray_product_id'] = tstate.get('subarray_product_id', 'none') 25 | tstate_keys = tstate.keys() 26 | logger.info("Writing {} telescope state keys to {}".format(len(tstate_keys), base_path)) 27 | 28 | sensor_dtype = np.dtype( 29 | [('timestamp', np.float64), 30 | ('value', h5py.special_dtype(vlen=np.uint8))]) 31 | for key in tstate_keys: 32 | if tstate.key_type(key) == katsdptelstate.KeyType.MUTABLE: 33 | # retrieve all values for a particular key 34 | sensor_values = tstate.get_range(key, st=start_timestamp, 35 | include_previous=True, return_encoded=True) 36 | # swap value, timestamp to timestamp, value 37 | sensor_values = [(timestamp, np.frombuffer(value, dtype=np.uint8)) 38 | for (value, timestamp) in sensor_values] 39 | dset = np.rec.fromrecords(sensor_values, dtype=sensor_dtype, names='timestamp,value') 40 | tstate_group.create_dataset(key, data=dset) 41 | logger.debug("TelescopeState: Written {} values for key {} to file".format( 42 | len(dset), key)) 43 | else: 44 | tstate_group.attrs[key] = np.void(tstate.get(key, return_encoded=True)) 45 | logger.debug("TelescopeState: Key {} written as an attribute".format(key)) 46 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG KATSDPDOCKERBASE_REGISTRY=harbor.sdp.kat.ac.za/dpp 2 | 3 | FROM $KATSDPDOCKERBASE_REGISTRY/docker-base-build as build 4 | 5 | # Build libhdf5 from source so that the direct I/O VFD can be used. 6 | # The other flags are a subset of those used by debian.rules (subsetted 7 | # mostly because the flags were default anyway), except that Fortran is 8 | # disabled. 9 | # 10 | # The copy installed to /libhdf5-install is for the runtime image to copy from. 11 | USER root 12 | 13 | WORKDIR /tmp 14 | ENV HDF5_VERSION=1.10.3 15 | ARG KATSDPDOCKERBASE_MIRROR=http://sdp-services.kat.ac.za/mirror 16 | RUN mirror_wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.bz2 -O hdf5-$HDF5_VERSION.tar.bz2 17 | RUN tar -jxf hdf5-$HDF5_VERSION.tar.bz2 18 | WORKDIR /tmp/hdf5-$HDF5_VERSION 19 | RUN ./configure --prefix=/usr/local --enable-build-mode=production --enable-threadsafe \ 20 | --disable-fortran --enable-cxx --enable-direct-vfd \ 21 | --enable-unsupported 22 | RUN make -j4 23 | RUN make DESTDIR=/libhdf5-install install 24 | RUN make install 25 | RUN ldconfig 26 | RUN echo "Name: HDF5\nDescription: Hierarchical Data Format 5 (HDF5)\nVersion: $HDF5_VERSION\nRequires:\nCflags: -I/usr/local/include\nLibs: -L/usr/local/lib -lhdf5" \ 27 | > /usr/lib/x86_64-linux-gnu/pkgconfig/hdf5.pc 28 | USER kat 29 | 30 | # Install dependencies. We need to set library-dirs so that the new libhdf5 31 | # will be found. We must avoid using the h5py wheel, because it will contain 32 | # its own hdf5 libraries while we want to link to the system ones. 33 | ENV PATH="$PATH_PYTHON3" VIRTUAL_ENV="$VIRTUAL_ENV_PYTHON3" 34 | COPY --chown=kat:kat requirements.txt /tmp/install/requirements.txt 35 | WORKDIR /tmp/install 36 | RUN /bin/echo -e '[build_ext]\nlibrary-dirs=/usr/local/lib' > setup.cfg 37 | RUN install_pinned.py --no-binary=h5py -r /tmp/install/requirements.txt 38 | 39 | # Install the current package 40 | COPY --chown=kat:kat . /tmp/install/katsdpbfingest 41 | WORKDIR /tmp/install/katsdpbfingest 42 | RUN cp ../setup.cfg . 43 | RUN python ./setup.py clean 44 | RUN pip install --no-deps . 45 | RUN pip check 46 | 47 | ####################################################################### 48 | 49 | FROM $KATSDPDOCKERBASE_REGISTRY/docker-base-runtime 50 | LABEL maintainer="sdpdev+katsdpbfingest@ska.ac.za" 51 | 52 | COPY --from=build /libhdf5-install / 53 | USER root 54 | RUN ldconfig 55 | USER kat 56 | 57 | COPY --from=build --chown=kat:kat /home/kat/ve3 /home/kat/ve3 58 | ENV PATH="$PATH_PYTHON3" VIRTUAL_ENV="$VIRTUAL_ENV_PYTHON3" 59 | 60 | # Allow raw packets (for ibverbs raw QPs) 61 | USER root 62 | RUN setcap cap_net_raw+p /usr/local/bin/capambel 63 | USER kat 64 | 65 | EXPOSE 2050 66 | EXPOSE 7148/udp 67 | -------------------------------------------------------------------------------- /katsdpbfingest/stats.h: -------------------------------------------------------------------------------- 1 | #ifndef STATS_H 2 | #define STATS_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "common.h" 12 | 13 | /** 14 | * Statistics collection for signal displays. 15 | * 16 | * Because the values sent are tiny, transmission of each heap is done as fast 17 | * as possible and synchronously, rather than asynchronously at some defined 18 | * rate. This avoids the need to dedicate yet another thread to data 19 | * transmission, and the heaps should be small enough to fit entirely into 20 | * the various buffers between source and sink. 21 | * 22 | * When stats collection and disk write are both enabled, they run on the same 23 | * thread, so there is benefit in running much faster than real-time to make 24 | * more time available for disk writes. 25 | */ 26 | class stats_collector 27 | { 28 | private: 29 | /** 30 | * Backing data store for the dynamic data in a single heap. This is 31 | * grouped into its own structure to allow for double-buffering in 32 | * future. It's non-copyable because the heap is pre-constructed with 33 | * raw pointers. 34 | */ 35 | struct transmit_data : public boost::noncopyable 36 | { 37 | spead2::send::heap heap; 38 | /** 39 | * Packed per-channel data. The first half contains the power 40 | * spectrum, while the second half contains the fraction of samples 41 | * that are saturated. In both cases the imaginary part is all zeroes. 42 | */ 43 | std::vector> data; 44 | std::vector flags; ///< just data_lost if all samples lost 45 | std::uint64_t timestamp; ///< centre, in centiseconds since Unix epoch 46 | 47 | transmit_data(const session_config &config); 48 | }; 49 | 50 | /// Accumulated power per channel 51 | std::vector power_spectrum; 52 | /// Accumulated number of saturated samples per channel 53 | std::vector saturated; 54 | /// Number of valid samples collected 55 | std::vector weight; 56 | /// Persist allocation of data to send (only used transiently) 57 | transmit_data data; 58 | 59 | // Constants copied/derived from the session_config 60 | double sync_time; 61 | quantity scale_factor_timestamp; 62 | units::freq_system freq_sys; 63 | units::time_system time_sys; 64 | 65 | q::ticks interval; ///< transmit interval 66 | q::ticks start_timestamp{-1}; ///< first timestamp of current accumulation 67 | 68 | boost::asio::io_service io_service; 69 | spead2::send::udp_stream stream; 70 | spead2::send::heap data_heap; 71 | 72 | /// Synchronously send a heap 73 | void send_heap(const spead2::send::heap &heap); 74 | 75 | /// Flush the currently accumulated statistics 76 | void transmit(); 77 | 78 | public: 79 | stats_collector(const session_config &config); 80 | ~stats_collector(); 81 | 82 | /// Add a new slice of data 83 | void add(const slice &s); 84 | }; 85 | 86 | #endif // STATS_H 87 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from setuptools import setup, find_packages, Extension 3 | from distutils.command.build_ext import build_ext # type: ignore # typeshed doesn't capture it 4 | import configparser 5 | import glob 6 | import subprocess 7 | import os.path 8 | 9 | try: 10 | import pybind11 11 | import pkgconfig 12 | hdf5 = pkgconfig.parse('hdf5') 13 | pybind11_include = pybind11.get_include() 14 | except ImportError: 15 | # Just to make "./setup.py clean" work 16 | import collections 17 | hdf5 = collections.defaultdict(list) 18 | pybind11_include = '' 19 | 20 | 21 | tests_require = ['nose', 'spead2>=3.0.1', 'asynctest'] 22 | 23 | 24 | # Hack: this is copied and edited from spead2, so that we can run configure 25 | # inside the spead2 submodule. 26 | class BuildExt(build_ext): 27 | def run(self): 28 | self.mkpath(self.build_temp) 29 | subprocess.check_call(['./bootstrap.sh'], cwd='spead2') 30 | subprocess.check_call(os.path.abspath('spead2/configure'), cwd=self.build_temp) 31 | config = configparser.ConfigParser() 32 | config.read(os.path.join(self.build_temp, 'python-build.cfg')) 33 | for extension in self.extensions: 34 | extension.extra_compile_args.extend(config['compiler']['CFLAGS'].split()) 35 | extension.extra_link_args.extend(config['compiler']['LIBS'].split()) 36 | extension.include_dirs.insert(0, os.path.join(self.build_temp, 'include')) 37 | super().run() 38 | 39 | 40 | sources = (glob.glob('spead2/src/common_*.cpp') + 41 | glob.glob('spead2/src/recv_*.cpp') + 42 | glob.glob('spead2/src/send_*.cpp') + 43 | glob.glob('spead2/src/py_common.cpp') + 44 | glob.glob('katsdpbfingest/*.cpp')) 45 | # Generated files: might be missing from sources 46 | gen_sources = [ 47 | 'spead2/src/common_loader_ibv.cpp', 48 | 'spead2/src/common_loader_rdmacm.cpp', 49 | 'spead2/src/common_loader_mlx5dv.cpp' 50 | ] 51 | for source in gen_sources: 52 | if source not in sources: 53 | sources.append(source) 54 | extensions = [ 55 | Extension( 56 | '_bf_ingest', 57 | sources=sources, 58 | depends=(glob.glob('spead2/include/spead2/*.h') + 59 | glob.glob('katsdpbfingest/*.h')), 60 | language='c++', 61 | include_dirs=['spead2/include', pybind11_include] + hdf5['include_dirs'], 62 | define_macros=hdf5['define_macros'], 63 | extra_compile_args=['-std=c++14', '-g0', '-O3', '-fvisibility=hidden'], 64 | library_dirs=hdf5['library_dirs'], 65 | # libgcc needs to be explicitly linked for multi-function versioning 66 | libraries=['boost_system', 'hdf5_cpp', 'hdf5_hl', 'gcc'] + hdf5['libraries'] 67 | ) 68 | ] 69 | 70 | setup( 71 | name="katsdpbfingest", 72 | description="MeerKAT beamformer data capture", 73 | author="MeerKAT SDP team", 74 | author_email="sdpdev+katsdpbfingest@ska.ac.za", 75 | packages=find_packages(), 76 | ext_package='katsdpbfingest', 77 | ext_modules=extensions, 78 | cmdclass={'build_ext': BuildExt}, 79 | scripts=["scripts/bf_ingest.py"], 80 | install_requires=[ 81 | 'h5py', 82 | 'numpy', 83 | 'aiokatcp', 84 | 'katsdpservices[argparse,aiomonitor]', 85 | 'katsdptelstate >= 0.10', 86 | 'spead2 >= 3.0.1' 87 | ], 88 | extras_require={ 89 | 'test': tests_require 90 | }, 91 | tests_require=tests_require, 92 | use_katversion=True 93 | ) 94 | -------------------------------------------------------------------------------- /katsdpbfingest/common.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "common.h" 7 | 8 | static std::unique_ptr logger; 9 | 10 | void log_message(spead2::log_level level, const std::string &msg) 11 | { 12 | (*logger)(level, msg); 13 | } 14 | 15 | void set_logger(pybind11::object logger_object) 16 | { 17 | logger.reset(new spead2::log_function_python(logger_object)); 18 | } 19 | 20 | void clear_logger() 21 | { 22 | logger.reset(); 23 | } 24 | 25 | std::vector affinity_vector(int affinity) 26 | { 27 | if (affinity < 0) 28 | return {}; 29 | else 30 | return {affinity}; 31 | } 32 | 33 | std::size_t slice::bytes(q::samples n) 34 | { 35 | return 2 * sizeof(std::int8_t) * n.get(); 36 | } 37 | 38 | session_config::session_config(const std::string &filename) 39 | : filename(filename) 40 | { 41 | } 42 | 43 | void session_config::add_endpoint(const std::string &bind_host, std::uint16_t port) 44 | { 45 | endpoints.emplace_back(boost::asio::ip::address_v4::from_string(bind_host), port); 46 | } 47 | 48 | std::string session_config::get_interface_address() const 49 | { 50 | return interface_address.to_string(); 51 | } 52 | 53 | void session_config::set_interface_address(const std::string &address) 54 | { 55 | interface_address = boost::asio::ip::address_v4::from_string(address); 56 | } 57 | 58 | void session_config::set_stats_endpoint(const std::string &host, std::uint16_t port) 59 | { 60 | stats_endpoint = boost::asio::ip::udp::endpoint( 61 | boost::asio::ip::address_v4::from_string(host), port); 62 | } 63 | 64 | std::string session_config::get_stats_interface_address() const 65 | { 66 | return stats_interface_address.to_string(); 67 | } 68 | 69 | void session_config::set_stats_interface_address(const std::string &address) 70 | { 71 | stats_interface_address = boost::asio::ip::address_v4::from_string(address); 72 | } 73 | 74 | units::freq_system session_config::get_freq_system() const 75 | { 76 | return units::freq_system(channels_per_heap, channels / channels_per_heap); 77 | } 78 | 79 | units::time_system session_config::get_time_system() const 80 | { 81 | return units::time_system(ticks_between_spectra, 82 | spectra_per_heap, 83 | heaps_per_slice_time); 84 | } 85 | 86 | const session_config &session_config::validate() const 87 | { 88 | if (channels <= 0) 89 | throw std::invalid_argument("channels <= 0"); 90 | if (channels_per_heap <= 0) 91 | throw std::invalid_argument("channels_per_heap <= 0"); 92 | if (spectra_per_heap <= 0) 93 | throw std::invalid_argument("spectra_per_heap <= 0"); 94 | if (spectra_per_heap >= 32768) 95 | throw std::invalid_argument("spectra_per_heap >= 32768"); 96 | if (ticks_between_spectra <= 0) 97 | throw std::invalid_argument("ticks_between_spectra <= 0"); 98 | if (sync_time <= 0) 99 | throw std::invalid_argument("sync_time <= 0"); 100 | if (bandwidth <= 0) 101 | throw std::invalid_argument("bandwidth <= 0"); 102 | if (center_freq <= 0) 103 | throw std::invalid_argument("center_freq <= 0"); 104 | if (scale_factor_timestamp <= 0) 105 | throw std::invalid_argument("scale_factor_timestamp <= 0"); 106 | if (heaps_per_slice_time <= 0) 107 | throw std::invalid_argument("heaps_per_slice_time <= 0"); 108 | return *this; 109 | } 110 | -------------------------------------------------------------------------------- /katsdpbfingest/writer.h: -------------------------------------------------------------------------------- 1 | #ifndef WRITER_H 2 | #define WRITER_H 3 | 4 | #include 5 | #include 6 | #include "common.h" 7 | 8 | class hdf5_bf_raw_writer 9 | { 10 | private: 11 | units::freq_system freq_sys; 12 | units::time_system time_sys; 13 | std::size_t chunk_bytes; 14 | H5::DataSet dataset; 15 | 16 | public: 17 | hdf5_bf_raw_writer(H5::Group &parent, 18 | const units::freq_system &freq_sys, 19 | const units::time_system &time_sys, 20 | const char *name); 21 | 22 | void add(const slice &c); 23 | }; 24 | 25 | class hdf5_timestamps_writer 26 | { 27 | private: 28 | static constexpr hsize_t chunk = 1048576; 29 | H5::DataSet dataset; 30 | std::unique_ptr> buffer; 31 | hsize_t n_buffer = 0; 32 | hsize_t n_written = 0; 33 | const units::time_system time_sys; 34 | 35 | void flush(); 36 | public: 37 | 38 | hdf5_timestamps_writer(H5::Group &parent, const units::time_system &time_sys, 39 | const char *name); 40 | ~hdf5_timestamps_writer(); 41 | // Add a heap's worth of timestamps 42 | void add(q::ticks timestamp); 43 | }; 44 | 45 | /** 46 | * Memory storage for an HDF5 chunk of flags data. This covers the whole band 47 | * and also many heaps in time. 48 | */ 49 | struct flags_chunk 50 | { 51 | q::spectra spectrum{-1}; 52 | aligned_ptr data; 53 | 54 | explicit flags_chunk(q::heaps size); 55 | }; 56 | 57 | class hdf5_flags_writer : private window 58 | { 59 | private: 60 | friend class window; 61 | 62 | const unit_system freq_sys; 63 | const unit_system time_sys; 64 | const std::size_t chunk_bytes; 65 | q::slices_t n_slices{0}; ///< Total slices seen (including skipped ones) 66 | H5::DataSet dataset; 67 | 68 | static q::heaps heaps_per_slice( 69 | const units::freq_system &freq_sys, const units::time_system &time_sys); 70 | static q::slices compute_chunk_size_slices( 71 | const units::freq_system &freq_sys, const units::time_system &time_sys); 72 | static q::heaps compute_chunk_size_heaps( 73 | const units::freq_system &freq_sys, const units::time_system &time_sys); 74 | static std::size_t bytes(q::heaps n); 75 | 76 | void flush(flags_chunk &chunk); 77 | public: 78 | hdf5_flags_writer(H5::Group &parent, 79 | const units::freq_system &freq_sys, 80 | const units::time_system &time_sys, 81 | const char *name); 82 | ~hdf5_flags_writer(); 83 | void add(const slice &s); 84 | }; 85 | 86 | class hdf5_writer 87 | { 88 | private: 89 | q::ticks past_end_timestamp{-1}; 90 | H5::H5File file; 91 | H5::Group group; 92 | const units::freq_system freq_sys; 93 | const units::time_system time_sys; 94 | hdf5_bf_raw_writer bf_raw; 95 | hdf5_timestamps_writer timestamps; 96 | hdf5_flags_writer flags; 97 | 98 | static H5::FileAccPropList make_fapl(bool direct); 99 | 100 | public: 101 | hdf5_writer(const std::string &filename, bool direct, 102 | const units::freq_system &freq_sys, 103 | const units::time_system &time_sys); 104 | void add(const slice &s); 105 | int get_fd() const; 106 | }; 107 | 108 | #endif // COMMON_H 109 | -------------------------------------------------------------------------------- /katsdpbfingest/receiver.h: -------------------------------------------------------------------------------- 1 | #ifndef RECEIVER_H 2 | #define RECEIVER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "common.h" 13 | 14 | class receiver; 15 | 16 | namespace counters 17 | { 18 | 19 | // These must be kept in sync with the calls to add_stat in make_stream_config 20 | static constexpr std::size_t metadata_heaps = 0; 21 | static constexpr std::size_t bad_timestamp_heaps = 1; 22 | static constexpr std::size_t bad_channel_heaps = 2; 23 | static constexpr std::size_t bad_length_heaps = 3; 24 | static constexpr std::size_t before_start_heaps = 4; 25 | static constexpr std::size_t data_heaps = 5; 26 | static constexpr std::size_t total_heaps = 6; 27 | static constexpr std::size_t bytes = 7; 28 | 29 | } // namespace counters 30 | 31 | /** 32 | * Collects data from the network. It has a built-in thread pool with one 33 | * thread, and runs almost entirely on that thread. 34 | */ 35 | class receiver 36 | { 37 | private: 38 | const session_config config; 39 | bool use_ibv = false; 40 | 41 | /// Depth of window 42 | static constexpr std::size_t window_size = 8; 43 | 44 | // Metadata copied from or computed from the session_config 45 | const q::channels channel_offset; 46 | const units::freq_system freq_sys; 47 | const units::time_system time_sys; 48 | const std::size_t payload_size; 49 | 50 | // Hard-coded item IDs 51 | static constexpr int bf_raw_id = 0x5000; 52 | static constexpr int timestamp_id = 0x1600; 53 | static constexpr int frequency_id = 0x4103; 54 | 55 | q::ticks first_timestamp{-1}; 56 | 57 | spead2::thread_pool worker; 58 | /// Index of the first custom statistic 59 | std::size_t counter_base; 60 | 61 | /// Create a single fully-allocated slice 62 | std::unique_ptr make_slice(); 63 | 64 | /// Create the stream configuration for the stream 65 | spead2::recv::stream_config make_stream_config(); 66 | 67 | /// Create the chunk configuration for the stream 68 | spead2::recv::chunk_stream_config make_chunk_stream_config(); 69 | 70 | /// Add the readers to the already-allocated stream 71 | void emplace_readers(); 72 | 73 | /** 74 | * Process a timestamp and channel number from a heap into more useful 75 | * indices. Note: this function modifies state by setting @ref 76 | * first_timestamp if this is the first (valid) call. If it is invalid, 77 | * a suitable error counter is incremented. 78 | * 79 | * @param timestamp ADC timestamp 80 | * @param channel Channel number of first channel in heap 81 | * @param[out] spectrum Index of first spectrum in heap, counting from 0 82 | * for first heap 83 | * @param[out] heap_offset Byte offset from start of slice data for this heap 84 | * @param[out] present_idx Position in @ref slice::present to record this heap 85 | * @param batch_stats Pointer to stream's statistics for updating 86 | * @param quiet If true, do not log or increment counters on bad heaps 87 | * 88 | * @retval true if @a timestamp and @a channel are valid 89 | * @retval false otherwise, and a message is logged 90 | */ 91 | bool parse_timestamp_channel( 92 | q::ticks timestamp, q::channels channel, 93 | q::spectra &spectrum, 94 | std::size_t &heap_offset, q::heaps &present_idx, 95 | std::uint64_t *batch_stats, 96 | bool quiet = false); 97 | 98 | /** 99 | * Copy contents of one packet to a slice. 100 | */ 101 | void packet_memcpy(const spead2::memory_allocator::pointer &allocated, 102 | const spead2::recv::packet_header &packet); 103 | 104 | /** 105 | * Finish processing on a slice before flushing it. 106 | * 107 | * This fills in any missing heaps with zeros, and populates the 108 | * slice-specific fields. 109 | * 110 | * If the slice is completely empty, however, it only populates n_present. 111 | */ 112 | void finish_slice(slice &s, std::uint64_t *counter_stats) const; 113 | 114 | /** 115 | * chunk_place_function for the underlying stream. 116 | */ 117 | void place(spead2::recv::chunk_place_data *data, std::size_t size); 118 | 119 | public: 120 | spead2::recv::chunk_ring_stream<> stream; 121 | 122 | /** 123 | * Retrieve first timestamp, or -1 if no data was received. 124 | * It is only valid to call this once the receiver has been stopped 125 | * or a non-empty slice has been received. 126 | */ 127 | q::ticks get_first_timestamp() const 128 | { 129 | return first_timestamp; 130 | } 131 | 132 | explicit receiver(const session_config &config); 133 | ~receiver(); 134 | 135 | /// Add a TCP socket receiver to a running receiver (for testing only!) 136 | void add_tcp_reader(const spead2::socket_wrapper &acceptor); 137 | 138 | /// Stop immediately, without flushing any slices 139 | void stop(); 140 | 141 | /// Asynchronously stop, allowing buffered slices to flush 142 | void graceful_stop(); 143 | }; 144 | 145 | #endif // RECEIVER_H 146 | -------------------------------------------------------------------------------- /katsdpbfingest/bf_ingest.cpp: -------------------------------------------------------------------------------- 1 | /* Backend implementation of beamformer ingest, written in C++ for efficiency. 2 | * 3 | * Even though the data rates are not that high in absolute terms, careful 4 | * design is needed. Simply setting the HDF5 chunk size to match the heap size 5 | * will not be sufficient, since the heaps are small enough that this is very 6 | * inefficient with O_DIRECT (and O_DIRECT is needed to keep performance 7 | * predictable enough). The high heap rate also makes a single ring buffer with 8 | * entry per heap unattractive. 9 | * 10 | * Instead, the network thread assembles heaps into "slices", which span the 11 | * entire band. Slices are then passed through a ring buffer to the disk 12 | * writer thread. At present, slices also match HDF5 chunks, although if 13 | * desirable they could be split into smaller chunks for more efficient reads 14 | * of subbands (this will, however, reduce write performance). 15 | * 16 | * libhdf5 doesn't support scatter-gather, so each slice needs to be collected 17 | * in contiguous memory. To avoid extra copies, a custom allocator is used to 18 | * provision space in the slice so that spead2 will fill in the payload 19 | * in-place. 20 | */ 21 | 22 | /* Still TODO: 23 | * - improve libhdf5 exception handling: 24 | * - put full backtrace into exception object 25 | * - debug segfault in exit handlers 26 | * - grow the file in batches, shrink again at end? 27 | * - make Python code more robust to the file being corrupt? 28 | */ 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "common.h" 38 | #include "session.h" 39 | 40 | /* Work around pybind11 not supporting experimental::optional 41 | * if exists. Needed for GCC 7.3 at least. 42 | */ 43 | #if !PYBIND11_HAS_EXP_OPTIONAL 44 | namespace pybind11 45 | { 46 | namespace detail 47 | { 48 | template struct type_caster> 49 | : public optional_caster> {}; 50 | } 51 | } 52 | #endif 53 | 54 | namespace py = pybind11; 55 | 56 | static std::unique_ptr spead2_logger; 57 | 58 | PYBIND11_MODULE(_bf_ingest, m) 59 | { 60 | using namespace pybind11::literals; 61 | m.doc() = "C++ backend of beamformer capture"; 62 | 63 | py::class_(m, "SessionConfig", "Configuration data for the backend") 64 | .def(py::init(), "filename"_a) 65 | .def_readwrite("filename", &session_config::filename) 66 | .def_readwrite("endpoints_str", &session_config::endpoints_str) 67 | .def_property("interface_address", &session_config::get_interface_address, &session_config::set_interface_address) 68 | .def_readwrite("max_packet", &session_config::max_packet) 69 | .def_readwrite("buffer_size", &session_config::buffer_size) 70 | .def_readwrite("live_heaps_per_substream", &session_config::live_heaps_per_substream) 71 | .def_readwrite("ring_slots", &session_config::ring_slots) 72 | .def_readwrite("ibv", &session_config::ibv) 73 | .def_readwrite("comp_vector", &session_config::comp_vector) 74 | .def_readwrite("network_affinity", &session_config::network_affinity) 75 | .def_readwrite("disk_affinity", &session_config::disk_affinity) 76 | .def_readwrite("direct", &session_config::direct) 77 | .def_readwrite("channels", &session_config::channels) 78 | .def_readwrite("stats_int_time", &session_config::stats_int_time) 79 | .def_readwrite("heaps_per_slice_time", &session_config::heaps_per_slice_time) 80 | .def_readwrite("ticks_between_spectra", &session_config::ticks_between_spectra) 81 | .def_readwrite("spectra_per_heap", &session_config::spectra_per_heap) 82 | .def_readwrite("channels_per_heap", &session_config::channels_per_heap) 83 | .def_readwrite("sync_time", &session_config::sync_time) 84 | .def_readwrite("bandwidth", &session_config::bandwidth) 85 | .def_readwrite("center_freq", &session_config::center_freq) 86 | .def_readwrite("scale_factor_timestamp", &session_config::scale_factor_timestamp) 87 | .def_readwrite("channel_offset", &session_config::channel_offset) 88 | .def("add_endpoint", &session_config::add_endpoint, "bind_host"_a, "port"_a) 89 | .def_property("stats_interface_address", &session_config::get_stats_interface_address, &session_config::set_stats_interface_address) 90 | .def("set_stats_endpoint", &session_config::set_stats_endpoint, "host"_a, "port"_a) 91 | .def("validate", &session_config::validate) 92 | ; 93 | // Declared module-local to prevent conflicts with spead2's registration 94 | py::class_(m, "ReceiverCounters", py::module_local(), 95 | "Heap counters for a live capture session") 96 | .def("__getitem__", [](spead2::recv::stream_stats &stats, const std::string &name) 97 | { 98 | return stats[name]; 99 | }) 100 | ; 101 | py::class_(m, "Session", "Capture session") 102 | .def(py::init(), "config"_a) 103 | .def("join", &session::join) 104 | .def("stop_stream", &session::stop_stream) 105 | .def_property_readonly("counters", &session::get_counters) 106 | .def_property_readonly("first_timestamp", &session::get_first_timestamp) 107 | .def("add_tcp_reader", &session::add_tcp_reader) 108 | ; 109 | 110 | py::object logging_module = py::module::import("logging"); 111 | py::object my_logger_obj = logging_module.attr("getLogger")("katsdpbfingest.bf_ingest"); 112 | set_logger(my_logger_obj); 113 | 114 | py::module atexit_mod = py::module::import("atexit"); 115 | atexit_mod.attr("register")(py::cpp_function(clear_logger)); 116 | 117 | spead2::register_logging(); 118 | spead2::register_atexit(); 119 | } 120 | -------------------------------------------------------------------------------- /katsdpbfingest/session.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "common.h" 11 | #include "session.h" 12 | #include "writer.h" 13 | #include "stats.h" 14 | 15 | // TODO: only used for gil_scoped_release. Would be nice to find a way to avoid 16 | // having this file depend on pybind11. 17 | namespace py = pybind11; 18 | 19 | session::session(const session_config &config) : 20 | config(config.validate()), 21 | recv(config), 22 | run_future(std::async(std::launch::async, &session::run, this)) 23 | { 24 | } 25 | 26 | session::~session() 27 | { 28 | py::gil_scoped_release gil; 29 | recv.stop(); 30 | if (run_future.valid()) 31 | run_future.wait(); // don't get(), since that could throw 32 | } 33 | 34 | void session::join() 35 | { 36 | py::gil_scoped_release gil; 37 | if (run_future.valid()) 38 | { 39 | run_future.wait(); 40 | // The run function should have done this, but if it exited by 41 | // exception then it won't. It's idempotent, so call it again 42 | // to be sure. 43 | recv.stop(); 44 | run_future.get(); // this can throw an exception 45 | } 46 | } 47 | 48 | void session::stop_stream() 49 | { 50 | py::gil_scoped_release gil; 51 | recv.graceful_stop(); 52 | } 53 | 54 | void session::run() 55 | { 56 | try 57 | { 58 | //H5::Exception::dontPrint(); 59 | run_impl(); 60 | } 61 | catch (H5::Exception &e) 62 | { 63 | throw std::runtime_error(e.getFuncName() + ": " + e.getDetailMsg()); 64 | } 65 | } 66 | 67 | void session::run_impl() 68 | { 69 | if (config.disk_affinity >= 0) 70 | spead2::thread_pool::set_affinity(config.disk_affinity); 71 | 72 | auto ring = recv.stream.get_data_ringbuffer(); 73 | 74 | const unit_system time_sys( 75 | 2 * sizeof(std::int8_t), config.spectra_per_heap, 76 | config.heaps_per_slice_time); 77 | const unit_system freq_sys( 78 | config.channels_per_heap, config.channels / config.channels_per_heap); 79 | 80 | std::unique_ptr stats; 81 | if (!config.stats_endpoint.address().is_unspecified()) 82 | { 83 | stats.reset(new stats_collector(config)); 84 | } 85 | 86 | std::unique_ptr w; 87 | int fd = -1; 88 | std::size_t reserve_blocks = 0; 89 | q::heaps n_total_heaps{0}; 90 | struct statfs stat; 91 | // Number of heaps in time between disk space checks 92 | constexpr q::heaps_t check_cadence{1000}; 93 | if (config.filename) 94 | { 95 | w.reset(new hdf5_writer(*config.filename, config.direct, 96 | config.get_freq_system(), config.get_time_system())); 97 | fd = w->get_fd(); 98 | if (fstatfs(fd, &stat) < 0) 99 | throw std::system_error(errno, std::system_category(), "fstatfs failed"); 100 | const q::bytes check_bytes = 101 | freq_sys.scale_factor() 102 | * time_sys.convert(check_cadence); 103 | reserve_blocks = (1024 * 1024 * 1024 + check_bytes.get()) / stat.f_bsize; 104 | log_format(spead2::log_level::info, "capture will stop when disk space is %d bytes", 105 | reserve_blocks * stat.f_bsize); 106 | } 107 | 108 | bool done = false; 109 | // When time_heaps passes this value, we check disk space and log a message 110 | q::heaps_t next_check = check_cadence; 111 | while (!done) 112 | { 113 | try 114 | { 115 | std::unique_ptr s(static_cast(ring->pop().release())); 116 | if (stats) 117 | stats->add(*s); 118 | if (w) 119 | w->add(*s); 120 | q::heaps_t time_heaps = time_sys.convert_down(s->spectrum) + q::heaps_t(1); 121 | q::heaps total_heaps = time_heaps * freq_sys.convert_one(); 122 | if (total_heaps > n_total_heaps) 123 | { 124 | n_total_heaps = total_heaps; 125 | if (time_heaps >= next_check) 126 | { 127 | if (w) 128 | { 129 | if (fstatfs(fd, &stat) < 0) 130 | throw std::system_error(errno, std::system_category(), "fstatfs failed"); 131 | if (stat.f_bavail < reserve_blocks) 132 | { 133 | log_format(spead2::log_level::info, "stopping capture due to lack of free space: have %d blocks of %d, need %d", 134 | stat.f_bavail, stat.f_bsize, reserve_blocks); 135 | done = true; 136 | } 137 | } 138 | // Find next multiple of check_cadence strictly greater 139 | // than time_heaps. 140 | next_check = (time_heaps / check_cadence + 1) * check_cadence; 141 | } 142 | } 143 | recv.stream.add_free_chunk(std::move(s)); 144 | } 145 | catch (spead2::ringbuffer_stopped &e) 146 | { 147 | done = true; 148 | } 149 | } 150 | recv.stop(); 151 | } 152 | 153 | spead2::recv::stream_stats session::get_counters() const 154 | { 155 | return recv.stream.get_stats(); 156 | } 157 | 158 | std::int64_t session::get_first_timestamp() const 159 | { 160 | if (run_future.valid()) 161 | throw std::runtime_error("cannot retrieve first_timestamp while running"); 162 | return recv.get_first_timestamp().get(); 163 | } 164 | 165 | void session::add_tcp_reader(const spead2::socket_wrapper &acceptor) 166 | { 167 | recv.add_tcp_reader(acceptor); 168 | } 169 | -------------------------------------------------------------------------------- /katsdpbfingest/utils.py: -------------------------------------------------------------------------------- 1 | """Miscellaneous ingest utilities""" 2 | 3 | import logging 4 | from typing import Iterator, Tuple 5 | 6 | import katsdptelstate 7 | 8 | 9 | _logger = logging.getLogger(__name__) 10 | 11 | 12 | def cbf_telstate_view(telstate: katsdptelstate.TelescopeState, 13 | stream_name: str) -> katsdptelstate.TelescopeState: 14 | """Create a telstate view that allows querying properties from a stream. 15 | It supports only baseline-correlation-products and 16 | tied-array-channelised-voltage streams. Properties that don't exist on the 17 | stream are searched on the upstream antenna-channelised-voltage stream, and 18 | then the instrument of that stream. 19 | 20 | Returns 21 | ------- 22 | view 23 | Telstate view that allows stream properties to be searched 24 | """ 25 | prefixes = [] 26 | stream_name = stream_name.replace('.', '_').replace('-', '_') 27 | prefixes.append(stream_name) 28 | # Generate a list of places to look for attributes: 29 | # - the stream itself 30 | # - the upstream antenna-channelised-voltage stream, and its instrument 31 | src = telstate.view(stream_name, exclusive=True)['src_streams'][0] 32 | prefixes.append(src) 33 | instrument = telstate.view(src, exclusive=True)['instrument_dev_name'] 34 | prefixes.append(instrument) 35 | # Create a telstate view that has exactly the given prefixes (and no root prefix). 36 | for i, prefix in enumerate(reversed(prefixes)): 37 | telstate = telstate.view(prefix, exclusive=(i == 0)) 38 | return telstate 39 | 40 | 41 | class Range: 42 | """Representation of a range of values, as specified by a first and a 43 | past-the-end value. This can be seen as an extended form of `xrange` or 44 | `slice` (although without support for a non-unit step), where it is easy to 45 | query the start and stop values, along with other convenience methods. 46 | 47 | Ranges can be empty, in which case they still have a `start` and `stop` 48 | value that are equal, but the value itself is irrelevant. 49 | """ 50 | def __init__(self, start: int, stop: int) -> None: 51 | if start > stop: 52 | raise ValueError('start must be <= stop') 53 | self.start = start 54 | self.stop = stop 55 | 56 | @classmethod 57 | def parse(cls, value: str) -> 'Range': 58 | """Convert a string of the form 'A:B' to a :class:`~katsdpingest.utils.Range`, 59 | where A and B are integers. 60 | 61 | This is suitable as an argparse type converter. 62 | """ 63 | fields = value.split(':', 1) 64 | if len(fields) != 2: 65 | raise ValueError('Invalid range format {}'.format(value)) 66 | else: 67 | return Range(int(fields[0]), int(fields[1])) 68 | 69 | def __str__(self) -> str: 70 | return '{}:{}'.format(self.start, self.stop) 71 | 72 | def __repr__(self) -> str: 73 | return 'Range({}, {})'.format(self.start, self.stop) 74 | 75 | def __len__(self) -> int: 76 | return self.stop - self.start 77 | 78 | def __contains__(self, value: int) -> bool: 79 | return self.start <= value < self.stop 80 | 81 | def __eq__(self, other): 82 | if not isinstance(other, Range): 83 | return NotImplemented 84 | if not self: 85 | return not other 86 | else: 87 | return self.start == other.start and self.stop == other.stop 88 | 89 | def __ne__(self, other): 90 | if not isinstance(other, Range): 91 | return NotImplemented 92 | else: 93 | return not (self == other) 94 | 95 | # Can't prevent object from being mutated, but __eq__ is defined, so not 96 | # suitable for hashing. 97 | __hash__ = None # type: ignore # workaround for python/mypy#4266 98 | 99 | def issubset(self, other: 'Range') -> bool: 100 | return self.start == self.stop or (other.start <= self.start and self.stop <= other.stop) 101 | 102 | def issuperset(self, other: 'Range') -> bool: 103 | return other.issubset(self) 104 | 105 | def isaligned(self, alignment: int) -> bool: 106 | """Whether the start and end of this interval are aligned to multiples 107 | of `alignment`. 108 | """ 109 | return not self or (self.start % alignment == 0 and self.stop % alignment == 0) 110 | 111 | def alignto(self, alignment: int) -> 'Range': 112 | """Return the smallest range containing self for which 113 | ``r.isaligned()`` is true. 114 | """ 115 | if not self: 116 | return self 117 | else: 118 | return Range(self.start // alignment * alignment, 119 | (self.stop + alignment - 1) // alignment * alignment) 120 | 121 | def __floordiv__(self, alignment: int) -> 'Range': 122 | """Divide start and end by `alignment`. 123 | 124 | If they are not multiples, it is as if :meth:`alignto` was used first. 125 | """ 126 | if not self: 127 | return self 128 | else: 129 | return Range(self.start // alignment, 130 | (self.stop + alignment - 1) // alignment) 131 | 132 | def intersection(self, other: 'Range') -> 'Range': 133 | start = max(self.start, other.start) 134 | stop = min(self.stop, other.stop) 135 | if start > stop: 136 | return Range(0, 0) 137 | else: 138 | return Range(start, stop) 139 | 140 | def union(self, other: 'Range') -> 'Range': 141 | """Return the smallest range containing both ranges.""" 142 | if not self: 143 | return other 144 | if not other: 145 | return self 146 | return Range(min(self.start, other.start), max(self.stop, other.stop)) 147 | 148 | def __iter__(self) -> Iterator[int]: 149 | return iter(range(self.start, self.stop)) 150 | 151 | def relative_to(self, other: 'Range') -> 'Range': 152 | """Return a new range that represents `self` as a range relative to 153 | `other` (i.e. where the start element of `other` is numbered 0). If 154 | `self` is an empty range, an undefined empty range is returned. 155 | 156 | Raises 157 | ------ 158 | ValueError 159 | if `self` is not a subset of `other` 160 | """ 161 | if not self.issubset(other): 162 | raise ValueError('self is not a subset of other') 163 | return Range(self.start - other.start, self.stop - other.start) 164 | 165 | def asslice(self) -> slice: 166 | """Return a slice object representing the same range""" 167 | return slice(self.start, self.stop) 168 | 169 | def astuple(self) -> Tuple[int, int]: 170 | """Return a tuple containing the start and end values""" 171 | return (self.start, self.stop) 172 | 173 | def split(self, chunks: int, chunk_id: int) -> 'Range': 174 | """Return the `chunk_id`-th of `chunks` equally-sized pieces. 175 | 176 | Raises 177 | ------ 178 | ValueError 179 | if chunk_id is not in the range [0, chunks) or the range does not 180 | divide evenly. 181 | """ 182 | if not 0 <= chunk_id < chunks: 183 | raise ValueError('chunk_id is out of range') 184 | if len(self) % chunks != 0: 185 | raise ValueError('range {} does not divide into {} chunks'.format(self, chunks)) 186 | chunk_size = len(self) // chunks 187 | return Range(self.start + chunk_id * chunk_size, 188 | self.start + (chunk_id + 1) * chunk_size) 189 | 190 | 191 | __all__ = ['cbf_telstate_view', 'Range'] 192 | -------------------------------------------------------------------------------- /katsdpbfingest/common.h: -------------------------------------------------------------------------------- 1 | #ifndef COMMON_H 2 | #define COMMON_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "units.h" 18 | 19 | // Forward-declare to avoid sucking in pybind11.h 20 | namespace pybind11 { class object; } 21 | 22 | static constexpr std::uint8_t data_lost = 1 << 3; 23 | 24 | void log_message(spead2::log_level level, const std::string &msg); 25 | void set_logger(pybind11::object logger); 26 | void clear_logger(); 27 | 28 | /** 29 | * Recursively push a variadic list of arguments into a @c boost::format. This 30 | * is copied from the spead2 codebase. 31 | */ 32 | static inline void apply_format(boost::format &formatter) 33 | { 34 | } 35 | 36 | template 37 | static void apply_format(boost::format &formatter, T0 &&arg0, Ts&&... args) 38 | { 39 | formatter % std::forward(arg0); 40 | apply_format(formatter, std::forward(args)...); 41 | } 42 | 43 | template 44 | static void log_format(spead2::log_level level, const std::string &format, Ts&&... args) 45 | { 46 | boost::format formatter(format); 47 | apply_format(formatter, args...); 48 | log_message(level, formatter.str()); 49 | } 50 | 51 | template 52 | struct free_delete 53 | { 54 | void operator()(T *ptr) const 55 | { 56 | free(ptr); 57 | } 58 | }; 59 | 60 | template 61 | using aligned_ptr = std::unique_ptr>; 62 | 63 | static constexpr int ALIGNMENT = 4096; // For O_DIRECT file access 64 | 65 | /** 66 | * Allocate memory that is aligned to a multiple of @c ALIGNMENT. This is used 67 | * with O_DIRECT. 68 | */ 69 | template 70 | static aligned_ptr make_aligned(std::size_t elements) 71 | { 72 | void *ptr = aligned_alloc(ALIGNMENT, elements * sizeof(T)); 73 | if (!ptr) 74 | throw std::bad_alloc(); 75 | return std::unique_ptr>(static_cast(ptr)); 76 | } 77 | 78 | namespace units 79 | { 80 | template 81 | struct time_unit 82 | { 83 | typedef U base; 84 | static const char *name() { return U::name(); } 85 | }; 86 | 87 | template 88 | struct freq_unit 89 | { 90 | typedef U base; 91 | static const char *name() { return U::name(); } 92 | }; 93 | 94 | template 95 | struct twod // base class for units with time and frequency axes 96 | { 97 | typedef time_unit time; 98 | typedef freq_unit freq; 99 | }; 100 | 101 | // _t suffix means time axis, _f suffix means frequency axis 102 | struct heaps : public twod { static const char *name() { return "heaps"; } }; 103 | struct slices : public twod { static const char *name() { return "slices"; } }; 104 | struct chunks : public twod { static const char *name() { return "chunks"; } }; 105 | struct spectra { static const char *name() { return "spectra"; } }; 106 | struct channels { static const char *name() { return "channels"; } }; 107 | struct ticks { static const char *name() { return "ticks"; } }; 108 | struct samples { static const char *name() { return "samples"; } }; 109 | struct bytes { static const char *name() { return "bytes"; } }; 110 | 111 | typedef unit_system time_system; 112 | typedef unit_system freq_system; 113 | } 114 | 115 | // Some shortcuts for quantities of each unit 116 | namespace q 117 | { 118 | typedef quantity heaps_t; 119 | typedef quantity heaps_f; 120 | typedef quantity heaps; 121 | typedef quantity slices_t; 122 | typedef quantity slices_f; 123 | typedef quantity slices; 124 | typedef quantity chunks_t; 125 | typedef quantity chunks_f; 126 | typedef quantity chunks; 127 | typedef quantity spectra; 128 | typedef quantity channels; 129 | typedef quantity ticks; 130 | typedef quantity samples; 131 | typedef quantity bytes; 132 | } 133 | 134 | template 135 | struct unit_product, units::freq_unit> 136 | { 137 | typedef Base type; 138 | }; 139 | 140 | template 141 | struct unit_product, units::time_unit> 142 | { 143 | typedef Base type; 144 | }; 145 | 146 | template<> struct unit_product { typedef units::samples type; }; 147 | template<> struct unit_product { typedef units::samples type; }; 148 | 149 | /** 150 | * Return a vector that can be passed to a @c spead2::thread_pool constructor 151 | * to bind to core @a affinity, or to no core if it is negative. 152 | */ 153 | std::vector affinity_vector(int affinity); 154 | 155 | /** 156 | * Storage for a collection of heaps. 157 | * 158 | * A slice is a rectangle in the time-frequency space. It currently covers the 159 | * whole of the frequency axis, and some fixed number of heaps in the time 160 | * axis. It corresponds to the HDF5 chunk size for the data, so needs to be 161 | * large enough to be spread across the stripes in a RAID. 162 | */ 163 | class slice : public spead2::recv::chunk 164 | { 165 | public: 166 | q::ticks timestamp{-1}; ///< Timestamp (of the first heap) 167 | q::spectra spectrum{-1}; ///< Number of spectra since start (for first heap) 168 | q::heaps n_present{0}; ///< Number of 1 bits in @a present 169 | 170 | /// Number of bytes for n samples of data 171 | static std::size_t bytes(q::samples n); 172 | }; 173 | 174 | /** 175 | * Generic class (using Curiously Recursive Template Pattern) for managing an 176 | * (unbounded) sequence of items, each of which is built up from smaller 177 | * items, which do not necessarily arrive strictly in order. This class 178 | * manages a fixed-size window of slots. The derived class can ask for access 179 | * to a specific element of the sequence, and if necessary the window is 180 | * advanced, flushing trailing items. The window is never retracted; if the 181 | * caller wants to access an item that is too old, it is refused. 182 | * 183 | * The subclass must provide a @c flush method to process an item from the 184 | * trailing edge and then reset its state. Note that @c flush may be called on 185 | * items that have never been accessed with @ref get, so subclasses must 186 | * handle this. 187 | */ 188 | template 189 | class window 190 | { 191 | public: 192 | typedef T value_type; 193 | 194 | private: 195 | /** 196 | * The window itself. Item @c i is stored in index 197 | * i % window_size, provided that 198 | * start <= i < start + window_size. 199 | */ 200 | std::vector slots; 201 | std::size_t oldest = 0; ///< Index into @ref slots of oldest item 202 | std::int64_t start = 0; ///< ID of oldest item in window 203 | 204 | void flush_oldest(); 205 | 206 | public: 207 | // The args are passed to the constructor for T to construct the slots 208 | template 209 | explicit window(std::size_t window_size, const Args&... args); 210 | 211 | /// Returns pointer to the slot for @a id, or @c nullptr if the window has moved on 212 | T *get(std::int64_t id); 213 | 214 | /// Flush the next @a window_size items (even if they have never been requested) 215 | void flush_all(); 216 | }; 217 | 218 | template 219 | template 220 | window::window(std::size_t window_size, const Args&... args) 221 | { 222 | slots.reserve(window_size); 223 | for (std::size_t i = 0; i < window_size; i++) 224 | slots.emplace_back(args...); 225 | } 226 | 227 | template 228 | void window::flush_oldest() 229 | { 230 | T &item = slots[oldest]; 231 | static_cast(this)->flush(item); 232 | oldest++; 233 | if (oldest == slots.size()) 234 | oldest = 0; 235 | start++; 236 | } 237 | 238 | template 239 | T *window::get(std::int64_t id) 240 | { 241 | if (id < start) 242 | return nullptr; 243 | // TODO: fast-forward mechanism for big differences? 244 | while (id - start >= std::int64_t(slots.size())) 245 | flush_oldest(); 246 | std::size_t pos = id % slots.size(); 247 | return &slots[pos]; 248 | } 249 | 250 | template 251 | void window::flush_all() 252 | { 253 | for (std::size_t i = 0; i < slots.size(); i++) 254 | flush_oldest(); 255 | } 256 | 257 | struct session_config 258 | { 259 | std::experimental::optional filename; 260 | std::vector endpoints; 261 | std::string endpoints_str; ///< Human-readable version of endpoints 262 | boost::asio::ip::address interface_address; 263 | 264 | #if SPEAD2_USE_IBV 265 | std::size_t max_packet = spead2::recv::udp_ibv_config::default_max_size; 266 | #else 267 | std::size_t max_packet = spead2::recv::udp_reader::default_max_size; 268 | #endif 269 | std::size_t buffer_size = 64 * 1024 * 1024; 270 | int live_heaps_per_substream = 2; 271 | int ring_slots = 2; // Caller should increase it 272 | bool ibv = false; 273 | int comp_vector = 0; 274 | int network_affinity = -1; 275 | 276 | int disk_affinity = -1; 277 | bool direct = false; 278 | 279 | // First channel 280 | int channel_offset = 0; 281 | // Number of channels, counting from channel_offset 282 | int channels = -1; 283 | // Time (in seconds) over which to accumulate stats 284 | double stats_int_time = 1.0; 285 | // Number of heaps accumulated into an HDF5 chunk 286 | int heaps_per_slice_time = -1; 287 | 288 | // Metadata derived from telescope state. 289 | std::int64_t ticks_between_spectra = -1; 290 | int spectra_per_heap = -1; 291 | int channels_per_heap = -1; 292 | double sync_time = -1.0; 293 | double bandwidth = -1.0; 294 | double center_freq = -1.0; 295 | double scale_factor_timestamp = -1.0; 296 | 297 | boost::asio::ip::udp::endpoint stats_endpoint; 298 | boost::asio::ip::address stats_interface_address; 299 | 300 | explicit session_config(const std::string &filename); 301 | void add_endpoint(const std::string &bind_host, std::uint16_t port); 302 | std::string get_interface_address() const; 303 | void set_interface_address(const std::string &address); 304 | 305 | void set_stats_endpoint(const std::string &host, std::uint16_t port); 306 | std::string get_stats_interface_address() const; 307 | void set_stats_interface_address(const std::string &address); 308 | 309 | units::freq_system get_freq_system() const; 310 | units::time_system get_time_system() const; 311 | 312 | // Check that all required items have been set and return self. 313 | // Throws invalid_value if not. 314 | const session_config &validate() const; 315 | }; 316 | 317 | #endif // COMMON_H 318 | -------------------------------------------------------------------------------- /katsdpbfingest/writer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "common.h" 7 | #include "writer.h" 8 | 9 | // See write_direct below for explanation 10 | #if !H5_VERSION_GE(1, 10, 3) 11 | #include 12 | #endif 13 | 14 | static void write_direct( 15 | H5::DataSet &dataset, const hsize_t *offset, std::size_t data_size, const void *buf) 16 | { 17 | #if H5_VERSION_GE(1, 10, 3) 18 | // 1.10.3 moved this functionality from H5DO into the core 19 | herr_t ret = H5Dwrite_chunk(dataset.getId(), H5P_DEFAULT, 0, offset, data_size, buf); 20 | #else 21 | herr_t ret = H5DOwrite_chunk(dataset.getId(), H5P_DEFAULT, 0, offset, data_size, buf); 22 | #endif 23 | if (ret < 0) 24 | throw H5::DataSetIException("DataSet::write_chunk", "H5Dwrite_chunk failed"); 25 | } 26 | 27 | hdf5_bf_raw_writer::hdf5_bf_raw_writer( 28 | H5::Group &parent, 29 | const units::freq_system &freq_sys, 30 | const units::time_system &time_sys, 31 | const char *name) 32 | : freq_sys(freq_sys), time_sys(time_sys), 33 | chunk_bytes(slice::bytes(time_sys.convert_one() 34 | * freq_sys.convert_one())) 35 | { 36 | hsize_t chunk[3] = { 37 | hsize_t(freq_sys.scale_factor()), 38 | hsize_t(time_sys.scale_factor()), 39 | 2 40 | }; 41 | hsize_t dims[3] = {chunk[0], 0, chunk[2]}; 42 | hsize_t maxdims[3] = {chunk[0], H5S_UNLIMITED, chunk[2]}; 43 | H5::DataSpace file_space(3, dims, maxdims); 44 | H5::DSetCreatPropList dcpl; 45 | dcpl.setChunk(3, chunk); 46 | std::int8_t fill = 0; 47 | dcpl.setFillValue(H5::PredType::NATIVE_INT8, &fill); 48 | dataset = parent.createDataSet(name, H5::PredType::STD_I8BE, file_space, dcpl); 49 | } 50 | 51 | void hdf5_bf_raw_writer::add(const slice &s) 52 | { 53 | q::spectra end = s.spectrum + time_sys.convert_one(); 54 | q::channels channels = freq_sys.convert_one(); 55 | hsize_t new_size[3] = {hsize_t(channels.get()), hsize_t(end.get()), 2}; 56 | dataset.extend(new_size); 57 | const hsize_t offset[3] = {0, hsize_t(s.spectrum.get()), 0}; 58 | write_direct(dataset, offset, chunk_bytes, s.data.get()); 59 | } 60 | 61 | constexpr hsize_t hdf5_timestamps_writer::chunk; 62 | 63 | static void set_string_attribute(H5::H5Object &location, const std::string &name, const std::string &value) 64 | { 65 | H5::DataSpace scalar; 66 | H5::StrType type(H5::PredType::C_S1, value.size()); 67 | H5::Attribute attribute = location.createAttribute(name, type, scalar); 68 | attribute.write(type, value); 69 | } 70 | 71 | hdf5_timestamps_writer::hdf5_timestamps_writer( 72 | H5::Group &parent, const units::time_system &time_sys, 73 | const char *name) 74 | : time_sys(time_sys) 75 | { 76 | hsize_t dims[1] = {0}; 77 | hsize_t maxdims[1] = {H5S_UNLIMITED}; 78 | H5::DataSpace file_space(1, dims, maxdims); 79 | H5::DSetCreatPropList dcpl; 80 | dcpl.setChunk(1, &chunk); 81 | std::uint64_t fill = 0; 82 | dcpl.setFillValue(H5::PredType::NATIVE_UINT64, &fill); 83 | dataset = parent.createDataSet( 84 | name, H5::PredType::NATIVE_UINT64, file_space, dcpl); 85 | buffer = make_aligned(chunk); 86 | n_buffer = 0; 87 | set_string_attribute(dataset, "timestamp_reference", "start"); 88 | set_string_attribute(dataset, "timestamp_type", "adc"); 89 | } 90 | 91 | hdf5_timestamps_writer::~hdf5_timestamps_writer() 92 | { 93 | if (!std::uncaught_exception() && n_buffer > 0) 94 | flush(); 95 | } 96 | 97 | void hdf5_timestamps_writer::flush() 98 | { 99 | hsize_t new_size = n_written + n_buffer; 100 | dataset.extend(&new_size); 101 | const hsize_t offset[1] = {n_written}; 102 | if (n_buffer < chunk) 103 | { 104 | // Pad extra space with zeros - shouldn't matter, but this case 105 | // only arises when closing the file so should be cheap. 106 | std::memset(buffer.get() + n_buffer, 0, (chunk - n_buffer) * sizeof(std::uint64_t)); 107 | } 108 | write_direct(dataset, offset, chunk * sizeof(std::uint64_t), buffer.get()); 109 | n_written += n_buffer; 110 | n_buffer = 0; 111 | } 112 | 113 | void hdf5_timestamps_writer::add(q::ticks timestamp) 114 | { 115 | q::spectra heap_spectra = time_sys.convert_one(); 116 | for (q::spectra i{0}; i < heap_spectra; ++i) 117 | buffer[n_buffer++] = (timestamp + time_sys.convert(i)).get(); 118 | assert(n_buffer <= chunk); 119 | if (n_buffer == chunk) 120 | flush(); 121 | } 122 | 123 | flags_chunk::flags_chunk(q::heaps size) 124 | : data(make_aligned(size.get())) 125 | { 126 | std::memset(data.get(), data_lost, size.get() * sizeof(std::uint8_t)); 127 | } 128 | 129 | q::heaps hdf5_flags_writer::heaps_per_slice(const units::freq_system &freq_sys, 130 | const units::time_system &time_sys) 131 | { 132 | return time_sys.convert_one() 133 | * freq_sys.convert_one(); 134 | } 135 | 136 | q::slices hdf5_flags_writer::compute_chunk_size_slices(const units::freq_system &freq_sys, 137 | const units::time_system &time_sys) 138 | { 139 | std::size_t n = bytes(heaps_per_slice(freq_sys, time_sys)); 140 | // Make each chunk about 4MiB, rounding up if needed 141 | std::size_t slices = (4 * 1024 * 1024 + n - 1) / n; 142 | return q::slices(slices); 143 | } 144 | 145 | q::heaps hdf5_flags_writer::compute_chunk_size_heaps(const units::freq_system &freq_sys, 146 | const units::time_system &time_sys) 147 | { 148 | return compute_chunk_size_slices(freq_sys, time_sys).get() 149 | * heaps_per_slice(freq_sys, time_sys); 150 | } 151 | 152 | std::size_t hdf5_flags_writer::bytes(q::heaps n) 153 | { 154 | return sizeof(std::uint8_t) * n.get(); 155 | } 156 | 157 | hdf5_flags_writer::hdf5_flags_writer( 158 | H5::Group &parent, 159 | const units::freq_system &freq_sys, const units::time_system &time_sys, 160 | const char *name) 161 | : window( 162 | 1, compute_chunk_size_heaps(freq_sys, time_sys)), 163 | freq_sys(freq_sys.append(1)), 164 | time_sys(time_sys.append(compute_chunk_size_slices(freq_sys, time_sys).get())), 165 | chunk_bytes(bytes( 166 | this->freq_sys.convert_one() 167 | * this->time_sys.convert_one())) 168 | { 169 | hsize_t dims[2] = { 170 | hsize_t(this->freq_sys.scale_factor()), 171 | 0 172 | }; 173 | hsize_t maxdims[2] = {dims[0], H5S_UNLIMITED}; 174 | hsize_t chunk[2] = { 175 | hsize_t(this->freq_sys.scale_factor()), 176 | hsize_t(this->time_sys.scale_factor()) 177 | }; 178 | H5::DataSpace file_space(2, dims, maxdims); 179 | H5::DSetCreatPropList dcpl; 180 | dcpl.setChunk(2, chunk); 181 | dcpl.setFillValue(H5::PredType::NATIVE_UINT8, &data_lost); 182 | dataset = parent.createDataSet(name, H5::PredType::STD_U8BE, file_space, dcpl); 183 | } 184 | 185 | hdf5_flags_writer::~hdf5_flags_writer() 186 | { 187 | if (!std::uncaught_exception()) 188 | flush_all(); 189 | } 190 | 191 | void hdf5_flags_writer::flush(flags_chunk &chunk) 192 | { 193 | if (chunk.spectrum != q::spectra(-1)) 194 | { 195 | hsize_t new_size[2] = { 196 | hsize_t(freq_sys.scale_factor()), 197 | hsize_t(time_sys.convert(n_slices).get()) 198 | }; 199 | dataset.extend(new_size); 200 | const hsize_t offset[2] = { 201 | 0, 202 | hsize_t(time_sys.convert_down(chunk.spectrum).get()) 203 | }; 204 | write_direct(dataset, offset, chunk_bytes, chunk.data.get()); 205 | } 206 | chunk.spectrum = q::spectra(-1); 207 | std::memset(chunk.data.get(), data_lost, chunk_bytes); 208 | } 209 | 210 | void hdf5_flags_writer::add(const slice &s) 211 | { 212 | q::slices_t slice_id = time_sys.convert_down(s.spectrum); 213 | q::chunks_t id = time_sys.convert_down(slice_id); 214 | flags_chunk *chunk = get(id.get()); 215 | assert(chunk != nullptr); // we are given slices in-order, so cannot be behind the window 216 | // Note: code below doesn't yet allow for slice != chunk on frequency axis 217 | q::heaps_t offset = time_sys.convert(slice_id) 218 | - time_sys.convert(id); 219 | q::heaps_f slice_heaps_f = freq_sys.convert_one(); 220 | q::heaps_t slice_heaps_t = time_sys.convert_one(); 221 | q::heaps_t stride = time_sys.convert_one(); 222 | std::size_t present_idx = 0; 223 | for (q::heaps_f f{0}; f < slice_heaps_f; ++f) 224 | for (q::heaps_t t{0}; t < slice_heaps_t; ++t, present_idx++) 225 | { 226 | q::heaps pos = f * stride + (t + offset) * q::heaps_f(1); 227 | chunk->data[pos.get()] = s.present[present_idx] ? 0 : data_lost; 228 | } 229 | chunk->spectrum = time_sys.convert(id); 230 | n_slices = slice_id + q::slices_t(1); 231 | } 232 | 233 | hdf5_writer::hdf5_writer(const std::string &filename, bool direct, 234 | const units::freq_system &freq_sys, 235 | const units::time_system &time_sys) 236 | : file(filename, H5F_ACC_TRUNC, H5::FileCreatPropList::DEFAULT, make_fapl(direct)), 237 | group(file.createGroup("Data")), 238 | freq_sys(freq_sys), time_sys(time_sys), 239 | bf_raw(group, freq_sys, time_sys, "bf_raw"), 240 | timestamps(group, time_sys, "timestamps"), 241 | flags(group, freq_sys, time_sys, "flags") 242 | { 243 | H5::DataSpace scalar; 244 | // 1.8.11 doesn't have the right C++ wrapper for this to work, so we 245 | // duplicate its work 246 | hid_t attr_id = H5Acreate2( 247 | file.getId(), "version", H5::PredType::NATIVE_INT32.getId(), 248 | scalar.getId(), H5P_DEFAULT, H5P_DEFAULT); 249 | if (attr_id < 0) 250 | throw H5::AttributeIException("createAttribute", "H5Acreate2 failed"); 251 | H5::Attribute version_attr(attr_id); 252 | /* Release the ref created by H5Acreate2 (version_attr has its own). 253 | * HDF5 1.8.11 has a bug where version_attr doesn't get its own 254 | * reference, so to handle both cases we have to check the current 255 | * value. 256 | */ 257 | if (version_attr.getCounter() > 1) 258 | version_attr.decRefCount(); 259 | const std::int32_t version = 4; 260 | version_attr.write(H5::PredType::NATIVE_INT32, &version); 261 | } 262 | 263 | H5::FileAccPropList hdf5_writer::make_fapl(bool direct) 264 | { 265 | H5::FileAccPropList fapl; 266 | if (direct) 267 | { 268 | #ifdef H5_HAVE_DIRECT 269 | if (H5Pset_fapl_direct(fapl.getId(), ALIGNMENT, ALIGNMENT, 128 * 1024) < 0) 270 | throw H5::PropListIException("hdf5_writer::make_fapl", "H5Pset_fapl_direct failed"); 271 | #else 272 | throw std::runtime_error("H5_HAVE_DIRECT not defined"); 273 | #endif 274 | } 275 | else 276 | { 277 | fapl.setSec2(); 278 | } 279 | // Older versions of libhdf5 are missing the C++ version of setLibverBounds 280 | #ifdef H5F_LIBVER_110 281 | const auto version = H5F_LIBVER_110; 282 | #else 283 | const auto version = H5F_LIBVER_LATEST; 284 | #endif 285 | if (H5Pset_libver_bounds(fapl.getId(), version, version) < 0) 286 | throw H5::PropListIException("FileAccPropList::setLibverBounds", "H5Pset_libver_bounds failed"); 287 | fapl.setAlignment(ALIGNMENT, ALIGNMENT); 288 | fapl.setFcloseDegree(H5F_CLOSE_SEMI); 289 | return fapl; 290 | } 291 | 292 | void hdf5_writer::add(const slice &s) 293 | { 294 | q::ticks timestamp{s.timestamp}; 295 | if (past_end_timestamp == q::ticks(-1)) 296 | past_end_timestamp = timestamp; 297 | q::ticks next_timestamp = timestamp + time_sys.convert_one(); 298 | q::ticks step = time_sys.convert_one(); 299 | while (past_end_timestamp < next_timestamp) 300 | { 301 | timestamps.add(past_end_timestamp); 302 | past_end_timestamp += step; 303 | } 304 | bf_raw.add(s); 305 | flags.add(s); 306 | } 307 | 308 | int hdf5_writer::get_fd() const 309 | { 310 | void *fd_ptr; 311 | file.getVFDHandle(&fd_ptr); 312 | return *reinterpret_cast(fd_ptr); 313 | } 314 | -------------------------------------------------------------------------------- /katsdpbfingest/stats.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "common.h" 14 | #include "stats.h" 15 | 16 | #define TARGET "default" 17 | #include "stats_ops.h" 18 | #undef TARGET 19 | 20 | #define TARGET "avx" 21 | #include "stats_ops.h" 22 | #undef TARGET 23 | 24 | #define TARGET "avx2" 25 | #include "stats_ops.h" 26 | #undef TARGET 27 | 28 | // Taken from https://docs.google.com/spreadsheets/d/1XojAI9O9pSSXN8vyb2T97Sd875YCWqie8NY8L02gA_I/edit#gid=0 29 | static constexpr int id_n_bls = 0x1008; 30 | static constexpr int id_n_chans = 0x1009; 31 | static constexpr int id_center_freq = 0x1011; 32 | static constexpr int id_bandwidth = 0x1013; 33 | static constexpr int id_bls_ordering = 0x100C; 34 | static constexpr int id_sd_data = 0x3501; 35 | static constexpr int id_sd_timestamp = 0x3502; 36 | static constexpr int id_sd_flags = 0x3503; 37 | static constexpr int id_sd_timeseries = 0x3504; 38 | static constexpr int id_sd_percspectrum = 0x3505; 39 | static constexpr int id_sd_percspectrumflags = 0x3506; 40 | static constexpr int id_sd_blmxdata = 0x3507; 41 | static constexpr int id_sd_blmxflags = 0x3508; 42 | static constexpr int id_sd_data_index = 0x3509; 43 | static constexpr int id_sd_blmx_n_chans = 0x350A; 44 | static constexpr int id_sd_flag_fraction = 0x350B; 45 | static constexpr int id_sd_timeseriesabs = 0x3510; 46 | 47 | /// Make SPEAD 64-48 flavour 48 | static spead2::flavour make_flavour() 49 | { 50 | return spead2::flavour(4, 64, 48); 51 | } 52 | 53 | /** 54 | * Helper to add a descriptor to a heap with numpy-style descriptor. 55 | * 56 | * The @a dtype should not have an endianness indicator. It will be added by 57 | * this function. 58 | */ 59 | static void add_descriptor(spead2::send::heap &heap, 60 | spead2::s_item_pointer_t id, 61 | const std::string &name, const std::string &description, 62 | const std::vector &shape, 63 | const std::string &dtype) 64 | { 65 | spead2::descriptor d; 66 | d.id = id; 67 | d.name = name; 68 | d.description = description; 69 | std::ostringstream numpy_header; 70 | numpy_header << "{'shape': ("; 71 | for (auto s : shape) 72 | { 73 | assert(s >= 0); 74 | numpy_header << s << ", "; 75 | } 76 | char endian_char = spead2::htobe(std::uint16_t(0x1234)) == 0x1234 ? '>' : '<'; 77 | numpy_header << "), 'fortran_order': False, 'descr': '" << endian_char << dtype << "'}"; 78 | d.numpy_header = numpy_header.str(); 79 | heap.add_descriptor(d); 80 | } 81 | 82 | /** 83 | * Add a value to a heap, copying the data. 84 | * 85 | * The heap takes ownership of the copied data, so it is not necessary for @a 86 | * value to remain live after the call. 87 | */ 88 | template::value>::type> 90 | static void add_constant(spead2::send::heap &heap, spead2::s_item_pointer_t id, const T &value) 91 | { 92 | std::unique_ptr dup(new std::uint8_t[sizeof(T)]); 93 | std::memcpy(dup.get(), &value, sizeof(T)); 94 | heap.add_item(id, dup.get(), sizeof(T), true); 95 | heap.add_pointer(std::move(dup)); // Give the heap ownership of the memory 96 | } 97 | 98 | /** 99 | * Add a value to a heap, copying the data. 100 | * 101 | * This overload takes the value as a string. 102 | */ 103 | static void add_constant(spead2::send::heap &heap, spead2::s_item_pointer_t id, 104 | const std::string &value) 105 | { 106 | /* This code is written to handle arbitrary containers, but the function 107 | * isn't templated for them because the required SFINAE checks to ensure 108 | * that it is safe become very messy. 109 | */ 110 | auto first = std::begin(value); 111 | auto last = std::end(value); 112 | auto n = last - first; 113 | using T = std::iterator_traits::value_type; 114 | std::unique_ptr dup(new std::uint8_t[sizeof(T) * n]); 115 | std::uninitialized_copy(first, last, reinterpret_cast(dup.get())); 116 | heap.add_item(id, dup.get(), sizeof(T) * n, false); 117 | heap.add_pointer(std::move(dup)); 118 | } 119 | 120 | /** 121 | * Helper to call @ref add_descriptor and add a variable vector item. 122 | */ 123 | template::value>::type> 125 | static void add_vector(spead2::send::heap &heap, spead2::s_item_pointer_t id, 126 | const std::string &name, const std::string &description, 127 | const std::vector &shape, 128 | const std::string &dtype, 129 | std::vector &data) 130 | { 131 | add_descriptor(heap, id, name, description, shape, dtype); 132 | #ifndef NDEBUG 133 | std::size_t expected_size = std::accumulate(shape.begin(), shape.end(), std::size_t(1), 134 | std::multiplies<>()); 135 | assert(expected_size == data.size()); 136 | #endif 137 | heap.add_item(id, data.data(), data.size() * sizeof(T), false); 138 | } 139 | 140 | /** 141 | * Helper to call @ref add_descriptor and @ref add_constant with zeros. 142 | * 143 | * This is used just to fake up items that are currently expected by 144 | * timeplot. It should be removed once timeplot supports beamformer signal 145 | * displays. 146 | */ 147 | template 148 | static void add_zeros(spead2::send::heap &heap, spead2::s_item_pointer_t id, 149 | const std::string &name, 150 | const std::vector &shape, 151 | const std::string &dtype) 152 | { 153 | std::size_t n = 1; 154 | for (int s : shape) 155 | n *= s; 156 | add_descriptor(heap, id, name, "Dummy item", shape, dtype); 157 | add_constant(heap, id, std::string(n * sizeof(T), '\0')); 158 | } 159 | 160 | stats_collector::transmit_data::transmit_data(const session_config &config) 161 | : heap(make_flavour()), 162 | data(2 * config.channels), 163 | flags(2 * config.channels) 164 | { 165 | using namespace std::literals; 166 | 167 | add_vector(heap, id_sd_data, "sd_data", 168 | "Power spectrum and fraction of samples that are saturated. These are encoded " 169 | "as baselines with inputs m999h,m999h and m999v,m999v respectively.", 170 | {config.channels, 2, 2}, "f4", data); 171 | add_vector(heap, id_sd_flags, "sd_flags", "8bit packed flags for each data point.", 172 | {config.channels, 2}, "u1", flags); 173 | add_descriptor(heap, id_sd_timestamp, "sd_timestamp", "Timestamp of this sd frame in centiseconds since epoch", 174 | {}, "u8"); 175 | heap.add_item(id_sd_timestamp, ×tamp, sizeof(timestamp), true); 176 | 177 | // TODO: more fields 178 | add_descriptor(heap, id_n_chans, "n_chans", "Number of channels", {}, "u4"); 179 | add_constant(heap, id_n_chans, std::uint32_t(config.channels)); 180 | add_descriptor(heap, id_bandwidth, "bandwidth", "The analogue bandwidth of the digitally processed signal, in Hz.", 181 | {}, "f8"); 182 | add_constant(heap, id_bandwidth, config.bandwidth); 183 | add_descriptor(heap, id_center_freq, "center_freq", "The center frequency of the DBE in Hz, 64-bit IEEE floating-point number.", 184 | {}, "f8"); 185 | add_constant(heap, id_center_freq, config.center_freq); 186 | add_descriptor(heap, id_bls_ordering, "bls_ordering", "Baseline output ordering.", 187 | {2, 2}, "S5"); // Must match the chosen input name 188 | // TODO: use a proper name. The m999h/v is to fit the signal display's expectations 189 | add_constant(heap, id_bls_ordering, "m999hm999hm999vm999v"s); 190 | add_descriptor(heap, id_sd_data_index, "sd_data_index", "Indices for transmitted sd_data.", 191 | {2}, "u4"); 192 | add_constant(heap, id_sd_data_index, std::array{{0, 1}}); 193 | 194 | // TODO: fields below here are just for testing against a correlator signal 195 | // display server, and should mostly be removed. 196 | add_descriptor(heap, id_sd_blmx_n_chans, "sd_blmx_n_chans", "Dummy item", {}, "u4"); 197 | add_constant(heap, id_sd_blmx_n_chans, std::uint32_t(config.channels)); 198 | add_vector(heap, id_sd_blmxdata, "sd_blmxdata", "Dummy item", 199 | {config.channels, 2, 2}, "f4", data); 200 | add_vector(heap, id_sd_blmxflags, "sd_blmxflags", "Dummy item", 201 | {config.channels, 2}, "u1", flags); 202 | add_zeros(heap, id_sd_flag_fraction, "sd_flag_fraction", {2, 8}, "f4"); 203 | add_zeros(heap, id_sd_timeseries, "sd_timeseries", {2, 2}, "f4"); 204 | add_zeros(heap, id_sd_timeseriesabs, "sd_timeseriesabs", {2}, "f4"); 205 | add_zeros(heap, id_sd_percspectrum, "sd_percspectrum", {config.channels, 40}, "f4"); 206 | add_zeros(heap, id_sd_percspectrumflags, "sd_percspectrumflags", 207 | {config.channels, 40}, "u1"); 208 | } 209 | 210 | void stats_collector::send_heap(const spead2::send::heap &heap) 211 | { 212 | auto handler = [](const boost::system::error_code &ec, 213 | spead2::item_pointer_t bytes_transferred) 214 | { 215 | if (ec) 216 | log_format(spead2::log_level::warning, "Error sending heap: %s", ec.message()); 217 | }; 218 | stream.async_send_heap(heap, handler); 219 | io_service.run(); 220 | // io_service will refuse to run again unless reset is called 221 | io_service.reset(); 222 | } 223 | 224 | stats_collector::stats_collector(const session_config &config) 225 | : power_spectrum(config.channels), 226 | saturated(config.channels), 227 | weight(config.channels), 228 | data(config), 229 | sync_time(config.sync_time), 230 | scale_factor_timestamp(config.scale_factor_timestamp), 231 | freq_sys(config.get_freq_system()), 232 | time_sys(config.get_time_system()), 233 | stream(io_service, {config.stats_endpoint}, 234 | spead2::send::stream_config().set_max_packet_size(8872), 235 | spead2::send::udp_stream::default_buffer_size, 236 | 1, config.stats_interface_address) 237 | { 238 | /* spectra_per_heap is checked by session_config::validate, so this 239 | * is just a sanity check. It's necessary to limit spectra_per_heap 240 | * to avoid overflowing narrow integers during accumulation. If there 241 | * is a future need for larger values it can be handled by splitting 242 | * the accumulations into shorter pieces. 243 | */ 244 | assert((time_sys.scale_factor()) < 32768); 245 | spead2::send::heap start_heap; 246 | start_heap.add_start(); 247 | send_heap(start_heap); 248 | 249 | // Convert config.stats_int_time to timestamp units and round to whole heaps 250 | q::ticks interval_align = time_sys.convert_one(); 251 | interval = q::ticks(std::round(config.stats_int_time * config.scale_factor_timestamp)); 252 | interval = interval / interval_align * interval_align; 253 | if (interval <= q::ticks(0)) 254 | interval = interval_align; 255 | } 256 | 257 | void stats_collector::add(const slice &s) 258 | { 259 | q::ticks timestamp = s.timestamp; 260 | if (start_timestamp == q::ticks(-1)) 261 | start_timestamp = timestamp; 262 | assert(timestamp >= start_timestamp); // timestamps must be provided in order 263 | if (timestamp >= start_timestamp + interval) 264 | { 265 | transmit(); 266 | // Get start timestamp that is of form first_timestamp + i * interval 267 | start_timestamp += (timestamp - start_timestamp) / interval * interval; 268 | } 269 | 270 | // Update the statistics using the heaps in the slice 271 | const int8_t *data = reinterpret_cast(s.data.get()); 272 | std::size_t present_idx = 0; 273 | const q::spectra stride = time_sys.convert_one(); 274 | auto spectra_per_heap = time_sys.scale_factor(); 275 | for (q::heaps_f hf{0}; hf < freq_sys.convert_one(); ++hf) 276 | for (q::heaps_t ht{0}; ht < time_sys.convert_one(); ++ht, present_idx++) 277 | { 278 | if (!s.present[present_idx]) 279 | continue; 280 | q::channels start_channel = freq_sys.convert(hf); 281 | q::channels end_channel = start_channel + freq_sys.convert_one(); 282 | q::samples offset = time_sys.convert(ht) * q::channels(1); 283 | for (q::channels channel = start_channel; channel < end_channel; ++channel) 284 | { 285 | const int8_t *cdata = data + 2 * (channel * stride + offset).get(); 286 | power_spectrum[channel.get()] += power_sum(spectra_per_heap, cdata); 287 | saturated[channel.get()] += count_saturated(spectra_per_heap, cdata); 288 | weight[channel.get()] += spectra_per_heap; 289 | } 290 | } 291 | } 292 | 293 | void stats_collector::transmit() 294 | { 295 | // Compute derived values 296 | int channels = power_spectrum.size(); 297 | std::fill(data.flags.begin(), data.flags.end(), 0); 298 | for (int i = 0; i < channels; i++) 299 | { 300 | if (weight[i] != 0) 301 | { 302 | float w = 1.0f / weight[i]; 303 | data.data[2 * i] = power_spectrum[i] * w; 304 | data.data[2 * i + 1] = saturated[i] * w; 305 | } 306 | else 307 | { 308 | data.data[2 * i] = 0; 309 | data.data[2 * i + 1] = 0; 310 | data.flags[2 * i] = data_lost; 311 | data.flags[2 * i + 1] = data_lost; 312 | } 313 | } 314 | double timestamp_unix = sync_time + (start_timestamp + 0.5 * interval) / scale_factor_timestamp; 315 | // Convert to centiseconds, since that's what signal display uses 316 | data.timestamp = std::uint64_t(std::round(timestamp_unix * 100.0)); 317 | 318 | send_heap(data.heap); 319 | 320 | // Reset for the next interval 321 | std::fill(power_spectrum.begin(), power_spectrum.end(), 0); 322 | std::fill(saturated.begin(), saturated.end(), 0); 323 | std::fill(weight.begin(), weight.end(), 0); 324 | } 325 | 326 | stats_collector::~stats_collector() 327 | { 328 | // If start_timestamp != -1 then we received at least one heap, and from 329 | // then on we will always have an interval in progress. 330 | if (start_timestamp != q::ticks{-1}) 331 | transmit(); 332 | // Send stop heap 333 | spead2::send::heap heap; 334 | heap.add_end(); 335 | send_heap(heap); 336 | } 337 | -------------------------------------------------------------------------------- /katsdpbfingest/units.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 3 | * 4 | * Compile-time units and quantity types. 5 | * 6 | * This has some of the same basic motivation as Boost.Units, but it's not 7 | * quite the same thing - mainly because it is designed for cases where unit 8 | * conversion factors are only known at runtime. 9 | * 10 | * A @ref unit_system is an ordered set of units, where each is an integer 11 | * multiple of the previous one. Units are specified by type tags. The ratios 12 | * are stored in an instance of @ref unit_system. 13 | * 14 | * An example of usage: 15 | * @code 16 | * struct cores {}; 17 | * struct servers {}; 18 | * struct racks {}; 19 | * 20 | * typedef unit_system cpu_system; 21 | * 22 | * // 4-core servers, with 10 servers to a rack 23 | * cpu_system my_dc(4, 10); 24 | * // How many cores in 5 racks? 25 | * quantity c = my_dc.convert(make_quantity(5)); 26 | * // How many racks are needed to provide 50 cores (rounding up)? 27 | * quantity r = my_dc.convert_up(make_quantity(50)); 28 | * // How many servers per rack (as an int)? 29 | * int ratio = my_dc.scale_factor(); 30 | * // And as a quantity? 31 | * quantity ratioq = my_dc.convert_one(); 32 | * @endcode 33 | * 34 | * Quantities with the same unit can be used with addition, subtraction, 35 | * modulo, assignment, and comparison operators. Quantities can also be 36 | * multiplied if a specialization of @ref unit_product is provided to 37 | * indicate the unit of the product. For example: 38 | * 39 | * @code 40 | * struct newtons {}; 41 | * struct metres {}; 42 | * struct joules {}; 43 | * template<> struct unit_product { using type = joules; } 44 | * template<> struct unit_product { using type = joules; } 45 | * 46 | * quantity force(5); 47 | * quantity dist(3); 48 | * quantity work = force * dist; 49 | * @endcode 50 | */ 51 | 52 | #include 53 | #include 54 | #include 55 | 56 | // Forward declarations 57 | template class quantity; 58 | template constexpr quantity make_quantity(const T &amount); 59 | 60 | namespace detail 61 | { 62 | 63 | template 64 | static inline T div_round_up(T a, T b) 65 | { 66 | return (a + b - 1) / b; 67 | } 68 | 69 | /** 70 | * Identity type transformation that fails SFINAE for quantities. 71 | */ 72 | template 73 | struct not_quantity_helper 74 | { 75 | typedef T type; 76 | }; 77 | 78 | template 79 | struct not_quantity_helper> 80 | { 81 | }; 82 | 83 | template 84 | using not_quantity = not_quantity_helper::type>; 85 | 86 | template 87 | using not_quantity_t = typename not_quantity::type; 88 | 89 | } // namespace detail 90 | 91 | /** 92 | * A quantity of some unit. This is a type-safe wrapper of @a T, that does not 93 | * allow implicit conversions to or from @a T. It is intended to be used with 94 | * integral types. 95 | */ 96 | template 97 | class quantity 98 | { 99 | private: 100 | T amount; 101 | 102 | public: 103 | constexpr quantity() : amount() {} 104 | constexpr quantity(const quantity &other) = default; 105 | 106 | /// Retrieve the underlying value 107 | constexpr const T &get() const { return amount; } 108 | 109 | /// Construct from an integral type 110 | template::value>::type> 111 | constexpr explicit quantity(T2&& x) : amount(std::forward(x)) {} 112 | 113 | /// Construct from another quantity with the same units 114 | template::value>::type> 115 | constexpr quantity(const quantity &other) : amount(other.get()) {} 116 | 117 | #define MAKE_BASIC_OPERATOR(op) \ 118 | template \ 119 | quantity() op std::declval()), U> \ 120 | constexpr operator op(const quantity &other) const \ 121 | { \ 122 | return make_quantity(get() op other.get()); \ 123 | } 124 | 125 | #define MAKE_COMPARE_OPERATOR(op) \ 126 | template \ 127 | decltype(std::declval() op std::declval()) \ 128 | constexpr operator op(const quantity &other) const \ 129 | { \ 130 | return get() op other.get(); \ 131 | } 132 | 133 | // second template arg is just for SFINAE 134 | #define MAKE_ASSIGN_OPERATOR(op) \ 135 | template() op std::declval())> \ 136 | quantity &operator op(const quantity &other) \ 137 | { \ 138 | amount op other.get(); \ 139 | return *this; \ 140 | } 141 | 142 | MAKE_BASIC_OPERATOR(+) 143 | MAKE_BASIC_OPERATOR(-) 144 | MAKE_BASIC_OPERATOR(%) 145 | MAKE_COMPARE_OPERATOR(==) 146 | MAKE_COMPARE_OPERATOR(!=) 147 | MAKE_COMPARE_OPERATOR(<=) 148 | MAKE_COMPARE_OPERATOR(>=) 149 | MAKE_COMPARE_OPERATOR(<) 150 | MAKE_COMPARE_OPERATOR(>) 151 | MAKE_ASSIGN_OPERATOR(=) 152 | MAKE_ASSIGN_OPERATOR(+=) 153 | MAKE_ASSIGN_OPERATOR(-=) 154 | MAKE_ASSIGN_OPERATOR(%=) 155 | 156 | #undef MAKE_BASIC_OPERATOR 157 | #undef MAKE_COMPARE_OPERATOR 158 | #undef MAKE_ASSIGN_OPERATOR 159 | 160 | /// Multiply by a scalar. 161 | template 162 | quantity() * std::declval>()), U> 163 | constexpr operator *(T2 &&other) const 164 | { 165 | return make_quantity(get() * std::forward(other)); 166 | } 167 | 168 | // The second typename is just for SFINAE 169 | template() *= std::declval>())> 170 | quantity &operator *=(T2 &&other) 171 | { 172 | amount *= std::forward(other); 173 | return *this; 174 | } 175 | 176 | /** 177 | * Ratio of quantities, as a unitless scalar. This just uses the 178 | * underlying division operator, with the usual behaviour for integral 179 | * types. 180 | */ 181 | template 182 | decltype(std::declval() / std::declval()) 183 | constexpr operator /(const quantity &other) const 184 | { 185 | return get() / other.get(); 186 | } 187 | 188 | constexpr explicit operator bool() const 189 | { 190 | return (bool) amount; 191 | } 192 | 193 | constexpr bool operator !() const 194 | { 195 | return !amount; 196 | } 197 | 198 | // Prefix ++ 199 | quantity &operator++() 200 | { 201 | ++amount; 202 | return *this; 203 | } 204 | 205 | // Prefix -- 206 | quantity &operator--() 207 | { 208 | --amount; 209 | return *this; 210 | } 211 | 212 | // Postfix ++ 213 | quantity operator++(int) 214 | { 215 | return quantity(amount++); 216 | } 217 | 218 | // Postfix -- 219 | quantity operator--(int) 220 | { 221 | return quantity(amount--); 222 | } 223 | }; 224 | 225 | /** 226 | * Helper to construct a quantity while inferring the type. 227 | */ 228 | template 229 | constexpr quantity make_quantity(const T &amount) 230 | { 231 | return quantity(amount); 232 | } 233 | 234 | /** 235 | * Multiplication of scalar * quantity. 236 | */ 237 | template 238 | quantity>() * std::declval()), U> 239 | constexpr operator *(T1&& a, const quantity &b) 240 | { 241 | return b * a; 242 | } 243 | 244 | /** 245 | * Metaprogramming to determine the results of unit products. 246 | * 247 | * Specialize this class with a memory called @c type to indicate the 248 | * resulting unit type. Note that for heterogeneous products you should 249 | * specialise in both directions. 250 | */ 251 | template 252 | struct unit_product {}; 253 | 254 | /// Product of two quantities 255 | template 256 | quantity() * std::declval()), 257 | typename unit_product::type> 258 | constexpr operator *(const quantity &a, const quantity &b) 259 | { 260 | return make_quantity::type>(a.get() * b.get()); 261 | } 262 | 263 | /** 264 | * Output a quantity to a stream. The unit tag class must define a static 265 | * member function called @c name. 266 | */ 267 | template 268 | std::ostream &operator<<(std::ostream &os, const quantity &q) 269 | { 270 | return os << q.get() << ' ' << U::name(); 271 | } 272 | 273 | namespace detail 274 | { 275 | 276 | // Whether U is one of the types in Tags 277 | template 278 | class is_one_of : public std::false_type {}; 279 | 280 | template 281 | class is_one_of : public is_one_of {}; 282 | 283 | template 284 | class is_one_of : public std::true_type {}; 285 | 286 | 287 | // Position of U within Tags (or empty class if not in tags) 288 | template 289 | class index_of {}; 290 | 291 | template 292 | class index_of 293 | : public std::integral_constant::value> {}; 294 | 295 | template 296 | class index_of : public std::integral_constant {}; 297 | 298 | /** 299 | * Back-end of @ref unit_system. This is put into a separate class because it 300 | * needs several specializations to do its work. The template parameters are 301 | * the same as for @ref unit_system. 302 | */ 303 | template 304 | class unit_system_base {}; 305 | 306 | // At least two units, so at least one ratio 307 | template 308 | class unit_system_base 309 | { 310 | public: 311 | typedef unit_system_base tail_type; 312 | 313 | T ratio1 = 1; ///< Number of U1's per U2 314 | tail_type tail; ///< Remaining ratios (recursively) 315 | 316 | /// Scale factor to convert from @a src to @a dst 317 | template 318 | constexpr T scale_factor_impl(V1 *src, V2 *dst) const 319 | { 320 | // This is the recursive case, when V1 is not U1, so we recurse 321 | return tail.scale_factor_impl(src, dst); 322 | } 323 | 324 | template 325 | constexpr T scale_factor_impl(V *src, U1 *) const 326 | { 327 | // Base case when dst is U1 but src is not 328 | return ratio1 * tail.scale_factor_impl(src, (U2 *) nullptr); 329 | } 330 | 331 | constexpr T scale_factor_impl(U1 *, U1 *) const 332 | { 333 | // Base case when both src and dst are U1 334 | return T(1); 335 | } 336 | 337 | public: 338 | constexpr unit_system_base() = default; 339 | 340 | template 341 | constexpr unit_system_base(typename std::enable_if::type ratio1, 342 | Args&&... ratios) 343 | : ratio1(ratio1), tail(std::forward(ratios)...) 344 | { 345 | } 346 | 347 | constexpr unit_system_base(T ratio1, unit_system_base tail) 348 | : ratio1(ratio1), tail(tail) 349 | { 350 | } 351 | 352 | template 353 | constexpr unit_system_base append(T ratio) const 354 | { 355 | return unit_system_base(ratio1, tail.template append(ratio)); 356 | } 357 | }; 358 | 359 | // Base case of recursion: only one unit, so no ratios 360 | template 361 | class unit_system_base 362 | { 363 | public: 364 | constexpr unit_system_base() = default; 365 | 366 | constexpr T scale_factor_impl(U *, U *) const 367 | { 368 | return T(1); 369 | } 370 | 371 | template 372 | constexpr unit_system_base append(T ratio) const 373 | { 374 | return unit_system_base(ratio); 375 | } 376 | }; 377 | 378 | } // namespace detail 379 | 380 | /** 381 | * Collection of units with defined conversion factors. 382 | * 383 | * Provides conversions between instances of @c quantity for any @a U in 384 | * @a Units. All calculations are done in the integral type @a T. 385 | */ 386 | template 387 | class unit_system : private detail::unit_system_base 388 | { 389 | public: 390 | using detail::unit_system_base::unit_system_base; 391 | 392 | constexpr unit_system(const detail::unit_system_base &base) 393 | : detail::unit_system_base(base) 394 | { 395 | } 396 | 397 | /// True type if @a U is one of the units 398 | template 399 | using is_unit = std::integral_constant::value>; 400 | 401 | /// Index of @a U amongst the units 402 | template 403 | using unit_index = std::integral_constant::value>; 404 | 405 | /** 406 | * Convert from one quantity to another. This overload only handles 407 | * conversions from larger to smaller units (conversions from smaller or 408 | * larger must be done via @ref convert_down or @ref convert_up. 409 | */ 410 | template 411 | typename std::enable_if::value < unit_index::value, quantity>::type 412 | constexpr convert(const quantity &value) const 413 | { 414 | return quantity(value.get() * scale_factor()); 415 | } 416 | 417 | /// No-op conversion. 418 | template 419 | typename std::enable_if::value, quantity>::type 420 | constexpr convert(const quantity &value) const 421 | { 422 | return value; 423 | } 424 | 425 | /** 426 | * Convert from smaller to larger units, rounding down result. 427 | */ 428 | template 429 | typename std::enable_if::value < unit_index::value, quantity>::type 430 | constexpr convert_down(const quantity &value) const 431 | { 432 | return quantity(value.get() / scale_factor()); 433 | } 434 | 435 | /** 436 | * Convert from smaller to larger units, round up result. 437 | */ 438 | template 439 | typename std::enable_if::value < unit_index::value, quantity>::type 440 | constexpr convert_up(const quantity &value) const 441 | { 442 | return quantity(detail::div_round_up(value.get(), scale_factor())); 443 | } 444 | 445 | /** 446 | * Number of @a U2 in each @a U1. Only valid if U1 appears at or after U2 447 | * in unit list. 448 | */ 449 | template 450 | typename std::enable_if::value >= unit_index::value, T>::type 451 | constexpr scale_factor() const 452 | { 453 | return detail::unit_system_base::scale_factor_impl((U1 *) nullptr, (U2 *) nullptr); 454 | } 455 | 456 | /** 457 | * Number of @a U2 in each @a U1, as a quantity. Only valid if U1 appears at or after U2 458 | * in unit list. 459 | */ 460 | template 461 | typename std::enable_if::value >= unit_index::value, quantity>::type 462 | constexpr convert_one() const 463 | { 464 | return quantity(scale_factor()); 465 | } 466 | 467 | /** 468 | * Create a new unit system with an additional unit at the end. 469 | */ 470 | template 471 | constexpr unit_system append(T ratio) const 472 | { 473 | return unit_system( 474 | detail::unit_system_base::unit_system_base::template append(ratio)); 475 | } 476 | }; 477 | -------------------------------------------------------------------------------- /katsdpbfingest/receiver.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include "common.h" 23 | #include "receiver.h" 24 | 25 | // TODO: only used for gil_scoped_release. Would be nice to find a way to avoid 26 | // having this file depend on pybind11. 27 | namespace py = pybind11; 28 | 29 | constexpr std::size_t receiver::window_size; 30 | constexpr int receiver::bf_raw_id; 31 | constexpr int receiver::timestamp_id; 32 | constexpr int receiver::frequency_id; 33 | 34 | std::unique_ptr receiver::make_slice() 35 | { 36 | std::unique_ptr s{new slice}; 37 | auto slice_samples = 38 | time_sys.convert_one() 39 | * freq_sys.convert_one(); 40 | auto present_size = 41 | time_sys.convert_one() 42 | * freq_sys.convert_one(); 43 | s->data = make_aligned(slice::bytes(slice_samples)); 44 | // Fill the data just to pre-fault it 45 | std::memset(s->data.get(), 0, slice::bytes(slice_samples)); 46 | s->present = std::make_unique(present_size.get()); 47 | s->present_size = present_size.get(); 48 | return s; 49 | } 50 | 51 | void receiver::emplace_readers() 52 | { 53 | #if SPEAD2_USE_IBV 54 | if (use_ibv) 55 | { 56 | log_format(spead2::log_level::info, "Listening on %1% with interface %2% using ibverbs", 57 | config.endpoints_str, config.interface_address); 58 | stream.emplace_reader( 59 | spead2::recv::udp_ibv_config() 60 | .set_endpoints(config.endpoints) 61 | .set_interface_address(config.interface_address) 62 | .set_max_size(config.max_packet) 63 | .set_buffer_size(config.buffer_size) 64 | .set_comp_vector(config.comp_vector)); 65 | } 66 | else 67 | #endif 68 | if (!config.interface_address.is_unspecified()) 69 | { 70 | log_format(spead2::log_level::info, "Listening on %1% with interface %2%", 71 | config.endpoints_str, config.interface_address); 72 | for (const auto &endpoint : config.endpoints) 73 | stream.emplace_reader( 74 | endpoint, config.max_packet, config.buffer_size, 75 | config.interface_address); 76 | } 77 | else 78 | { 79 | log_format(spead2::log_level::info, "Listening on %1%", config.endpoints_str); 80 | for (const auto &endpoint : config.endpoints) 81 | stream.emplace_reader( 82 | endpoint, config.max_packet, config.buffer_size); 83 | } 84 | } 85 | 86 | void receiver::add_tcp_reader(const spead2::socket_wrapper &acceptor) 87 | { 88 | stream.emplace_reader( 89 | acceptor.copy(stream.get_io_service()), config.max_packet); 90 | } 91 | 92 | bool receiver::parse_timestamp_channel( 93 | q::ticks timestamp, q::channels channel, 94 | q::spectra &spectrum, 95 | std::size_t &heap_offset, q::heaps &present_idx, 96 | std::uint64_t *counter_stats, bool quiet) 97 | { 98 | if (timestamp < first_timestamp) 99 | { 100 | if (!quiet) 101 | { 102 | counter_stats[counters::before_start_heaps]++; 103 | log_format(spead2::log_level::debug, "timestamp %1% pre-dates start %2%, discarding", 104 | timestamp, first_timestamp); 105 | } 106 | return false; 107 | } 108 | bool have_first = (first_timestamp != q::ticks(-1)); 109 | q::ticks rel = !have_first ? q::ticks(0) : timestamp - first_timestamp; 110 | q::ticks one_heap_ts = time_sys.convert_one(); 111 | q::channels one_slice_f = freq_sys.convert_one(); 112 | q::channels one_heap_f = freq_sys.convert_one(); 113 | if (rel % one_heap_ts) 114 | { 115 | if (!quiet) 116 | { 117 | counter_stats[counters::bad_timestamp_heaps]++; 118 | log_format(spead2::log_level::debug, "timestamp %1% is not properly aligned to %2%, discarding", 119 | timestamp, one_heap_ts); 120 | } 121 | return false; 122 | } 123 | if (channel % one_heap_f) 124 | { 125 | if (!quiet) 126 | { 127 | counter_stats[counters::bad_channel_heaps]++; 128 | log_format(spead2::log_level::debug, "frequency %1% is not properly aligned to %2%, discarding", 129 | channel, one_heap_f); 130 | } 131 | return false; 132 | } 133 | if (channel < channel_offset || channel >= one_slice_f + channel_offset) 134 | { 135 | if (!quiet) 136 | { 137 | counter_stats[counters::bad_channel_heaps]++; 138 | log_format(spead2::log_level::debug, "frequency %1% is outside of range [%2%, %3%), discarding", 139 | channel, channel_offset, one_slice_f + channel_offset); 140 | } 141 | return false; 142 | } 143 | 144 | channel -= channel_offset; 145 | spectrum = time_sys.convert_down(rel); 146 | 147 | // Pre-compute some conversion factors 148 | q::slices_t one_slice(1); 149 | q::heaps_t slice_heaps = time_sys.convert(one_slice); 150 | q::spectra slice_spectra = time_sys.convert(slice_heaps); 151 | 152 | // Compute slice-local coordinates 153 | q::heaps_t time_heaps = time_sys.convert_down(spectrum % slice_spectra); 154 | q::samples time_samples = time_sys.convert(time_heaps) * q::channels(1); 155 | q::heaps_f freq_heaps = freq_sys.convert_down(channel); 156 | heap_offset = slice::bytes(time_samples + channel * slice_spectra); 157 | present_idx = time_heaps * q::heaps_f(1) + freq_heaps * slice_heaps; 158 | 159 | if (!have_first) 160 | first_timestamp = timestamp; 161 | return true; 162 | } 163 | 164 | void receiver::finish_slice(slice &s, std::uint64_t *counter_stats) const 165 | { 166 | std::size_t n_present = std::accumulate(s.present.get(), 167 | s.present.get() + s.present_size, std::size_t(0)); 168 | s.n_present = q::heaps(n_present); 169 | if (n_present == 0) 170 | return; 171 | 172 | q::slices_t slice_id{s.chunk_id}; 173 | s.spectrum = time_sys.convert(slice_id); 174 | s.timestamp = time_sys.convert(s.spectrum) + first_timestamp; 175 | 176 | counter_stats[counters::data_heaps] += s.n_present.get(); 177 | counter_stats[counters::bytes] += s.n_present.get() * payload_size; 178 | std::int64_t total_heaps = (slice_id.get() + 1) * s.present_size; 179 | if (total_heaps > counter_stats[counters::total_heaps]) 180 | counter_stats[counters::total_heaps] = total_heaps; 181 | 182 | // If any heaps got lost, fill them with zeros 183 | if (n_present != s.present_size) 184 | { 185 | const q::heaps_f slice_heaps_f = freq_sys.convert_one(); 186 | const q::heaps_t slice_heaps_t = time_sys.convert_one(); 187 | const std::size_t heap_row = 188 | slice::bytes(time_sys.convert_one() * q::channels(1)); 189 | const q::channels heap_channels = freq_sys.convert_one(); 190 | const q::spectra stride = time_sys.convert_one(); 191 | const std::size_t stride_bytes = slice::bytes(stride * q::channels(1)); 192 | q::heaps present_idx{0}; 193 | for (q::heaps_f i{0}; i < slice_heaps_f; i++) 194 | for (q::heaps_t j{0}; j < slice_heaps_t; j++, present_idx++) 195 | if (!s.present[present_idx.get()]) 196 | { 197 | auto start_channel = freq_sys.convert(i); 198 | const q::samples dst_offset = 199 | start_channel * stride 200 | + time_sys.convert(j) * q::channels(1); 201 | std::uint8_t *ptr = s.data.get() + slice::bytes(dst_offset); 202 | for (q::channels k{0}; k < heap_channels; k++, ptr += stride_bytes) 203 | std::memset(ptr, 0, heap_row); 204 | } 205 | } 206 | } 207 | 208 | void receiver::packet_memcpy(const spead2::memory_allocator::pointer &allocation, 209 | const spead2::recv::packet_header &packet) 210 | { 211 | typedef unit_system stride_system; 212 | stride_system src_sys( 213 | slice::bytes(time_sys.convert_one() * q::channels(1))); 214 | stride_system dst_sys( 215 | slice::bytes(time_sys.convert_one() * q::channels(1))); 216 | q::bytes src_stride = src_sys.convert_one(); 217 | /* Copy one channel at a time. Some extra index manipulation is needed 218 | * because the packet might have partial channels at the start and end, 219 | * or only a middle part of a channel. 220 | * 221 | * Some of this could be optimised by handling the complete channels 222 | * separately from the leftovers (particularly since in MeerKAT we expect 223 | * there not to be any leftovers). 224 | * 225 | * coordinates are all relative to the start of the heap. 226 | */ 227 | q::bytes payload_start(packet.payload_offset); 228 | q::bytes payload_length(packet.payload_length); 229 | q::bytes payload_end = payload_start + payload_length; 230 | q::channels channel_start = src_sys.convert_down(payload_start); 231 | q::channels channel_end = src_sys.convert_up(payload_end); 232 | for (q::channels c = channel_start; c < channel_end; c++) 233 | { 234 | q::bytes src_start = src_sys.convert(c); 235 | q::bytes src_end = src_start + src_stride; 236 | q::bytes dst_start = dst_sys.convert(c); 237 | if (payload_start > src_start) 238 | { 239 | dst_start += payload_start - src_start; 240 | src_start = payload_start; 241 | } 242 | if (payload_end < src_end) 243 | src_end = payload_end; 244 | std::memcpy(allocation.get() + dst_start.get(), 245 | packet.payload + (src_start - payload_start).get(), 246 | (src_end - src_start).get()); 247 | } 248 | } 249 | 250 | void receiver::place(spead2::recv::chunk_place_data *data, std::size_t data_size) 251 | { 252 | // This assert will fail if the spead2 library at runtime is older than the 253 | // version we were compiled against. 254 | assert(data_size >= sizeof(*data)); 255 | 256 | q::ticks timestamp{data->items[0]}; 257 | q::channels channel{data->items[1]}; 258 | spead2::item_pointer_t heap_size = data->items[2]; 259 | // Metadata heaps will be missing timestamp and channel 260 | std::uint64_t *counter_stats = data->batch_stats + counter_base; 261 | if (timestamp == q::ticks(-1) || channel == q::channels(-1)) 262 | { 263 | counter_stats[counters::metadata_heaps]++; 264 | return; 265 | } 266 | 267 | q::spectra spectrum; 268 | std::size_t heap_offset; 269 | q::heaps present_idx; 270 | if (!parse_timestamp_channel(timestamp, channel, spectrum, heap_offset, present_idx, 271 | counter_stats)) 272 | return; 273 | 274 | if (heap_size != payload_size) 275 | { 276 | counter_stats[counters::bad_length_heaps]++; 277 | log_format(spead2::log_level::debug, "heap has wrong length (%1% != %2%), discarding", 278 | heap_size, payload_size); 279 | return; 280 | } 281 | 282 | data->chunk_id = time_sys.convert_down(spectrum).get(); 283 | data->heap_offset = heap_offset; 284 | data->heap_index = present_idx.get(); 285 | } 286 | 287 | void receiver::graceful_stop() 288 | { 289 | // Abuse the fact that a mem_reader calls stop_received when it gets to the end 290 | stream.emplace_reader(nullptr, 0); 291 | } 292 | 293 | void receiver::stop() 294 | { 295 | stream.stop(); 296 | } 297 | 298 | spead2::recv::stream_config receiver::make_stream_config() 299 | { 300 | spead2::recv::stream_config stream_config; 301 | stream_config.set_max_heaps( 302 | std::max(1, config.channels / config.channels_per_heap) * config.live_heaps_per_substream); 303 | stream_config.set_memcpy( 304 | [this](const spead2::memory_allocator::pointer &allocation, 305 | const spead2::recv::packet_header &packet) 306 | { 307 | packet_memcpy(allocation, packet); 308 | }); 309 | stream_config.add_stat("katsdpbfingest.metadata_heaps"); 310 | stream_config.add_stat("katsdpbfingest.bad_timestamp_heaps"); 311 | stream_config.add_stat("katsdpbfingest.bad_channel_heaps"); 312 | stream_config.add_stat("katsdpbfingest.bad_length_heaps"); 313 | stream_config.add_stat("katsdpbfingest.before_start_heaps"); 314 | stream_config.add_stat("katsdpbfingest.data_heaps"); 315 | stream_config.add_stat("katsdpbfingest.total_heaps", 316 | spead2::recv::stream_stat_config::mode::MAXIMUM); 317 | stream_config.add_stat("katsdpbfingest.bytes"); 318 | return stream_config; 319 | } 320 | 321 | spead2::recv::chunk_stream_config receiver::make_chunk_stream_config() 322 | { 323 | spead2::recv::chunk_stream_config config; 324 | config.set_items({timestamp_id, frequency_id, spead2::item_id::HEAP_LENGTH_ID}); 325 | config.set_max_chunks(window_size); 326 | config.set_place([this](spead2::recv::chunk_place_data *data, std::size_t data_size) 327 | { 328 | place(data, data_size); 329 | }); 330 | config.set_ready([this](std::unique_ptr &&chunk, std::uint64_t *batch_stats) 331 | { 332 | finish_slice(static_cast(*chunk), batch_stats + counter_base); 333 | }); 334 | return config; 335 | } 336 | 337 | using ringbuffer_t = spead2::ringbuffer>; 338 | 339 | receiver::receiver(const session_config &config) 340 | : config(config), 341 | channel_offset(config.channel_offset), 342 | freq_sys(config.get_freq_system()), 343 | time_sys(config.get_time_system()), 344 | payload_size(2 * sizeof(std::int8_t) * config.spectra_per_heap * config.channels_per_heap), 345 | worker(1, affinity_vector(config.network_affinity)), 346 | stream( 347 | worker, 348 | make_stream_config(), 349 | make_chunk_stream_config(), 350 | std::make_shared(config.ring_slots), 351 | std::make_shared(window_size + config.ring_slots + 1)) 352 | { 353 | py::gil_scoped_release gil; 354 | 355 | counter_base = stream.get_config().get_stat_index("katsdpbfingest.metadata_heaps"); 356 | try 357 | { 358 | if (config.ibv) 359 | { 360 | use_ibv = true; 361 | #if !SPEAD2_USE_IBV 362 | log_message(spead2::log_level::warning, "Not using ibverbs because support is not compiled in"); 363 | use_ibv = false; 364 | #endif 365 | if (use_ibv) 366 | { 367 | for (const auto &endpoint : config.endpoints) 368 | if (!endpoint.address().is_multicast()) 369 | { 370 | log_format(spead2::log_level::warning, "Not using ibverbs because endpoint %1% is not multicast", 371 | endpoint); 372 | use_ibv = false; 373 | break; 374 | } 375 | } 376 | if (use_ibv && config.interface_address.is_unspecified()) 377 | { 378 | log_message(spead2::log_level::warning, "Not using ibverbs because interface address is not specified"); 379 | use_ibv = false; 380 | } 381 | } 382 | 383 | for (std::size_t i = 0; i < window_size + config.ring_slots + 1; i++) 384 | stream.add_free_chunk(make_slice()); 385 | 386 | emplace_readers(); 387 | } 388 | catch (std::exception &) 389 | { 390 | /* Normally we can rely on the destructor to call stop() (which is 391 | * necessary to ensure that the stream isn't going to make more calls 392 | * into the receiver while it is being destroyed), but an exception 393 | * thrown from the constructor does not cause the destructor to get 394 | * called. 395 | * 396 | * TODO: is this still necessary? The stream now interacts directly 397 | * with the ringbuffers. 398 | */ 399 | stop(); 400 | throw; 401 | } 402 | } 403 | 404 | receiver::~receiver() 405 | { 406 | stop(); 407 | } 408 | -------------------------------------------------------------------------------- /katsdpbfingest/test/test_bf_ingest_server.py: -------------------------------------------------------------------------------- 1 | """Tests for the bf_ingest_server module""" 2 | 3 | import argparse 4 | import tempfile 5 | import shutil 6 | import os.path 7 | import contextlib 8 | import socket 9 | import asyncio 10 | from unittest import mock 11 | from typing import List, Optional 12 | 13 | import h5py 14 | import numpy as np 15 | 16 | import spead2 17 | import spead2.recv 18 | import spead2.send 19 | 20 | import asynctest 21 | from nose.tools import assert_equal, assert_true, assert_false, assert_is_none 22 | 23 | import katsdptelstate 24 | from katsdptelstate import endpoint 25 | 26 | from katsdpbfingest import bf_ingest_server, _bf_ingest 27 | 28 | 29 | DATA_LOST = 1 << 3 30 | 31 | 32 | class TestSession: 33 | def setup(self) -> None: 34 | # To avoid collisions when running tests in parallel on a single host, 35 | # create a socket for the duration of the test and use its port as the 36 | # port for the test. Sockets in the same network namespace should have 37 | # unique ports. 38 | self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 39 | self._sock.bind(('127.0.0.1', 0)) 40 | self.port = self._sock.getsockname()[1] 41 | self.tmpdir = tempfile.mkdtemp() 42 | 43 | def teardown(self) -> None: 44 | shutil.rmtree(self.tmpdir) 45 | self._sock.close() 46 | 47 | def test_no_stop(self) -> None: 48 | """Deleting a session without stopping it must tidy up""" 49 | config = _bf_ingest.SessionConfig(os.path.join(self.tmpdir, 'test_no_stop.h5')) 50 | config.add_endpoint('239.1.2.3', self.port) 51 | config.channels = 4096 52 | config.channels_per_heap = 256 53 | config.spectra_per_heap = 256 54 | config.ticks_between_spectra = 8192 55 | config.sync_time = 1111111111.0 56 | config.bandwidth = 856e6 57 | config.center_freq = 1284e6 58 | config.scale_factor_timestamp = 1712e6 59 | config.heaps_per_slice_time = 2 60 | _bf_ingest.Session(config) 61 | 62 | 63 | def _make_listen_socket(): 64 | sock = socket.socket() 65 | sock.bind(('127.0.0.1', 0)) 66 | sock.listen() 67 | return sock 68 | 69 | 70 | class TestCaptureServer(asynctest.TestCase): 71 | def setUp(self) -> None: 72 | self.tmpdir = tempfile.mkdtemp() 73 | self.addCleanup(shutil.rmtree, self.tmpdir) 74 | self.port = 7148 75 | self.n_channels = 1024 76 | self.spectra_per_heap = 256 77 | # No data actually travels through these multicast groups; 78 | # it gets mocked out to use local TCP sockets instead. 79 | self.endpoints = endpoint.endpoint_list_parser(self.port)( 80 | '239.102.2.0+7:{}'.format(self.port)) 81 | self.tcp_acceptors = [_make_listen_socket() for endpoint in self.endpoints] 82 | self.tcp_endpoints = [endpoint.Endpoint(*sock.getsockname()) for sock in self.tcp_acceptors] 83 | self.n_bengs = 16 84 | self.ticks_between_spectra = 8192 85 | self.adc_sample_rate = 1712000000.0 86 | self.heaps_per_stats = 6 87 | self.channels_per_heap = self.n_channels // self.n_bengs 88 | self.channels_per_endpoint = self.n_channels // len(self.endpoints) 89 | attrs = { 90 | 'i0_tied_array_channelised_voltage_0x_n_chans': self.n_channels, 91 | 'i0_tied_array_channelised_voltage_0x_n_chans_per_substream': self.channels_per_heap, 92 | 'i0_tied_array_channelised_voltage_0x_spectra_per_heap': self.spectra_per_heap, 93 | 'i0_tied_array_channelised_voltage_0x_src_streams': [ 94 | 'i0_antenna_channelised_voltage'], 95 | 'i0_tied_array_channelised_voltage_0x_bandwidth': self.adc_sample_rate / 2, 96 | 'i0_tied_array_channelised_voltage_0x_center_freq': 3 * self.adc_sample_rate / 2, 97 | 'i0_antenna_channelised_voltage_ticks_between_spectra': self.ticks_between_spectra, 98 | 'i0_antenna_channelised_voltage_instrument_dev_name': 'i0', 99 | 'i0_sync_time': 111111111.0, 100 | 'i0_scale_factor_timestamp': self.adc_sample_rate 101 | } 102 | telstate = katsdptelstate.TelescopeState() 103 | for key, value in attrs.items(): 104 | telstate[key] = value 105 | stats_int_time = (self.heaps_per_stats * self.ticks_between_spectra * 106 | self.spectra_per_heap / self.adc_sample_rate) 107 | self.args = bf_ingest_server.parse_args([ 108 | '--cbf-spead=' + endpoint.endpoints_to_str(self.endpoints), 109 | '--channels=128:768', 110 | '--file-base=' + self.tmpdir, 111 | '--stream-name=i0_tied_array_channelised_voltage_0x', 112 | '--interface=lo', 113 | '--stats=239.102.3.0:7149', 114 | '--stats-int-time={}'.format(stats_int_time), 115 | '--stats-interface=lo'], 116 | argparse.Namespace(telstate=telstate)) 117 | self.loop = asyncio.get_event_loop() 118 | self.patch_add_endpoint() 119 | self.patch_create_session_config() 120 | self.patch_session_factory() 121 | 122 | def patch_add_endpoint(self): 123 | """Prevent actual endpoints from being added, since we're using TCP instead.""" 124 | patcher = mock.patch.object(_bf_ingest.SessionConfig, 'add_endpoint') 125 | patcher.start() 126 | self.addCleanup(patcher.stop) 127 | 128 | def patch_create_session_config(self): 129 | """Force heaps_per_slice_time to 2. 130 | 131 | The test is written around this value, but the default is to compute 132 | it from other parameters. 133 | """ 134 | def create_session_config(args: argparse.Namespace) -> _bf_ingest.SessionConfig: 135 | config = orig_create_session_config(args) 136 | config.heaps_per_slice_time = 2 137 | return config 138 | 139 | orig_create_session_config = bf_ingest_server.create_session_config 140 | patcher = mock.patch.object( 141 | bf_ingest_server, 'create_session_config', create_session_config) 142 | patcher.start() 143 | self.addCleanup(patcher.stop) 144 | 145 | def patch_session_factory(self): 146 | def session_factory(config: _bf_ingest.SessionConfig) -> _bf_ingest.Session: 147 | session = _bf_ingest.Session(config) 148 | for sock in self.tcp_acceptors: 149 | session.add_tcp_reader(sock) 150 | sock.close() 151 | return session 152 | 153 | patcher = mock.patch.object(bf_ingest_server, 'session_factory', session_factory) 154 | patcher.start() 155 | self.addCleanup(patcher.stop) 156 | 157 | async def test_manual_stop_no_data(self) -> None: 158 | """Manual stop before any data is received""" 159 | server = bf_ingest_server.CaptureServer(self.args, self.loop) 160 | assert_false(server.capturing) 161 | await server.start_capture('1122334455') 162 | assert_true(server.capturing) 163 | await asyncio.sleep(0.01) 164 | await server.stop_capture() 165 | assert_false(server.capturing) 166 | 167 | async def _test_stream(self, end: bool, write: bool) -> None: 168 | n_heaps = 30 # number of heaps in time 169 | n_spectra = self.spectra_per_heap * n_heaps 170 | # Pick some heaps to drop, including an entire slice and 171 | # an entire channel for one stats dump 172 | drop = np.zeros((self.n_bengs, n_heaps), np.bool_) 173 | drop[:, 4] = True 174 | drop[2, 9] = True 175 | drop[7, 24] = True 176 | drop[10, 12:18] = True 177 | if not write: 178 | self.args.file_base = None 179 | 180 | # Start a receiver to get the signal display stream. 181 | # It needs a deep queue because we don't service it while it is 182 | # running. 183 | rx = spead2.recv.Stream( 184 | spead2.ThreadPool(), 185 | spead2.recv.StreamConfig(max_heaps=2, stop_on_stop_item=False), 186 | spead2.recv.RingStreamConfig(heaps=100) 187 | ) 188 | rx.add_udp_reader(self.args.stats.host, self.args.stats.port, 189 | interface_address='127.0.0.1') 190 | 191 | # Start up the server 192 | server = bf_ingest_server.CaptureServer(self.args, self.loop) 193 | filename = await server.start_capture('1122334455') 194 | # Send it a SPEAD stream. Use small packets to ensure that each heap is 195 | # split into multiple packets, to check that the data scatter works. 196 | config = spead2.send.StreamConfig(max_packet_size=256) 197 | flavour = spead2.Flavour(4, 64, 48, 0) 198 | ig = spead2.send.ItemGroup(flavour=flavour) 199 | ig.add_item(name='timestamp', id=0x1600, 200 | description='Timestamp', shape=(), format=[('u', 48)]) 201 | ig.add_item(name='frequency', id=0x4103, 202 | description='The frequency channel of the data in this HEAP.', 203 | shape=(), format=[('u', 48)]) 204 | ig.add_item(name='bf_raw', id=0x5000, 205 | description='Beamformer data', 206 | shape=(self.channels_per_heap, self.spectra_per_heap, 2), 207 | dtype=np.dtype(np.int8)) 208 | # To guarantee in-order delivery (and hence make the test 209 | # reliable/reproducible), we send all the data for the channels of 210 | # interest through a single TCP socket. Data for channels outside the 211 | # subscribed range is discarded. Note that giving multiple TcpStream's 212 | # the same socket is dangerous because individual write() calls could 213 | # interleave; it's safe only because we use only blocking calls so 214 | # there is no concurrency between the streams. 215 | subscribed_streams = self.args.channels // self.channels_per_endpoint 216 | subscribed_bengs = self.args.channels // self.channels_per_heap 217 | expected_heaps = 0 218 | streams = [] # type: List[Optional[spead2.send.TcpStream]] 219 | primary_ep = self.tcp_endpoints[subscribed_streams.start] 220 | sock = socket.socket() 221 | sock.setblocking(False) 222 | await self.loop.sock_connect(sock, (primary_ep.host, primary_ep.port)) 223 | for i in range(len(self.endpoints)): 224 | if i not in subscribed_streams: 225 | streams.append(None) 226 | continue 227 | stream = spead2.send.TcpStream(spead2.ThreadPool(), sock, config) 228 | streams.append(stream) 229 | stream.set_cnt_sequence(i, len(self.endpoints)) 230 | stream.send_heap(ig.get_heap(descriptors='all')) 231 | stream.send_heap(ig.get_start()) 232 | expected_heaps += 2 233 | sock.close() 234 | ts = 1234567890 235 | for i in range(n_heaps): 236 | data = np.zeros((self.n_channels, self.spectra_per_heap, 2), np.int8) 237 | for channel in range(self.n_channels): 238 | data[channel, :, 0] = channel % 255 - 128 239 | for t in range(self.spectra_per_heap): 240 | data[:, t, 1] = (i * self.spectra_per_heap + t) % 255 - 128 241 | for j in range(self.n_bengs): 242 | ig['timestamp'].value = ts 243 | ig['frequency'].value = j * self.channels_per_heap 244 | ig['bf_raw'].value = data[j * self.channels_per_heap 245 | : (j + 1) * self.channels_per_heap, ...] 246 | if not drop[j, i]: 247 | heap = ig.get_heap() 248 | # The receiver looks at inline items in each packet to place 249 | # data correctly. 250 | heap.repeat_pointers = True 251 | if j in subscribed_bengs: 252 | out_stream = streams[j // (self.n_bengs // len(self.endpoints))] 253 | assert out_stream is not None 254 | out_stream.send_heap(heap) 255 | expected_heaps += 1 256 | ts += self.spectra_per_heap * self.ticks_between_spectra 257 | if end: 258 | for out_stream in streams: 259 | if out_stream is not None: 260 | out_stream.send_heap(ig.get_end()) 261 | # They're all pointing at the same receiver, which will 262 | # shut down after the first stop heap 263 | break 264 | streams = [] 265 | 266 | if not end: 267 | # We only want to stop the capture once all the heaps we expect 268 | # have been received, but time out after 5 seconds to avoid 269 | # hanging the test. 270 | for i in range(100): 271 | if server.counters['heaps'] >= expected_heaps: 272 | break 273 | else: 274 | print('Only {} / {} heaps received so far'.format( 275 | server.counters['heaps'], expected_heaps)) 276 | await asyncio.sleep(0.05) 277 | else: 278 | print('Giving up waiting for heaps') 279 | await server.stop_capture(force=not end) 280 | 281 | expected_data = np.zeros((self.n_channels, n_spectra, 2), np.int8) 282 | expected_weight = np.ones((self.n_channels, n_spectra), np.int8) 283 | for channel in range(self.n_channels): 284 | expected_data[channel, :, 0] = channel % 255 - 128 285 | for t in range(n_spectra): 286 | expected_data[:, t, 1] = t % 255 - 128 287 | for i in range(self.n_bengs): 288 | for j in range(n_heaps): 289 | if drop[i, j]: 290 | channel0 = i * self.channels_per_heap 291 | spectrum0 = j * self.spectra_per_heap 292 | index = np.s_[channel0 : channel0 + self.channels_per_heap, 293 | spectrum0 : spectrum0 + self.spectra_per_heap] 294 | expected_data[index] = 0 295 | expected_weight[index] = 0 296 | expected_data = expected_data[self.args.channels.asslice()] 297 | expected_weight = expected_weight[self.args.channels.asslice()] 298 | 299 | # Validate the output 300 | if write: 301 | h5file = h5py.File(filename, 'r') 302 | with contextlib.closing(h5file): 303 | bf_raw = h5file['/Data/bf_raw'] 304 | np.testing.assert_equal(expected_data, bf_raw) 305 | 306 | timestamps = h5file['/Data/timestamps'] 307 | expected = 1234567890 \ 308 | + self.ticks_between_spectra * np.arange(self.spectra_per_heap * n_heaps) 309 | np.testing.assert_equal(expected, timestamps) 310 | 311 | flags = h5file['/Data/flags'] 312 | expected = np.where(drop, 8, 0).astype(np.uint8) 313 | expected = expected[self.args.channels.start // self.channels_per_heap : 314 | self.args.channels.stop // self.channels_per_heap] 315 | np.testing.assert_equal(expected, flags) 316 | 317 | data_set = h5file['/Data'] 318 | assert_equal('i0_tied_array_channelised_voltage_0x', data_set.attrs['stream_name']) 319 | assert_equal(self.args.channels.start, data_set.attrs['channel_offset']) 320 | else: 321 | assert_is_none(filename) 322 | 323 | # Validate the signal display stream 324 | rx.stop() 325 | heaps = list(rx) 326 | # Note: would need updating if n_heaps is not a multiple of heaps_per_stats 327 | assert_equal(n_heaps // self.heaps_per_stats + 2, len(heaps)) 328 | assert_true(heaps[0].is_start_of_stream()) 329 | assert_true(heaps[-1].is_end_of_stream()) 330 | ig = spead2.send.ItemGroup() 331 | spectrum = 0 332 | spectra_per_stats = self.heaps_per_stats * self.spectra_per_heap 333 | for rx_heap in heaps[1:-1]: 334 | updated = ig.update(rx_heap) 335 | rx_data = updated['sd_data'].value 336 | rx_flags = updated['sd_flags'].value 337 | rx_timestamp = updated['sd_timestamp'].value 338 | 339 | # Check types and shapes 340 | assert_equal((len(self.args.channels), 2, 2), rx_data.shape) 341 | assert_equal(np.float32, rx_data.dtype) 342 | assert_equal((len(self.args.channels), 2), rx_flags.shape) 343 | assert_equal(np.uint8, rx_flags.dtype) 344 | np.testing.assert_equal(0, rx_data[..., 1]) # Should be real only 345 | 346 | rx_power = rx_data[:, 0, 0] 347 | rx_saturated = rx_data[:, 1, 0] 348 | 349 | # Check calculations 350 | ts_unix = (spectrum + 0.5 * spectra_per_stats) * self.ticks_between_spectra \ 351 | / self.adc_sample_rate + 111111111.0 352 | np.testing.assert_allclose(ts_unix * 100.0, rx_timestamp) 353 | 354 | index = np.s_[:, spectrum : spectrum + spectra_per_stats] 355 | frame_data = expected_data[index] 356 | frame_weight = expected_weight[index] 357 | weight_sum = np.sum(frame_weight, axis=1) 358 | power = np.sum(frame_data.astype(np.float64)**2, axis=2) # Sum real+imag 359 | saturated = (frame_data == -128) | (frame_data == 127) 360 | saturated = np.logical_or.reduce(saturated, axis=2) # Combine real+imag 361 | saturated = saturated.astype(np.float64) 362 | # Average over time. Can't use np.average because it complains if 363 | # weights sum to zero instead of giving a NaN. 364 | with np.errstate(divide='ignore', invalid='ignore'): 365 | power = np.sum(power * frame_weight, axis=1) / weight_sum 366 | saturated = np.sum(saturated * frame_weight, axis=1) / weight_sum 367 | power = np.where(weight_sum, power, 0) 368 | saturated = np.where(weight_sum, saturated, 0) 369 | np.testing.assert_allclose(power, rx_power) 370 | np.testing.assert_allclose(saturated, rx_saturated) 371 | flags = np.where(weight_sum, 0, DATA_LOST) 372 | np.testing.assert_equal(flags, rx_flags[:, 0]) 373 | 374 | spectrum += spectra_per_stats 375 | 376 | async def test_stream_end(self) -> None: 377 | """Stream ends with an end-of-stream""" 378 | await self._test_stream(True, True) 379 | 380 | async def test_stream_no_end(self) -> None: 381 | """Stream ends with a stop request""" 382 | await self._test_stream(False, True) 383 | 384 | async def test_stream_no_write(self) -> None: 385 | """Stream with only statistics, no output file""" 386 | await self._test_stream(True, False) 387 | -------------------------------------------------------------------------------- /katsdpbfingest/bf_ingest_server.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import os.path 4 | import logging 5 | import socket 6 | import contextlib 7 | import argparse 8 | import asyncio 9 | import concurrent.futures 10 | from typing import Optional, Callable # noqa: F401 11 | 12 | import h5py 13 | import numpy as np 14 | 15 | import aiokatcp 16 | from aiokatcp import FailReply, Sensor 17 | import spead2 18 | 19 | import katsdpservices 20 | import katsdptelstate 21 | from katsdptelstate.endpoint import endpoints_to_str 22 | 23 | from ._bf_ingest import Session, SessionConfig, ReceiverCounters 24 | from . import utils, file_writer 25 | from .utils import Range 26 | import katsdpbfingest 27 | 28 | 29 | _logger = logging.getLogger(__name__) 30 | 31 | 32 | def _warn_if_positive(value: float) -> aiokatcp.Sensor.Status: 33 | """Status function for sensors that count problems""" 34 | return Sensor.Status.WARN if value > 0 else Sensor.Status.NOMINAL 35 | 36 | 37 | def _config_from_telstate(telstate: katsdptelstate.TelescopeState, 38 | config: SessionConfig, 39 | attr_name: str, telstate_name: str = None) -> None: 40 | """Populate a SessionConfig from telstate entries. 41 | 42 | Parameters 43 | ---------- 44 | telstate 45 | Telescope state with views for the CBF stream 46 | config 47 | Configuration object to populate 48 | attr_name 49 | Attribute name to set in `config` 50 | telstate_name 51 | Name to look up in `telstate` (defaults to `attr_name`) 52 | """ 53 | if telstate_name is None: 54 | telstate_name = attr_name 55 | value = telstate[telstate_name] 56 | _logger.info('Setting %s to %s from telstate', attr_name, value) 57 | setattr(config, attr_name, value) 58 | 59 | 60 | def create_session_config(args: argparse.Namespace) -> SessionConfig: 61 | """Creates a SessionConfig object for a :class:`CaptureServer`. 62 | 63 | Note that this function makes blocking calls to telstate. The returned 64 | config has no filename. 65 | 66 | Parameters 67 | ---------- 68 | args 69 | Command-line arguments. See :class:`CaptureServer`. 70 | """ 71 | config = SessionConfig('') # Real filename supplied later 72 | config.max_packet = args.max_packet 73 | config.buffer_size = args.buffer_size 74 | if args.interface is not None: 75 | config.interface_address = katsdpservices.get_interface_address(args.interface) 76 | config.ibv = args.ibv 77 | if args.affinity: 78 | config.disk_affinity = args.affinity[0] 79 | config.network_affinity = args.affinity[1] 80 | if args.direct_io: 81 | config.direct = True 82 | 83 | # Load external config from telstate 84 | telstate = utils.cbf_telstate_view(args.telstate, args.stream_name) 85 | _config_from_telstate(telstate, config, 'channels', 'n_chans') 86 | _config_from_telstate(telstate, config, 'channels_per_heap', 'n_chans_per_substream') 87 | for name in ['ticks_between_spectra', 'spectra_per_heap', 'sync_time', 88 | 'bandwidth', 'center_freq', 'scale_factor_timestamp']: 89 | _config_from_telstate(telstate, config, name) 90 | 91 | # Set up batching to get 32MB per slice 92 | config.heaps_per_slice_time = max(1, 2**25 // (config.channels * config.spectra_per_heap * 2)) 93 | # 256MB of buffer 94 | config.ring_slots = 8 95 | 96 | # Check that the requested channel range is valid. 97 | all_channels = Range(0, config.channels) 98 | if args.channels is None: 99 | args.channels = all_channels 100 | channels_per_endpoint = config.channels // len(args.cbf_spead) 101 | if not args.channels.isaligned(channels_per_endpoint): 102 | raise ValueError( 103 | '--channels is not aligned to multiples of {}'.format(channels_per_endpoint)) 104 | if not args.channels.issubset(all_channels): 105 | raise ValueError( 106 | '--channels does not fit inside range {}'.format(all_channels)) 107 | 108 | # Update for selected channel range 109 | channel_shift = (args.channels.start + args.channels.stop - config.channels) / 2 110 | config.center_freq += channel_shift * config.bandwidth / config.channels 111 | config.bandwidth = config.bandwidth * len(args.channels) / config.channels 112 | config.channels = len(args.channels) 113 | config.channel_offset = args.channels.start 114 | 115 | endpoint_range = np.s_[args.channels.start // channels_per_endpoint: 116 | args.channels.stop // channels_per_endpoint] 117 | endpoints = args.cbf_spead[endpoint_range] 118 | for endpoint in endpoints: 119 | config.add_endpoint(socket.gethostbyname(endpoint.host), endpoint.port) 120 | config.endpoints_str = endpoints_to_str(endpoints) 121 | if args.stats is not None: 122 | config.set_stats_endpoint(args.stats.host, args.stats.port) 123 | config.stats_int_time = args.stats_int_time 124 | if args.stats_interface is not None: 125 | config.stats_interface_address = \ 126 | katsdpservices.get_interface_address(args.stats_interface) 127 | 128 | return config 129 | 130 | 131 | def session_factory(config: SessionConfig) -> Session: 132 | """Thin wrapper around :class:`_bf_ingest.Session` constructor to ease mocking.""" 133 | return Session(config) 134 | 135 | 136 | class _CaptureSession: 137 | """Object encapsulating a co-routine that runs for a single capture session 138 | (from ``capture-init`` to end of stream or ``capture-done``). 139 | 140 | Parameters 141 | ---------- 142 | config 143 | Configuration generated by :meth:`create_session_config` 144 | telstate 145 | Telescope state (optional) 146 | stream_name 147 | Name of the beamformer stream being captured 148 | update_counters 149 | Called once a second with progress counters 150 | loop 151 | IO Loop for the coroutine 152 | 153 | Attributes 154 | ---------- 155 | filename : :class:`str` or ``None`` 156 | Filename of the HDF5 file written 157 | _telstate : :class:`katsdptelstate.TelescopeState` 158 | Telescope state interface, if any 159 | _loop : :class:`asyncio.AbstractEventLoop` 160 | Event loop passed to the constructor 161 | _session : :class:`katsdpbfingest._bf_ingest.Session` 162 | C++-driven capture session 163 | _run_future : :class:`asyncio.Task` 164 | Task for the coroutine that waits for the C++ code and finalises 165 | """ 166 | def __init__(self, config: SessionConfig, telstate: katsdptelstate.TelescopeState, 167 | stream_name: str, update_counters: Callable[[ReceiverCounters], None], 168 | loop: asyncio.AbstractEventLoop) -> None: 169 | self._start_time = time.time() 170 | self._loop = loop 171 | self._telstate = telstate 172 | self.filename = config.filename 173 | self.stream_name = stream_name 174 | self.update_counters = update_counters 175 | self._config = config 176 | self._session = session_factory(config) 177 | self._run_future = loop.create_task(self._run()) 178 | 179 | def _write_metadata(self) -> None: 180 | telstate = self._telstate 181 | view = utils.cbf_telstate_view(telstate, self.stream_name) 182 | try: 183 | sync_time = view['sync_time'] 184 | scale_factor_timestamp = view['scale_factor_timestamp'] 185 | first_timestamp = sync_time + self._session.first_timestamp / scale_factor_timestamp 186 | except KeyError: 187 | _logger.warn('Failed to get timestamp conversion items, so skipping metadata') 188 | return 189 | # self._start_time should always be earlier, except when a clock is wrong. 190 | start_time = min(first_timestamp, self._start_time) 191 | h5file = h5py.File(self.filename, 'r+') 192 | with contextlib.closing(h5file): 193 | file_writer.set_telescope_state(h5file, telstate, start_timestamp=start_time) 194 | if self.stream_name is not None: 195 | data_group = h5file['/Data'] 196 | data_group.attrs['stream_name'] = self.stream_name 197 | data_group.attrs['channel_offset'] = self._config.channel_offset 198 | 199 | async def _run(self) -> None: 200 | with concurrent.futures.ThreadPoolExecutor(1) as pool: 201 | try: 202 | self.update_counters(self._session.counters) 203 | # Just passing self._session.join causes an exception in Python 204 | # 3.5 because iscoroutinefunction doesn't work on functions 205 | # defined by extensions. Hence the lambda. 206 | join_future = self._loop.run_in_executor(pool, lambda: self._session.join()) 207 | # Update the sensors once per second until termination 208 | while not (await asyncio.wait([join_future], timeout=1.0))[0]: 209 | self.update_counters(self._session.counters) 210 | await join_future # To re-raise any exception 211 | counters = self._session.counters 212 | self.update_counters(counters) 213 | if counters["katsdpbfingest.data_heaps"] > 0 and self.filename is not None: 214 | # Write the metadata to file 215 | self._write_metadata() 216 | _logger.info( 217 | 'Capture complete, %d heaps, of which %d dropped', 218 | counters["katsdpbfingest.total_heaps"], 219 | counters["katsdpbfingest.total_heaps"] - counters["katsdpbfingest.data_heaps"]) 220 | except Exception: 221 | _logger.error("Capture threw exception", exc_info=True) 222 | 223 | async def stop(self, force: bool = True) -> None: 224 | """Shut down the stream and wait for the session to end. This 225 | is a coroutine. 226 | 227 | If `force` is False, it will wait until the stream stops on its 228 | own in response to a stop heap. 229 | """ 230 | if force: 231 | self._session.stop_stream() 232 | await self._run_future 233 | 234 | @property 235 | def counters(self) -> ReceiverCounters: 236 | return self._session.counters 237 | 238 | 239 | class CaptureServer: 240 | """Beamformer capture. This contains all the core functionality of the 241 | katcp device server, without depending on katcp. It is split like this 242 | to facilitate unit testing. 243 | 244 | Parameters 245 | ---------- 246 | args 247 | Command-line arguments. The following arguments are must be present, although 248 | some of them can be ``None``. Refer to the script for documentation of 249 | these options. 250 | 251 | - cbf_spead 252 | - file_base 253 | - buffer 254 | - affinity 255 | - telstate 256 | - stream_name 257 | - stats 258 | - stats_int_time 259 | - stats_interface 260 | 261 | loop 262 | IO Loop for running coroutines 263 | 264 | Attributes 265 | ---------- 266 | capturing : :class:`bool` 267 | Whether a capture session is in progress. Note that a session is 268 | considered to be in progress until explicitly stopped with 269 | :class:`stop_capture`, even if the stream has terminated. 270 | _args : :class:`argparse.Namespace` 271 | Command-line arguments passed to constructor 272 | _loop : :class:`asyncio.AbstractEventLoop` 273 | IO Loop passed to constructor 274 | _capture : :class:`_CaptureSession` 275 | Current capture session, or ``None`` if not capturing 276 | _config : :class:`katsdpbfingest.bf_ingest.SessionConfig` 277 | Configuration, with the filename to be filled in on capture-init 278 | """ 279 | def __init__(self, args: argparse.Namespace, loop: asyncio.AbstractEventLoop) -> None: 280 | self._args = args 281 | self._loop = loop 282 | self._capture = None # type: Optional[_CaptureSession] 283 | self._config = create_session_config(args) 284 | 285 | @property 286 | def capturing(self): 287 | return self._capture is not None 288 | 289 | async def start_capture(self, capture_block_id: str) -> str: 290 | """Start capture, if not already in progress. 291 | 292 | This is a co-routine. 293 | """ 294 | if self._capture is None: 295 | if self._args.file_base is not None: 296 | basename = '{}_{}.h5'.format(capture_block_id, self._args.stream_name) 297 | self._config.filename = os.path.join(self._args.file_base, basename) 298 | else: 299 | self._config.filename = None 300 | self._capture = _CaptureSession( 301 | self._config, self._args.telstate, self._args.stream_name, 302 | self.update_counters, self._loop) 303 | return self._capture.filename 304 | 305 | async def stop_capture(self, force: bool = True) -> None: 306 | """Stop capture, if currently running. This is a co-routine.""" 307 | if self._capture is not None: 308 | capture = self._capture 309 | await capture.stop(force) 310 | # Protect against a concurrent stop and start changing to a new 311 | # capture. 312 | if self._capture is capture: 313 | self._capture = None 314 | 315 | def update_counters(self, counters: ReceiverCounters) -> None: 316 | pass # Implemented by subclass 317 | 318 | @property 319 | def counters(self) -> ReceiverCounters: 320 | if self._capture is not None: 321 | return self._capture.counters 322 | else: 323 | return ReceiverCounters() 324 | 325 | 326 | class KatcpCaptureServer(CaptureServer, aiokatcp.DeviceServer): 327 | """katcp device server for beamformer capture. 328 | 329 | Parameters 330 | ---------- 331 | args : :class:`argparse.Namespace` 332 | Command-line arguments (see :class:`CaptureServer`). 333 | The following additional arguments are required: 334 | 335 | host 336 | Hostname to bind to ('' for none) 337 | port 338 | Port number to bind to 339 | loop : :class:`asyncio.AbstractEventLoop` 340 | IO Loop for running coroutines 341 | """ 342 | 343 | VERSION = 'bf-ingest-1.0' 344 | BUILD_STATE = 'katsdpbfingest-' + katsdpbfingest.__version__ 345 | 346 | def __init__(self, args: argparse.Namespace, loop: asyncio.AbstractEventLoop) -> None: 347 | CaptureServer.__init__(self, args, loop) 348 | aiokatcp.DeviceServer.__init__(self, args.host, args.port, loop=loop) 349 | sensors = [ 350 | Sensor(int, "input-heaps-total", 351 | "Number of payload heaps received from CBF in this session " 352 | "(prometheus: counter)", 353 | initial_status=Sensor.Status.NOMINAL), 354 | Sensor(int, "input-bytes-total", 355 | "Number of payload bytes received from CBF in this session " 356 | "(prometheus: counter)", 357 | initial_status=Sensor.Status.NOMINAL), 358 | Sensor(int, "input-missing-heaps-total", 359 | "Number of heaps we expected but never saw " 360 | "(prometheus: counter)", 361 | initial_status=Sensor.Status.NOMINAL, 362 | status_func=_warn_if_positive), 363 | Sensor(int, "input-too-old-heaps-total", 364 | "Number of heaps rejected because they arrived too late " 365 | "(prometheus: counter)", 366 | initial_status=Sensor.Status.NOMINAL, 367 | status_func=_warn_if_positive), 368 | Sensor(int, "input-incomplete-heaps-total", 369 | "Number of heaps rejected due to missing packets " 370 | "(prometheus: counter)", 371 | initial_status=Sensor.Status.NOMINAL, 372 | status_func=_warn_if_positive), 373 | Sensor(int, "input-metadata-heaps-total", 374 | "Number of heaps that do not contain data " 375 | "(prometheus: counter)", 376 | initial_status=Sensor.Status.NOMINAL), 377 | Sensor(int, "input-bad-timestamp-heaps-total", 378 | "Number of heaps rejected due to bad timestamp " 379 | "(prometheus: counter)", 380 | initial_status=Sensor.Status.NOMINAL, 381 | status_func=_warn_if_positive), 382 | Sensor(int, "input-bad-channel-heaps-total", 383 | "Number of heaps rejected due to bad channel offset " 384 | "(prometheus: counter)", 385 | initial_status=Sensor.Status.NOMINAL, 386 | status_func=_warn_if_positive), 387 | Sensor(int, "input-bad-length-heaps-total", 388 | "Number of heaps rejected due to bad payload length " 389 | "(prometheus: counter)", 390 | initial_status=Sensor.Status.NOMINAL, 391 | status_func=_warn_if_positive), 392 | Sensor(int, "input-packets-total", 393 | "Total number of packets received (prometheus: counter)", 394 | initial_status=Sensor.Status.NOMINAL), 395 | Sensor(int, "input-batches-total", 396 | "Number of batches of packets processed (prometheus: counter)", 397 | initial_status=Sensor.Status.NOMINAL), 398 | Sensor(int, "input-max-batch", 399 | "Maximum number of packets processed in a batch (prometheus: gauge)", 400 | initial_status=Sensor.Status.NOMINAL) 401 | ] 402 | for sensor in sensors: 403 | self.sensors.add(sensor) 404 | 405 | def update_counters(self, counters: ReceiverCounters) -> None: 406 | timestamp = time.time() 407 | # Map sensor name to counter name 408 | sensors = { 409 | 'input-bytes-total': 'katsdpbfingest.bytes', 410 | 'input-packets-total': 'packets', 411 | 'input-batches-total': 'batches', 412 | 'input-heaps-total': 'katsdpbfingest.data_heaps', 413 | 'input-too-old-heaps-total': 'too_old_heaps', 414 | 'input-incomplete-heaps-total': 'incomplete_heaps_evicted', 415 | 'input-metadata-heaps-total': 'katsdpbfingest.metadata_heaps', 416 | 'input-bad-timestamp-heaps-total': 'katsdpbfingest.bad_timestamp_heaps', 417 | 'input-bad-channel-heaps-total': 'katsdpbfingest.bad_channel_heaps', 418 | 'input-bad-length-heaps-total': 'katsdpbfingest.bad_length_heaps', 419 | 'input-max-batch': 'max_batch' 420 | } 421 | for sensor_name, counter_name in sensors.items(): 422 | sensor = self.sensors[sensor_name] 423 | value = counters[counter_name] 424 | sensor.set_value(value, timestamp=timestamp) 425 | self.sensors['input-missing-heaps-total'].set_value( 426 | counters["katsdpbfingest.total_heaps"] - counters["katsdpbfingest.data_heaps"], 427 | timestamp=timestamp 428 | ) 429 | 430 | async def request_capture_init(self, ctx, capture_block_id: str) -> None: 431 | """Start capture to file.""" 432 | if self.capturing: 433 | raise FailReply('already capturing') 434 | if self._args.file_base is not None: 435 | stat = os.statvfs(self._args.file_base) 436 | if stat.f_bavail / stat.f_blocks < 0.05: 437 | raise FailReply('less than 5% disk space free on {}'.format( 438 | os.path.abspath(self._args.file_base))) 439 | await self.start_capture(capture_block_id) 440 | 441 | async def request_capture_done(self, ctx) -> None: 442 | """Stop a capture that is in progress.""" 443 | if not self.capturing: 444 | raise FailReply('not capturing') 445 | await self.stop_capture() 446 | 447 | async def stop(self, cancel: bool = True) -> None: 448 | await self.stop_capture() 449 | await aiokatcp.DeviceServer.stop(self, cancel) 450 | 451 | stop.__doc__ = aiokatcp.DeviceServer.stop.__doc__ 452 | 453 | 454 | def parse_args(args=None, namespace=None): 455 | """Parse command-line arguments. 456 | 457 | Any arguments are forwarded to :meth:`katsdpservices.ArgumentParser.parse_args`. 458 | """ 459 | defaults = SessionConfig('') 460 | parser = katsdpservices.ArgumentParser( 461 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 462 | parser.add_argument( 463 | '--cbf-spead', type=katsdptelstate.endpoint.endpoint_list_parser(7148), 464 | default=':7148', metavar='ENDPOINTS', 465 | help=('endpoints to listen for CBF SPEAD stream (including multicast IPs). ' 466 | '[[+]][:port].')) 467 | parser.add_argument( 468 | '--stream-name', type=str, metavar='NAME', 469 | help='Stream name for metadata in telstate') 470 | parser.add_argument( 471 | '--channels', type=Range.parse, metavar='A:B', 472 | help='Output channels') 473 | parser.add_argument( 474 | '--log-level', '-l', type=str, metavar='LEVEL', default=None, 475 | help='log level') 476 | parser.add_argument( 477 | '--file-base', type=str, metavar='DIR', 478 | help='write HDF5 files in this directory if given') 479 | parser.add_argument( 480 | '--affinity', type=spead2.parse_range_list, metavar='CPU,CPU', 481 | help='List of CPUs to which to bind threads') 482 | parser.add_argument( 483 | '--interface', type=str, 484 | help='Network interface for multicast subscription') 485 | parser.add_argument( 486 | '--direct-io', action='store_true', 487 | help='Use Direct I/O VFD for writing the file') 488 | parser.add_argument( 489 | '--ibv', action='store_true', 490 | help='Use libibverbs when possible') 491 | parser.add_argument( 492 | '--buffer-size', type=int, metavar='BYTES', default=defaults.buffer_size, 493 | help='Network buffer size [%(default)s]') 494 | parser.add_argument( 495 | '--max-packet', type=int, metavar='BYTES', default=defaults.max_packet, 496 | help='Maximum packet size (UDP payload) [%(default)s]') 497 | parser.add_argument( 498 | '--stats', type=katsdptelstate.endpoint.endpoint_parser(7149), metavar='ENDPOINT', 499 | help='Send statistics to a signal display server at this address') 500 | parser.add_argument( 501 | '--stats-int-time', type=float, default=1.0, metavar='SECONDS', 502 | help='Interval between sending statistics to the signal displays') 503 | parser.add_argument( 504 | '--stats-interface', type=str, 505 | help='Network interface for signal display stream') 506 | parser.add_aiomonitor_arguments() 507 | parser.add_argument('--port', '-p', type=int, default=2050, help='katcp host port') 508 | parser.add_argument('--host', '-a', type=str, default='', help='katcp host address') 509 | args = parser.parse_args(args, namespace) 510 | if args.affinity and len(args.affinity) < 2: 511 | parser.error('At least 2 CPUs must be specified for --affinity') 512 | if args.telstate is None: 513 | parser.error('--telstate is required') 514 | if args.stream_name is None: 515 | parser.error('--stream-name is required') 516 | return args 517 | 518 | 519 | __all__ = ['CaptureServer', 'KatcpCaptureServer', 'parse_args'] 520 | --------------------------------------------------------------------------------