├── .gitattributes ├── .github └── workflows │ ├── build.yml │ ├── build_sync_container.yml │ ├── lint.yaml │ └── rebase_checker.yaml ├── .gitignore ├── COPYRIGHT ├── Dockerfile ├── LICENSE ├── README.rst ├── doc ├── .gitignore ├── SConscript ├── conf.py ├── doxygen.conf.in ├── index.rst ├── lsst.alert.packet │ └── index.rst └── manifest.yaml ├── python └── lsst │ └── alert │ └── packet │ ├── __init__.py │ ├── bin │ ├── simulateAlerts.py │ ├── syncAllSchemasToRegistry.py │ └── validateAvroRoundTrip.py │ ├── io.py │ ├── schema.py │ ├── schema │ ├── 3 │ │ └── 0 │ │ │ ├── lsst.v3_0.alert.avsc │ │ │ ├── lsst.v3_0.diaForcedSource.avsc │ │ │ ├── lsst.v3_0.diaNondetectionLimit.avsc │ │ │ ├── lsst.v3_0.diaObject.avsc │ │ │ ├── lsst.v3_0.diaSource.avsc │ │ │ ├── lsst.v3_0.ssObject.avsc │ │ │ └── sample_data │ │ │ ├── alert.json │ │ │ ├── fakeAlert.avro │ │ │ └── generate.py │ ├── 4 │ │ └── 0 │ │ │ ├── lsst.v4_0.alert.avsc │ │ │ ├── lsst.v4_0.diaForcedSource.avsc │ │ │ ├── lsst.v4_0.diaNondetectionLimit.avsc │ │ │ ├── lsst.v4_0.diaObject.avsc │ │ │ ├── lsst.v4_0.diaSource.avsc │ │ │ ├── lsst.v4_0.ssObject.avsc │ │ │ └── sample_data │ │ │ ├── alert.json │ │ │ ├── fakeAlert.avro │ │ │ └── generate.py │ ├── 5 │ │ └── 0 │ │ │ ├── lsst.v5_0.alert.avsc │ │ │ ├── lsst.v5_0.diaForcedSource.avsc │ │ │ ├── lsst.v5_0.diaNondetectionLimit.avsc │ │ │ ├── lsst.v5_0.diaObject.avsc │ │ │ ├── lsst.v5_0.diaSource.avsc │ │ │ ├── lsst.v5_0.ssObject.avsc │ │ │ └── sample_data │ │ │ ├── alert.json │ │ │ ├── fakeAlert.avro │ │ │ └── generate.py │ ├── 6 │ │ ├── 0 │ │ │ ├── lsst.v6_0.alert.avsc │ │ │ ├── lsst.v6_0.diaForcedSource.avsc │ │ │ ├── lsst.v6_0.diaNondetectionLimit.avsc │ │ │ ├── lsst.v6_0.diaObject.avsc │ │ │ ├── lsst.v6_0.diaSource.avsc │ │ │ ├── lsst.v6_0.ssObject.avsc │ │ │ └── sample_data │ │ │ │ ├── alert.json │ │ │ │ ├── fakeAlert.avro │ │ │ │ └── generate.py │ │ └── 1 │ │ │ ├── lsst.v6_1.alert.avsc │ │ │ ├── lsst.v6_1.diaForcedSource.avsc │ │ │ ├── lsst.v6_1.diaNondetectionLimit.avsc │ │ │ ├── lsst.v6_1.diaObject.avsc │ │ │ ├── lsst.v6_1.diaSource.avsc │ │ │ ├── lsst.v6_1.ssObject.avsc │ │ │ └── sample_data │ │ │ ├── alert.json │ │ │ ├── fakeAlert.avro │ │ │ └── generate.py │ ├── 7 │ │ ├── 0 │ │ │ ├── lsst.v7_0.alert.avsc │ │ │ ├── lsst.v7_0.diaForcedSource.avsc │ │ │ ├── lsst.v7_0.diaNondetectionLimit.avsc │ │ │ ├── lsst.v7_0.diaObject.avsc │ │ │ ├── lsst.v7_0.diaSource.avsc │ │ │ ├── lsst.v7_0.ssObject.avsc │ │ │ └── sample_data │ │ │ │ ├── alert.json │ │ │ │ ├── fakeAlert.avro │ │ │ │ └── generate.py │ │ ├── 1 │ │ │ ├── lsst.v7_1.alert.avsc │ │ │ ├── lsst.v7_1.diaForcedSource.avsc │ │ │ ├── lsst.v7_1.diaNondetectionLimit.avsc │ │ │ ├── lsst.v7_1.diaObject.avsc │ │ │ ├── lsst.v7_1.diaSource.avsc │ │ │ ├── lsst.v7_1.ssObject.avsc │ │ │ └── sample_data │ │ │ │ ├── alert.json │ │ │ │ ├── fakeAlert.avro │ │ │ │ └── generate.py │ │ ├── 2 │ │ │ ├── lsst.v7_2.alert.avsc │ │ │ ├── lsst.v7_2.diaForcedSource.avsc │ │ │ ├── lsst.v7_2.diaNondetectionLimit.avsc │ │ │ ├── lsst.v7_2.diaObject.avsc │ │ │ ├── lsst.v7_2.diaSource.avsc │ │ │ ├── lsst.v7_2.ssObject.avsc │ │ │ └── sample_data │ │ │ │ ├── alert.json │ │ │ │ ├── fakeAlert.avro │ │ │ │ └── generate.py │ │ ├── 3 │ │ │ ├── lsst.v7_3.alert.avsc │ │ │ ├── lsst.v7_3.diaForcedSource.avsc │ │ │ ├── lsst.v7_3.diaNondetectionLimit.avsc │ │ │ ├── lsst.v7_3.diaObject.avsc │ │ │ ├── lsst.v7_3.diaSource.avsc │ │ │ ├── lsst.v7_3.ssObject.avsc │ │ │ └── sample_data │ │ │ │ ├── alert.json │ │ │ │ ├── fakeAlert.avro │ │ │ │ └── generate.py │ │ └── 4 │ │ │ ├── lsst.v7_4.alert.avsc │ │ │ ├── lsst.v7_4.diaForcedSource.avsc │ │ │ ├── lsst.v7_4.diaNondetectionLimit.avsc │ │ │ ├── lsst.v7_4.diaObject.avsc │ │ │ ├── lsst.v7_4.diaSource.avsc │ │ │ ├── lsst.v7_4.ssObject.avsc │ │ │ └── sample_data │ │ │ ├── alert.json │ │ │ ├── fakeAlert.avro │ │ │ └── generate.py │ ├── examples │ │ ├── sample_data │ │ │ ├── simple.json │ │ │ └── simplest.json │ │ ├── simple.avsc │ │ └── simplest.avsc │ └── latest.txt │ ├── schemaRegistry.py │ ├── simulate.py │ └── updateSchema.py ├── schema ├── script └── docker-tag.sh ├── setup.cfg ├── setup.py ├── test ├── __init__.py ├── test_io.py ├── test_schema.py ├── test_schemaRegistry.py ├── test_schemas.py └── test_wheel_install.sh └── ups ├── alert_packet.table └── eupspkg.cfg.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | *.avro binary 2 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: [push] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | 7 | strategy: 8 | matrix: 9 | python-version: ["3.10", "3.11"] 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Set up Python ${{ matrix.python-version }} 14 | uses: actions/setup-python@v4 15 | with: 16 | python-version: ${{ matrix.python-version }} 17 | - name: Install yaml needed for build 18 | run: | 19 | pip install pyyaml 20 | - name: Build and install wheel 21 | run: | 22 | pip install virtualenv 23 | test/test_wheel_install.sh 24 | - name: Install pytest and fastavro (needed by tests) 25 | run: | 26 | pip install --upgrade pip 27 | pip install pytest fastavro 28 | - name: Ensure we have a usable version installed 29 | run: | 30 | pip install -e . 31 | - name: run tests 32 | run: | 33 | pytest test 34 | -------------------------------------------------------------------------------- /.github/workflows/build_sync_container.yml: -------------------------------------------------------------------------------- 1 | name: build_sync_container 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | tags: [ "*" ] 7 | pull_request: 8 | branches: [ main ] 9 | 10 | jobs: 11 | build_image: 12 | runs-on: ubuntu-latest 13 | 14 | # Only do Docker builds of tagged releases and pull requests from ticket 15 | # branches. This will still trigger on pull requests from untrusted 16 | # repositories whose branch names match our tickets/* branch convention, 17 | # but in this case the build will fail with an error since the secret 18 | # won't be set. 19 | if: > 20 | startsWith(github.ref, 'refs/tags/') 21 | || startsWith(github.head_ref, 'tickets/') 22 | 23 | steps: 24 | - uses: actions/checkout@v2 25 | 26 | - name: Define the Docker tag 27 | id: vars 28 | run: echo ::set-output name=tag::$(script/docker-tag.sh "$GITHUB_REF") 29 | 30 | - name: Print the tag 31 | id: print 32 | run: echo ${{ steps.vars.outputs.tag }} 33 | 34 | - name: Set up Docker Buildx 35 | uses: docker/setup-buildx-action@v1 36 | 37 | - name: Cache Docker layers 38 | uses: actions/cache@v2 39 | with: 40 | path: /tmp/.buildx-cache 41 | key: ${{ runner.os }}-buildx-${{ github.sha }} 42 | restore-keys: 43 | ${{ runner.os }}-buildx- 44 | 45 | - name: Log in to Docker Hub 46 | uses: docker/login-action@v1 47 | with: 48 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 49 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 50 | 51 | - name: Build and push 52 | uses: docker/build-push-action@v2 53 | with: 54 | context: . 55 | push: true 56 | tags: lsstdm/lsst_alert_packet:${{ steps.vars.outputs.tag }} 57 | cache-from: type=local,src=/tmp/.buildx-cache 58 | cache-to: type=local,dest=/tmp/.buildx-cache-new 59 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | call-workflow: 11 | uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main 12 | -------------------------------------------------------------------------------- /.github/workflows/rebase_checker.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Check that 'main' is not merged into the development branch 3 | 4 | on: pull_request 5 | 6 | jobs: 7 | call-workflow: 8 | uses: lsst/rubin_workflows/.github/workflows/rebase_checker.yaml@main 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | __pycache__/ 3 | *ipynb_checkpoints/ 4 | *.egg-info 5 | build/ 6 | dist/ 7 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright 2016, 2018-2019 University of Washington 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11.6-slim-bullseye as base-image 2 | 3 | # Create a Python virtual environment 4 | ENV VIRTUAL_ENV=/opt/venv 5 | RUN python -m venv $VIRTUAL_ENV 6 | ENV PATH="$VIRTUAL_ENV/bin:$PATH" 7 | 8 | # Put the latest pip and setuptools in the virtualenv 9 | RUN pip install --upgrade --no-cache-dir pip setuptools wheel 10 | 11 | COPY . /app 12 | WORKDIR /app 13 | 14 | # Install package 15 | RUN pip install --no-cache-dir . 16 | 17 | ENTRYPOINT ["sh", "-c"] 18 | CMD "syncAllSchemasToRegistry.py --help" 19 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ################# 2 | alert_packet 3 | ################# 4 | 5 | This package provides information about, and utilties for working with, LSST alerts in `Apache Avro`_ format. 6 | In particular, it includes: 7 | 8 | - Alert schemas; 9 | - Examples of alert contents; 10 | - Utilities code for working with alerts. 11 | 12 | Refer to `DMTN-093`_ for more information on the LSST alert format. 13 | 14 | .. _Apache Avro: https://avro.apache.org 15 | .. _DMTN-093: https://dmtn-093.lsst.io 16 | 17 | Schemas 18 | ======= 19 | 20 | Alert schemas are located in the ``schema`` directory. 21 | 22 | Schemas are filed according to their version number, following a ``MAJOR.MINOR`` scheme. 23 | We maintain ``FORWARD_TRANSITIVE`` compatibility within a major version, per the `Confluent compatibility model`_. 24 | However, for all schemas to exist in the same registry this is not enforced and the schema registry compatibility is set 25 | to ``NONE``. The latest version of the schema may always be found in ``schema/latest.txt``. 26 | 27 | Within the schema registry, versions are registered with space for two digits in the minor version. Example : 7.1 28 | becomes 701. Version 13.10 would be 1310. 29 | 30 | .. _Confluent compatibility model: https://docs.confluent.io/current/schema-registry/docs/avro.html#forward-compatibility 31 | 32 | Adding a new schema 33 | ------------------- 34 | 35 | Steps to update the alert schema (for example, when the APDB schema is updated). 36 | 37 | * Decide what the new schema version will be, following the guidelines given in `DMTN-093 `_, referring to the current version number directories in ``python/lsst/alert/packet/schema``. 38 | * Checkout the ticket branch for your schema changes. 39 | * Update the default ``schema_root`` kwarg in ``python/lsst/alert/packet/schemaRegistry.py:from_filesystem()`` to your new schema version number. 40 | 41 | * New schemas are built from the apdb. Any changes to a field should be done in ``sdm_schemas/yml/apdb.yaml``, and any changes to what is included/excluded should be made in ``updateSchema.py``. 42 | * To update the schema, you must have the path to the apdb.yaml file and have chosen a version number. If the directory for your version does not already exist, ``updateSchema.py`` will create it. 43 | 44 | * set up ``sdm_schemas`` and ``cd python/lsst/alert/packet/``. 45 | * run ``python updateSchema.py $SDM_SCHEMAS_DIR/yml/apdb.yaml ./schema "6.0"`` All Generated files do not need to be altered. 46 | * Navigate to the new schema. Copy in the previous ``lsst.vX_X.alert.avsc`` file and ``lsst.vX_X.diaNondetectionLimit.avsc``. 47 | * Within the two copied files, update ``"namespace": "lsst.vX_X",`` line at the top of each ``*.avsc`` file to the new version. 48 | * Update the contents of those avro schema files to reflect the new schema. 49 | 50 | * Update the sample alert packet in ``sample_data``: 51 | 52 | * Copy ``sample_data`` from previous schema into the new version's ``X_X`` directory. 53 | * Update ``alert.json`` to reflect the new schema. 54 | * Change the ``schema_root`` and ``get_by_version`` parameters in ``generate.py`` to your new version number. 55 | * Run ``python generate.py`` to produce a new ``fakeAlert.avro`` file with data filled in from the updated json file above and using the new schema files you made earlier. 56 | 57 | * Update the files ``*.avsc`` and ``*.json`` files in ``examples/`` to reflect the new schema. 58 | * Update the contents of ``latest.txt`` to your new schema version number. 59 | 60 | * Add all of your new and updated files to ``git``, commit them with a message that includes the new schema version number and why it was incremented, and push your new branch. 61 | * Test your changes with ap_association and ap_verify (these steps may be more involved if your ticket branch impacts multiple packets): 62 | 63 | * Clone a local copy of `ap_association `_ and cd into that directory. 64 | * Setup ap_association with ``setup -kr .`` (``-k`` to "keep" your previously setup ``alert_packet``). 65 | * Confirm that your modified alert_packet is still setup: ``eups list -s alert_packet`` should show a ``LOCAL:`` directory being setup. 66 | * Run ``scons`` to confirm that your updated schema works with the ``ap_association`` tests. 67 | * Run at least one of the `ap_verify datasets `_ to confirm that your new alert schema works with the broader tests in ap_verify. 68 | * Update the schema registry image for the alert stream. 69 | 70 | Update Schema Registry 71 | ---------------------- 72 | 73 | When a pull request is made for a change to ``alert_packet``, a docker container image is created which needs to be synced to 74 | the alert stream. The new docker image is present at `lsstdm/lsst_alert_packet ` on dockerhub. 75 | The image will appear with the same tag as the ticket branch you created to update ``alert_packet``. 76 | 77 | * Go to ``alert-stream-schema-registry`` in the ``alert-stream-broker`` charts folder in the ``phalanx`` github repository. 78 | * In the `values.yaml `_ file, 79 | update the `tag` in `schemaSync` to match the docker image tag. If the image is not updating in argo, use the image digest to force an update. 80 | * If you have access to ``argocd``, click ``sync`` at the top. A full sync is needed to load the registry with the schemas. If the application will not sync, a full restart may be required of the `alert-stream` application. 81 | * If you do not have access, alert whoever is in charge of the ``alert-stream`` so that they can sync the registry. 82 | 83 | More thorough instructions for updating the schema registry can be found in `DMTN-214 ` 84 | 85 | Example Alert Contents 86 | ====================== 87 | 88 | Example alert contents are provided in the ``sample_data`` directories included with the corresponding schema. 89 | 90 | Utility Code 91 | ============ 92 | 93 | All code is written in Python, and uses the `fastavro`_ library. 94 | Simulation code also requires `NumPy`_. 95 | Both of these may be installed using standard tooling (pip, Conda, etc). 96 | 97 | Although this package contains multiple versions of the alert schema, this library code is only written and tested using the latest version (``schema/latest``) at present. 98 | Future versions of this package should offer wider compatibility. 99 | 100 | Installation 101 | ------------ 102 | 103 | Using pip 104 | ^^^^^^^^^ 105 | 106 | The name of the package is `lsst-alert-packet`:: 107 | 108 | $ pip install lsst-alert-packet 109 | 110 | Using EUPS 111 | ^^^^^^^^^^ 112 | 113 | This package may be managed using `EUPS`_. 114 | Assuming EUPS is available on your system, simply:: 115 | 116 | $ git clone https://github.com/lsst/alert_packet.git 117 | $ setup -r alert_packet 118 | 119 | .. _EUPS: https://github.com/RobertLuptonTheGood/eups/ 120 | 121 | Library 122 | ------- 123 | 124 | The ``lsst.alert.packet`` Python package provides a suite of routines for working with alerts in the Avro format. 125 | 126 | Command Line 127 | ------------ 128 | 129 | ``validateAvroRoundTrip.py`` demonstrates round-tripping a simple alert through the Avro system. 130 | Sample data is provided in the ``schema/latest/sample_data/alert.json`` file, or an alternative may be provided on the command line. 131 | Optionally, the path to binary data files to be included in the packet as “postage stamp” images may be provided. 132 | If the ``--print`` flag is given, the alert contents are printed to screen for sanity checking. 133 | 134 | ``simulateAlerts.py`` writes simulated alert packets to disk in Avro format. 135 | The resultant data is schema compliant, but the simulations are not intended to be realistic: packets are populated with pseudorandom numbers. 136 | The number of visits per year (equivalent to the number of previous DIASources observed for each alert) and the number of alerts to simulate may be specified on the command line. 137 | Thus:: 138 | 139 | $ simulateAlerts.py --visits-per-year=100 --num-alerts=10 ./output_file.avro 140 | 141 | .. _fastavro: https://fastavro.readthedocs.io/en/latest/ 142 | .. _NumPy: http://www.numpy.org 143 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | # Doxygen products 2 | html 3 | xml 4 | *.tag 5 | *.inc 6 | doxygen.conf 7 | 8 | # Sphinx products 9 | _build 10 | py-api 11 | -------------------------------------------------------------------------------- /doc/SConscript: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | from lsst.sconsUtils import scripts 3 | scripts.BasicSConscript.doc() 4 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | """Sphinx configuration file for an LSST stack package. 2 | 3 | This configuration only affects single-package Sphinx documentation builds. 4 | For more information, see: 5 | https://developer.lsst.io/stack/building-single-package-docs.html 6 | """ 7 | 8 | from documenteer.conf.pipelinespkg import * 9 | 10 | 11 | project = "alert.packet" 12 | html_theme_options["logotext"] = project 13 | html_title = project 14 | html_short_title = project 15 | -------------------------------------------------------------------------------- /doc/doxygen.conf.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/doc/doxygen.conf.in -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | ############################# 2 | alert.packet documentation preview 3 | ############################# 4 | 5 | .. This page is for local development only. It isn't published to pipelines.lsst.io. 6 | 7 | .. Link the index pages of package and module documentation directions (listed in manifest.yaml). 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | 12 | lsst.alert.packet/index 13 | -------------------------------------------------------------------------------- /doc/lsst.alert.packet/index.rst: -------------------------------------------------------------------------------- 1 | .. py:currentmodule:: lsst.alert.packet 2 | 3 | .. _lsst.alert.packet: 4 | 5 | ############ 6 | lsst.alert.packet 7 | ############ 8 | 9 | .. Paragraph that describes what this Python module does and links to related modules and frameworks. 10 | 11 | .. .. _lsst.alert.packet-using: 12 | 13 | .. Using lsst.alert.packet 14 | .. ================== 15 | 16 | .. toctree linking to topics related to using the module's APIs. 17 | 18 | .. .. toctree:: 19 | .. :maxdepth: 1 20 | 21 | .. _lsst.alert.packet-contributing: 22 | 23 | Contributing 24 | ============ 25 | 26 | ``lsst.alert.packet`` is developed at https://github.com/lsst/alert_packet. 27 | You can find Jira issues for this module under the `alert_packet `_ component. 28 | 29 | .. If there are topics related to developing this module (rather than using it), link to this from a toctree placed here. 30 | 31 | .. .. toctree:: 32 | .. :maxdepth: 1 33 | 34 | .. _lsst.alert.packet-command-line-taskref: 35 | 36 | Python API reference 37 | ==================== 38 | 39 | .. automodapi:: lsst.alert.packet 40 | :no-main-docstr: 41 | :no-inheritance-diagram: 42 | -------------------------------------------------------------------------------- /doc/manifest.yaml: -------------------------------------------------------------------------------- 1 | # Documentation manifest. 2 | 3 | # List of names of Python modules in this package. 4 | # For each module there is a corresponding module doc subdirectory. 5 | modules: 6 | - "lsst.alert.packet" 7 | 8 | # Name of the static content directories (subdirectories of `_static`). 9 | # Static content directories are usually named after the package. 10 | # Most packages do not need a static content directory (leave commented out). 11 | # statics: 12 | # - "_static/alert.packet" 13 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | from .io import * 23 | from .schema import * 24 | from .schemaRegistry import * 25 | from .simulate import * 26 | from .updateSchema import * 27 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/bin/simulateAlerts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # This file is part of alert_packet. 4 | # 5 | # Developed for the LSST Data Management System. 6 | # This product includes software developed by the LSST Project 7 | # (https://www.lsst.org). 8 | # See the COPYRIGHT file at the top-level directory of this distribution 9 | # for details of code ownership. 10 | # 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | # 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | import argparse 25 | 26 | import fastavro # noqa: F401 27 | 28 | import lsst.alert.packet 29 | 30 | 31 | def parse_args(): 32 | parser = argparse.ArgumentParser() 33 | # Default value based on LSE-81 34 | parser.add_argument('--visits-per-year', type=int, default=1056//10, 35 | help='Number of visits per year') 36 | parser.add_argument('--num-alerts', type=int, default=10, 37 | help='Number of simulated alert packets to generate') 38 | parser.add_argument('output_filename', type=str, 39 | help="File to which to write alerts") 40 | return parser.parse_args() 41 | 42 | 43 | def main(): 44 | args = parse_args() 45 | 46 | schema = lsst.alert.packet.Schema.from_file() 47 | arrayCount = {'prvDiaSources': args.visits_per_year, 48 | 'prvDiaForcedSources': args.visits_per_year//12, 49 | 'prvDiaNondetectionLimits': 0} 50 | alerts = [lsst.alert.packet.simulate_alert(schema.definition, 51 | keepNull=['ssObject'], 52 | arrayCount=arrayCount) 53 | for _ in range(args.num_alerts)] 54 | 55 | for alert in alerts: 56 | assert schema.validate(alert) 57 | 58 | with open(args.output_filename, "wb") as f: 59 | schema.store_alerts(f, alerts) 60 | 61 | with open(args.output_filename, "rb") as f: 62 | writer_schema, loaded_alerts = schema.retrieve_alerts(f) 63 | 64 | assert schema == writer_schema 65 | for a1, a2 in zip(alerts, loaded_alerts): 66 | assert a1 == a2 67 | 68 | 69 | if __name__ == '__main__': 70 | main() 71 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/bin/syncAllSchemasToRegistry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # This file is part of alert_packet. 4 | # 5 | # Developed for the LSST Data Management System. 6 | # This product includes software developed by the LSST Project 7 | # (https://www.lsst.org). 8 | # See the COPYRIGHT file at the top-level directory of this distribution 9 | # for details of code ownership. 10 | # 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | # 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | import argparse 25 | import json 26 | import fastavro 27 | import requests 28 | 29 | import lsst.alert.packet 30 | 31 | 32 | def parse_args(): 33 | parser = argparse.ArgumentParser() 34 | parser.add_argument( 35 | "--schema-registry-url", 36 | type=str, 37 | default="http://alert-schemas.localhost", 38 | help="URL of a Confluent Schema Registry service", 39 | ) 40 | parser.add_argument( 41 | "--subject", 42 | type=str, 43 | default="alert-packet", 44 | help="Schema Registry subject name to use", 45 | ) 46 | return parser.parse_args() 47 | 48 | 49 | def upload_schemas(registry_url, subject, schema_registry): 50 | """Parse schema registry and upload all schemas. 51 | """ 52 | for schema_id in schema_registry.known_ids: 53 | schema = schema_registry.get_by_id(schema_id) 54 | normalized_schema = fastavro.schema.to_parsing_canonical_form( 55 | schema.definition) 56 | confluent_schema = {"version": schema_id, 57 | "id": schema_id, "schema": normalized_schema} 58 | payload = json.dumps(confluent_schema) 59 | headers = {"Content-Type": "application/vnd.schemaregistry.v1+json"} 60 | url = f"{registry_url}/subjects/{subject}/versions" 61 | print(f"uploading schema to {url}") 62 | response = requests.post(url=url, data=payload, headers=headers) 63 | response.raise_for_status() 64 | print(f"done, status={response.status_code}") 65 | print(f"response text={response.text}") 66 | 67 | 68 | def clear_schema_registry_for_import(registry_url, subject): 69 | """Delete schemas in the registry and then remake it in import mode""" 70 | # Define the URLs 71 | url_mode = f"{registry_url}/mode/{subject}" 72 | url_schemas = f"{registry_url}/subjects/{subject}" 73 | url_schema_versions = f"{registry_url}/subjects/{subject}/versions" 74 | response = requests.get(url_schema_versions) 75 | 76 | # Schema registry must be empty to put it in import mode. If it exists, 77 | # remove it and remake the schema. If not, continue. 78 | if response.status_code == 200: 79 | print('The schema will be deleted and remade in import mode.') 80 | response = requests.delete(url_schemas) 81 | print('Status Code:', response.status_code) 82 | print('Response Text:', response.text) 83 | else: 84 | print('The schema does not exist. Creating in import mode.') 85 | 86 | # Switch registry to import mode. 87 | headers = { 88 | 'Content-Type': 'application/json' 89 | } 90 | 91 | # Define the data to send 92 | data = { 93 | 'mode': 'IMPORT' 94 | } 95 | 96 | # Perform the PUT request 97 | response = requests.put(url_mode, headers=headers, data=json.dumps(data)) 98 | 99 | # Check the status code and response 100 | print('Status Code:', response.status_code) 101 | print('Response Text:', response.text) 102 | 103 | 104 | def close_schema_registry(registry_url, subject): 105 | """Return the schema registry from import mode to readwrite. 106 | """ 107 | data = { 108 | "mode": "READWRITE" 109 | } 110 | 111 | # Headers to specify the content type 112 | headers = { 113 | 'Content-Type': 'application/json' 114 | } 115 | 116 | url_mode = f"{registry_url}/mode/{subject}" 117 | # Send the PUT request 118 | response = requests.put(url_mode, json=data, headers=headers) 119 | print(f'Status Code: {response.status_code}') 120 | print(f'Response Text: {response.text}') 121 | 122 | 123 | def main(): 124 | args = parse_args() 125 | clear_schema_registry_for_import(args.schema_registry_url, args.subject) 126 | schema_registry = lsst.alert.packet.schemaRegistry.SchemaRegistry().all_schemas_from_filesystem() 127 | upload_schemas( 128 | args.schema_registry_url, 129 | subject=args.subject, 130 | schema_registry=schema_registry 131 | ) 132 | close_schema_registry(args.schema_registry_url, args.subject) 133 | 134 | 135 | if __name__ == "__main__": 136 | main() 137 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/bin/validateAvroRoundTrip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # This file is part of alert_packet. 4 | # 5 | # Developed for the LSST Data Management System. 6 | # This product includes software developed by the LSST Project 7 | # (https://www.lsst.org). 8 | # See the COPYRIGHT file at the top-level directory of this distribution 9 | # for details of code ownership. 10 | # 11 | # This program is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU General Public License as published by 13 | # the Free Software Foundation, either version 3 of the License, or 14 | # (at your option) any later version. 15 | # 16 | # This program is distributed in the hope that it will be useful, 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | # GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see . 23 | 24 | """Demonstrate round-tripping of alert data through Avro serialization. 25 | """ 26 | 27 | # Arguably, this should be a test, rather than an executable. 28 | 29 | import argparse 30 | import filecmp 31 | import json 32 | import os 33 | import tempfile 34 | 35 | import lsst.alert.packet 36 | 37 | # The default filename of per-schema sample alert data. 38 | SAMPLE_FILENAME = "alert.json" 39 | 40 | 41 | def schema_filename(major_version, minor_version): 42 | return f"lsst.v{major_version}_{minor_version}.alert.avsc" 43 | 44 | 45 | def check_file_round_trip(baseline, received_data): 46 | """Assert that the contents of baseline is equal to received_data. 47 | 48 | Parameters 49 | ---------- 50 | baseline : `str` 51 | The full path to a file on disk. 52 | received_data : `bytes` 53 | Raw bytes. 54 | """ 55 | with tempfile.TemporaryDirectory() as tempdir: 56 | filename = os.path.join(tempdir, "output_file") 57 | with open(filename, "wb") as f: 58 | f.write(received_data) 59 | assert filecmp.cmp(baseline, filename, shallow=False) 60 | 61 | 62 | def parse_args(): 63 | parser = argparse.ArgumentParser() 64 | parser.add_argument('--schema-version', type=str, 65 | help='Schema version to test (“latest” or MAJOR.MINOR)', default="latest") 66 | parser.add_argument('--input-data', type=str, 67 | help='Path to a file containing schema-compliant JSON Data to serialize', 68 | default=None) 69 | parser.add_argument('--cutout-difference', type=str, 70 | help='File for difference image postage stamp') 71 | parser.add_argument('--cutout-template', type=str, 72 | help='File for template image postage stamp') 73 | parser.add_argument('--print', action="store_true", 74 | help='Pretty-print alert contents') 75 | 76 | return parser.parse_args() 77 | 78 | 79 | def main(): 80 | args = parse_args() 81 | if args.schema_version == "latest": 82 | schema_major, schema_minor = lsst.alert.packet.get_latest_schema_version() 83 | else: 84 | schema_major, schema_minor = args.schema_version.split(".") 85 | schema_root = lsst.alert.packet.get_schema_path(schema_major, schema_minor) 86 | 87 | alert_schema = lsst.alert.packet.Schema.from_file( 88 | os.path.join(schema_root, 89 | schema_filename(schema_major, schema_minor)), 90 | ) 91 | if args.input_data: 92 | input_data = args.input_data 93 | else: 94 | input_data = os.path.join(schema_root, "sample_data", SAMPLE_FILENAME) 95 | with open(input_data) as f: 96 | json_data = json.load(f) 97 | 98 | # Load difference stamp if included 99 | stamp_size = 0 100 | if args.cutout_difference is not None: 101 | cutout_difference = lsst.alert.packet.load_stamp(args.cutout_difference) 102 | stamp_size += len(cutout_difference['stampData']) 103 | json_data['cutoutDifference'] = cutout_difference 104 | 105 | # Load template stamp if included 106 | if args.cutout_template is not None: 107 | cutout_template = lsst.alert.packet.load_stamp(args.cutout_template) 108 | stamp_size += len(cutout_template['stampData']) 109 | json_data['cutoutTemplate'] = cutout_template 110 | 111 | # Demonstrate round-trip through Avro serialization 112 | avro_bytes = alert_schema.serialize(json_data) 113 | message = alert_schema.deserialize(avro_bytes) 114 | 115 | # Check that postage stamps were preserved through (de)serialization 116 | if args.cutout_difference: 117 | check_file_round_trip(args.cutout_difference, 118 | message.pop('cutoutDifference')['stampData']) 119 | if args.cutout_template: 120 | check_file_round_trip(args.cutout_template, 121 | message.pop('cutoutTemplate')['stampData']) 122 | 123 | message_size = len(json.dumps(message, default=repr).encode('utf-8')) 124 | print("Size in bytes of JSON-encoded message (excl. stamps): %d" % (message_size,)) 125 | print("Size in bytes of stamps: %d" % (stamp_size,)) 126 | print("TOTAL: %d" % (stamp_size + message_size,)) 127 | print("Size in bytes of Avro-encoded message (incl. stamps): %d" % (len(avro_bytes),)) 128 | print("DIFFERENCE: %d" % (stamp_size + message_size 129 | - len(avro_bytes),)) 130 | 131 | # Pretty-print the received message. 132 | # Note that the that this won't be identitical to the input because: 133 | # 134 | # - Optional fields that were omitted in the input are now present, but 135 | # populated with nulls; 136 | # - Precision has been lost on the floats. 137 | if args.print: 138 | print(json.dumps(message, sort_keys=True, indent=4, default=repr)) 139 | 140 | 141 | if __name__ == "__main__": 142 | main() 143 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/io.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | """Routines for loading data from files. 23 | """ 24 | import itertools 25 | import os.path 26 | 27 | import fastavro 28 | 29 | from .schema import Schema 30 | 31 | __all__ = ["load_stamp", "retrieve_alerts"] 32 | 33 | 34 | def load_stamp(file_path): 35 | """Load a cutout postage stamp file to include in alert. 36 | """ 37 | _, fileoutname = os.path.split(file_path) 38 | with open(file_path, mode='rb') as f: 39 | cutout_data = f.read() 40 | cutout_dict = {"fileName": fileoutname, "stampData": cutout_data} 41 | return cutout_dict 42 | 43 | 44 | def retrieve_alerts(fp, reader_schema=None): 45 | """Read alert packets from the given I/O stream. 46 | 47 | Parameters 48 | ---------- 49 | fp : derivative of `IOBase` 50 | I/O stream from which data will be read. 51 | reader_schema : `dict` or `list`, optional 52 | A schema describing the contents of the Avro packets. If not provided, 53 | the schema used when writing the alert stream will be used. 54 | 55 | Returns 56 | ------- 57 | schema : `lsst.alert.Schema` 58 | The schema with which alerts were written (which may be different from 59 | the schema being used for deserialization). 60 | records : iterable of `dict` 61 | Alert records. 62 | 63 | Raises 64 | ------ 65 | RuntimeError 66 | Raised if alert data could not be read. 67 | """ 68 | try: 69 | reader = fastavro.reader(fp, reader_schema=reader_schema.definition if reader_schema else None) 70 | except Exception as e: 71 | raise RuntimeError(f"failed to find alert data in " 72 | f"{fp.name if hasattr(fp, 'name') else 'stream'}") from e 73 | 74 | # Peek at one record so that reader.writer_schema is populated, since it 75 | # gets loaded lazily. If you don't do this, then reader.writer_schema would 76 | # be None, which means Schema(reader.writer_schema) would get an empty 77 | # value. 78 | # 79 | # It would be simpler to do something like 'records = list(reader)', but 80 | # that would require loading all the records into memory in one gulp. Since 81 | # alert files can be huge, even terabytes, the extra complexity here is 82 | # worth it. 83 | try: 84 | first_record = next(reader) 85 | records = itertools.chain([first_record], reader) 86 | except StopIteration: 87 | # The file has zero records in it. It might still have a schema, 88 | # though. 89 | records = [] 90 | writer_schema = Schema(reader.writer_schema) 91 | return writer_schema, records 92 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/lsst.v3_0.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v3_0", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "sample avro alert schema v3.0", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v3_0.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v3_0.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v3_0.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v3_0.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v3_0.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v3_0.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/lsst.v3_0.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v3_0", 3 | "name": "diaForcedSource", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "diaForcedSourceId", "type": "long"}, 7 | {"name": "ccdVisitId", "type": "long"}, 8 | {"name": "diaObjectId", "type": "long"}, 9 | {"name": "midPointTai", "type": "double"}, 10 | {"name": "filterName", "type": "string"}, 11 | {"name": "psFlux", "type": "float"}, 12 | {"name": "psFluxErr", "type": "float"}, 13 | {"name": "totFlux", "type": "float"}, 14 | {"name": "totFluxErr", "type":"float"} 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/lsst.v3_0.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v3_0", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midPointTai", "type": "double"}, 8 | {"name": "filterName", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/lsst.v3_0.diaSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v3_0", 3 | "name": "diaSource", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "diaSourceId", "type": "long"}, 7 | {"name": "ccdVisitId", "type": "long"}, 8 | {"name": "diaObjectId", "type": ["null", "long"], "default": null}, 9 | {"name": "ssObjectId", "type": ["null", "long"], "default": null}, 10 | {"name": "parentDiaSourceId", "type": ["null", "long"], "default": null}, 11 | {"name": "midPointTai", "type": "double"}, 12 | {"name": "filterName", "type": "string"}, 13 | {"name": "programId", "type": "int"}, 14 | {"name": "ra", "type": "double"}, 15 | {"name": "decl", "type": "double"}, 16 | {"name": "raErr", "type": ["null", "float"], "default": null}, 17 | {"name": "declErr", "type": ["null", "float"], "default": null}, 18 | {"name": "ra_decl_Cov", "type": ["null", "float"], "default": null}, 19 | {"name": "x", "type": "float"}, 20 | {"name": "y", "type": "float"}, 21 | {"name": "xErr", "type": ["null", "float"], "default": null}, 22 | {"name": "yErr", "type": ["null", "float"], "default": null}, 23 | {"name": "x_y_Cov", "type": ["null", "float"], "default": null}, 24 | {"name": "apFlux", "type": "float"}, 25 | {"name": "apFluxErr", "type": "float"}, 26 | {"name": "snr", "type": "float"}, 27 | {"name": "psFlux", "type": "float"}, 28 | {"name": "psFluxErr", "type": "float"}, 29 | {"name": "psRa", "type": ["null", "double"], "default": null}, 30 | {"name": "psDecl", "type": ["null", "double"], "default": null}, 31 | {"name": "psRaErr", "type": ["null", "float"], "default": null}, 32 | {"name": "psDeclErr", "type": ["null", "float"], "default": null}, 33 | {"name": "psFlux_psRa_Cov", "type": ["null", "float"], "default": null}, 34 | {"name": "psFlux_psDecl_Cov", "type": ["null", "float"], "default": null}, 35 | {"name": "psRa_psDecl_Cov", "type": ["null", "float"], "default": null}, 36 | {"name": "psLnL", "type": ["null", "float"], "default": null}, 37 | {"name": "psChi2", "type": ["null", "float"], "default": null}, 38 | {"name": "psNdata", "type": ["null", "int"], "default": null}, 39 | {"name": "trailFlux", "type": ["null", "float"], "default": null}, 40 | {"name": "trailRa", "type": ["null", "double"], "default": null}, 41 | {"name": "trailDecl", "type": ["null", "double"], "default": null}, 42 | {"name": "trailLength", "type": ["null", "float"], "default": null}, 43 | {"name": "trailAngle", "type": ["null", "float"], "default": null}, 44 | {"name": "trailFluxErr", "type": ["null", "float"], "default": null}, 45 | {"name": "trailRaErr", "type": ["null", "float"], "default": null}, 46 | {"name": "trailDeclErr", "type": ["null", "float"], "default": null}, 47 | {"name": "trailLengthErr", "type": ["null", "float"], "default": null}, 48 | {"name": "trailAngleErr", "type": ["null", "float"], "default": null}, 49 | {"name": "trailFlux_trailRa_Cov", "type": ["null", "float"], "default": null}, 50 | {"name": "trailFlux_trailDecl_Cov", "type": ["null", "float"], "default": null}, 51 | {"name": "trailFlux_trailLength_Cov", "type": ["null", "float"], "default": null}, 52 | {"name": "trailFlux_trailAngle_Cov", "type": ["null", "float"], "default": null}, 53 | {"name": "trailRa_trailDecl_Cov", "type": ["null", "float"], "default": null}, 54 | {"name": "trailRa_trailLength_Cov", "type": ["null", "float"], "default": null}, 55 | {"name": "trailRa_trailAngle_Cov", "type": ["null", "float"], "default": null}, 56 | {"name": "trailDecl_trailLength_Cov", "type": ["null", "float"], "default": null}, 57 | {"name": "trailDecl_trailAngle_Cov", "type": ["null", "float"], "default": null}, 58 | {"name": "trailLength_trailAngle_Cov", "type": ["null", "float"], "default": null}, 59 | {"name": "trailLnL", "type": ["null", "float"], "default": null}, 60 | {"name": "trailChi2", "type": ["null", "float"], "default": null}, 61 | {"name": "trailNdata", "type": ["null", "int"], "default": null}, 62 | {"name": "dipMeanFlux", "type": ["null", "float"], "default": null}, 63 | {"name": "dipFluxDiff", "type": ["null", "float"], "default": null}, 64 | {"name": "dipRa", "type": ["null", "double"], "default": null}, 65 | {"name": "dipDecl", "type": ["null", "double"], "default": null}, 66 | {"name": "dipLength", "type": ["null", "float"], "default": null}, 67 | {"name": "dipAngle", "type": ["null", "float"], "default": null}, 68 | {"name": "dipMeanFluxErr", "type": ["null", "float"], "default": null}, 69 | {"name": "dipFluxDiffErr", "type": ["null", "float"], "default": null}, 70 | {"name": "dipRaErr", "type": ["null", "float"], "default": null}, 71 | {"name": "dipDeclErr", "type": ["null", "float"], "default": null}, 72 | {"name": "dipLengthErr", "type": ["null", "float"], "default": null}, 73 | {"name": "dipAngleErr", "type": ["null", "float"], "default": null}, 74 | {"name": "dipMeanFlux_dipFluxDiff_Cov", "type": ["null", "float"], "default": null}, 75 | {"name": "dipMeanFlux_dipRa_Cov", "type": ["null", "float"], "default": null}, 76 | {"name": "dipMeanFlux_dipDecl_Cov", "type": ["null", "float"], "default": null}, 77 | {"name": "dipMeanFlux_dipLength_Cov", "type": ["null", "float"], "default": null}, 78 | {"name": "dipMeanFlux_dipAngle_Cov", "type": ["null", "float"], "default": null}, 79 | {"name": "dipFluxDiff_dipRa_Cov", "type": ["null", "float"], "default": null}, 80 | {"name": "dipFluxDiff_dipDecl_Cov", "type": ["null", "float"], "default": null}, 81 | {"name": "dipFluxDiff_dipLength_Cov", "type": ["null", "float"], "default": null}, 82 | {"name": "dipFluxDiff_dipAngle_Cov", "type": ["null", "float"], "default": null}, 83 | {"name": "dipRa_dipDecl_Cov", "type": ["null", "float"], "default": null}, 84 | {"name": "dipRa_dipLength_Cov", "type": ["null", "float"], "default": null}, 85 | {"name": "dipRa_dipAngle_Cov", "type": ["null", "float"], "default": null}, 86 | {"name": "dipDecl_dipLength_Cov", "type": ["null", "float"], "default": null}, 87 | {"name": "dipDecl_dipAngle_Cov", "type": ["null", "float"], "default": null}, 88 | {"name": "dipLength_dipAngle_Cov", "type": ["null", "float"], "default": null}, 89 | {"name": "dipLnL", "type": ["null", "float"], "default": null}, 90 | {"name": "dipChi2", "type": ["null", "float"], "default": null}, 91 | {"name": "dipNdata", "type": ["null", "int"], "default": null}, 92 | {"name": "totFlux", "aliases": ["fpFlux"], "type": ["null", "float"], "default": null}, 93 | {"name": "totFluxErr", "type": ["null", "float"], "default": null}, 94 | {"name": "diffFlux", "type": ["null", "float"], "default": null}, 95 | {"name": "diffFluxErr", "type": ["null", "float"], "default": null}, 96 | {"name": "fpBkgd", "aliases": ["fpSky"], "type": ["null", "float"], "default": null}, 97 | {"name": "fpBkgdErr", "type": ["null", "float"], "default": null}, 98 | {"name": "ixx", "type": ["null", "float"], "default": null}, 99 | {"name": "iyy", "type": ["null", "float"], "default": null}, 100 | {"name": "ixy", "type": ["null", "float"], "default": null}, 101 | {"name": "ixxErr", "type": ["null", "float"], "default": null}, 102 | {"name": "iyyErr", "type": ["null", "float"], "default": null}, 103 | {"name": "ixyErr", "type": ["null", "float"], "default": null}, 104 | {"name": "ixx_iyy_Cov", "type": ["null", "float"], "default": null}, 105 | {"name": "ixx_ixy_Cov", "type": ["null", "float"], "default": null}, 106 | {"name": "iyy_ixy_Cov", "type": ["null", "float"], "default": null}, 107 | {"name": "ixxPSF", "type": ["null", "float"], "default": null}, 108 | {"name": "iyyPSF", "type": ["null", "float"], "default": null}, 109 | {"name": "ixyPSF", "type": ["null", "float"], "default": null}, 110 | {"name": "extendedness", "type": ["null", "float"], "default": null}, 111 | {"name": "spuriousness", "type": ["null", "float"], "default": null}, 112 | {"name": "flags", "type": "long"} 113 | ] 114 | } 115 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/lsst.v3_0.ssObject.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v3_0", 3 | "name": "ssObject", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ssObjectId", "type": "long"}, 7 | {"name": "q", "type": ["null", "double"], "default": null}, 8 | {"name": "e", "type": ["null", "double"], "default": null}, 9 | {"name": "i", "type": ["null", "double"], "default": null}, 10 | {"name": "lan", "type": ["null", "double"], "default": null}, 11 | {"name": "aop", "type": ["null", "double"], "default": null}, 12 | {"name": "M", "type": ["null", "double"], "default": null}, 13 | {"name": "epoch", "type": ["null", "double"], "default": null}, 14 | {"name": "qSigma", "type": ["null", "double"], "default": null}, 15 | {"name": "eSigma", "type": ["null", "double"], "default": null}, 16 | {"name": "iSigma", "type": ["null", "double"], "default": null}, 17 | {"name": "lanSigma", "type": ["null", "double"], "default": null}, 18 | {"name": "aopSigma", "type": ["null", "double"], "default": null}, 19 | {"name": "MSigma", "type": ["null", "double"], "default": null}, 20 | {"name": "epochSigma", "type": ["null", "double"], "default": null}, 21 | {"name": "q_e_Cov", "type": ["null", "double"], "default": null}, 22 | {"name": "q_i_Cov", "type": ["null", "double"], "default": null}, 23 | {"name": "q_lan_Cov", "type": ["null", "double"], "default": null}, 24 | {"name": "q_aop_Cov", "type": ["null", "double"], "default": null}, 25 | {"name": "q_M_Cov", "type": ["null", "double"], "default": null}, 26 | {"name": "q_epoch_Cov", "type": ["null", "double"], "default": null}, 27 | {"name": "e_i_Cov", "type": ["null", "double"], "default": null}, 28 | {"name": "e_lan_Cov", "type": ["null", "double"], "default": null}, 29 | {"name": "e_aop_Cov", "type": ["null", "double"], "default": null}, 30 | {"name": "e_M_Cov", "type": ["null", "double"], "default": null}, 31 | {"name": "e_epoch_Cov", "type": ["null", "double"], "default": null}, 32 | {"name": "i_lan_Cov", "type": ["null", "double"], "default": null}, 33 | {"name": "i_aop_Cov", "type": ["null", "double"], "default": null}, 34 | {"name": "i_M_Cov", "type": ["null", "double"], "default": null}, 35 | {"name": "i_epoch_Cov", "type": ["null", "double"], "default": null}, 36 | {"name": "lan_aop_Cov", "type": ["null", "double"], "default": null}, 37 | {"name": "lan_M_Cov", "type": ["null", "double"], "default": null}, 38 | {"name": "lan_epoch_Cov", "type": ["null", "double"], "default": null}, 39 | {"name": "aop_M_Cov", "type": ["null", "double"], "default": null}, 40 | {"name": "aop_epoch_Cov", "type": ["null", "double"], "default": null}, 41 | {"name": "M_epoch_Cov", "type": ["null", "double"], "default": null}, 42 | {"name": "arc", "type": ["null", "float"], "default": null}, 43 | {"name": "orbFitLnL", "type": ["null", "float"], "default": null}, 44 | {"name": "orbFitChi2", "type": ["null", "float"], "default": null}, 45 | {"name": "orbFitNdata", "type": ["null", "int"], "default": null}, 46 | {"name": "MOID1", "type": ["null", "float"], "default": null}, 47 | {"name": "MOID2", "type": ["null", "float"], "default": null}, 48 | {"name": "moidLon1", "type": ["null", "double"], "default": null}, 49 | {"name": "moidLon2", "type": ["null", "double"], "default": null}, 50 | {"name": "uH", "type": ["null", "float"], "default": null}, 51 | {"name": "uHErr", "type": ["null", "float"], "default": null}, 52 | {"name": "uG1", "type": ["null", "float"], "default": null}, 53 | {"name": "uG1Err", "type": ["null", "float"], "default": null}, 54 | {"name": "uG2", "type": ["null", "float"], "default": null}, 55 | {"name": "uG2Err", "type": ["null", "float"], "default": null}, 56 | {"name": "gH", "type": ["null", "float"], "default": null}, 57 | {"name": "gHErr", "type": ["null", "float"], "default": null}, 58 | {"name": "gG1", "type": ["null", "float"], "default": null}, 59 | {"name": "gG1Err", "type": ["null", "float"], "default": null}, 60 | {"name": "gG2", "type": ["null", "float"], "default": null}, 61 | {"name": "gG2Err", "type": ["null", "float"], "default": null}, 62 | {"name": "rH", "type": ["null", "float"], "default": null}, 63 | {"name": "rHErr", "type": ["null", "float"], "default": null}, 64 | {"name": "rG1", "type": ["null", "float"], "default": null}, 65 | {"name": "rG1Err", "type": ["null", "float"], "default": null}, 66 | {"name": "rG2", "type": ["null", "float"], "default": null}, 67 | {"name": "rG2Err", "type": ["null", "float"], "default": null}, 68 | {"name": "iH", "type": ["null", "float"], "default": null}, 69 | {"name": "iHErr", "type": ["null", "float"], "default": null}, 70 | {"name": "iG1", "type": ["null", "float"], "default": null}, 71 | {"name": "iG1Err", "type": ["null", "float"], "default": null}, 72 | {"name": "iG2", "type": ["null", "float"], "default": null}, 73 | {"name": "iG2Err", "type": ["null", "float"], "default": null}, 74 | {"name": "zH", "type": ["null", "float"], "default": null}, 75 | {"name": "zHErr", "type": ["null", "float"], "default": null}, 76 | {"name": "zG1", "type": ["null", "float"], "default": null}, 77 | {"name": "zG1Err", "type": ["null", "float"], "default": null}, 78 | {"name": "zG2", "type": ["null", "float"], "default": null}, 79 | {"name": "zG2Err", "type": ["null", "float"], "default": null}, 80 | {"name": "yH", "type": ["null", "float"], "default": null}, 81 | {"name": "yHErr", "type": ["null", "float"], "default": null}, 82 | {"name": "yG1", "type": ["null", "float"], "default": null}, 83 | {"name": "yG1Err", "type": ["null", "float"], "default": null}, 84 | {"name": "yG2", "type": ["null", "float"], "default": null}, 85 | {"name": "yG2Err", "type": ["null", "float"], "default": null}, 86 | {"name": "flags", "type": "long"} 87 | ] 88 | } 89 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "ccdVisitId": 111111, 8 | "midPointTai": 1480360995, 9 | "filterName": "r", 10 | "programId": 1, 11 | "ra": 351.570546978, 12 | "decl": 0.126243049656, 13 | "raSigma": 0.00028, 14 | "declSigma": 0.00028, 15 | "ra_decl_Cov": 0.00029, 16 | "x": 112.1, 17 | "y": 121.1, 18 | "xSigma": 1.2, 19 | "ySigma": 1.1, 20 | "x_y_Cov": 1.2, 21 | "apFlux": 1241.0, 22 | "apFluxErr": 12.0, 23 | "snr": 41.1, 24 | "psFlux": 1241.0, 25 | "psFluxErr": 12.0, 26 | "flags": 0 27 | }, 28 | "prvDiaSources": [ 29 | { 30 | "diaSourceId": 281323062375219198, 31 | "ccdVisitId": 111111, 32 | "midPointTai": 1480360995, 33 | "filterName": "r", 34 | "programId": 1, 35 | "ra": 351.570546978, 36 | "decl": 0.126243049656, 37 | "raSigma": 0.00028, 38 | "declSigma": 0.00028, 39 | "ra_decl_Cov": 0.00029, 40 | "x": 112.1, 41 | "y": 121.1, 42 | "xSigma": 1.2, 43 | "ySigma": 1.1, 44 | "x_y_Cov": 1.2, 45 | "apFlux": 1241.0, 46 | "apFluxErr": 12.0, 47 | "snr": 41.1, 48 | "psFlux": 1241.0, 49 | "psFluxErr": 12.0, 50 | "flags": 0 51 | }, 52 | { 53 | "diaSourceId": 281323062375219199, 54 | "ccdVisitId": 111111, 55 | "midPointTai": 1480360995, 56 | "filterName": "r", 57 | "programId": 1, 58 | "ra": 351.570546978, 59 | "decl": 0.126243049656, 60 | "raSigma": 0.00028, 61 | "declSigma": 0.00028, 62 | "ra_decl_Cov": 0.00029, 63 | "x": 112.1, 64 | "y": 121.1, 65 | "xSigma": 1.2, 66 | "ySigma": 1.1, 67 | "x_y_Cov": 1.2, 68 | "apFlux": 1241.0, 69 | "apFluxErr": 12.0, 70 | "snr": 41.1, 71 | "psFlux": 1241.0, 72 | "psFluxErr": 12.0, 73 | "flags": 0 74 | } 75 | ], 76 | "diaObject": { 77 | "diaObjectId": 281323062375219201, 78 | "ra": 351.570546978, 79 | "decl": 0.126243049656, 80 | "raSigma": 0.00028, 81 | "declSigma": 0.00028, 82 | "ra_decl_Cov": 0.00029, 83 | "radecTai": 1480360995, 84 | "pmRa": 0.00013, 85 | "pmDecl": 0.00014, 86 | "parallax": 2.124124, 87 | "pmRaSigma": 0.00013, 88 | "pmDeclSigma": 0.00013, 89 | "parallaxSigma": 0.00013, 90 | "pmRa_pmDecl_Cov": 0.00013, 91 | "pmRa_parallax_Cov": 0.00013, 92 | "pmDecl_parallax_Cov": 0.00013, 93 | "pmParallaxLnL": 0.00013, 94 | "pmParallaxChi2": 0.00013, 95 | "pmParallaxNdata": 1214, 96 | "flags": 0 97 | }, 98 | "ssObject": { 99 | "ssObjectId":5364546, 100 | "q": 6654.14, 101 | "e": 636.121, 102 | "i": 5436.2344, 103 | "lan": 54325.34, 104 | "aop": 344243.3, 105 | "M": 131.1241, 106 | "epoch": 134141, 107 | "arc":2.124124, 108 | "orbFitLnL": 1343141.0341, 109 | "orbFitChi2": 1341421.2414, 110 | "orbFitNdata": 1214, 111 | "MOID1": 3141.0, 112 | "MOID2": 23432.423, 113 | "moidLon1": 2143.213, 114 | "moidLon2": 3142.23123, 115 | "uH": 13231.231, 116 | "uHErr": 13213.213, 117 | "uG1": 32131.312, 118 | "uG1Err": 31232.2132, 119 | "uG2": 231.2313, 120 | "uG2Err": 23132.231, 121 | "flags": 0 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/3/0/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/3/0/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | 5 | with open("alert.json", "r") as f: 6 | data = json.load(f) 7 | 8 | schema = packet.SchemaRegistry.from_filesystem().get_by_version("3.0") 9 | with open("fakeAlert.avro", "wb") as f: 10 | schema.store_alerts(f, [data]) 11 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/lsst.v4_0.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v4_0", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "sample avro alert schema v4.0", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v4_0.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v4_0.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v4_0.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v4_0.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v4_0.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v4_0.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/lsst.v4_0.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v4_0", 3 | "name": "diaForcedSource", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "diaForcedSourceId", "type": "long"}, 7 | {"name": "ccdVisitId", "type": "long"}, 8 | {"name": "diaObjectId", "type": "long"}, 9 | {"name": "midPointTai", "type": "double"}, 10 | {"name": "filterName", "type": "string"}, 11 | {"name": "psFlux", "type": "float"}, 12 | {"name": "psFluxErr", "type": "float"}, 13 | {"name": "totFlux", "type": "float"}, 14 | {"name": "totFluxErr", "type":"float"} 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/lsst.v4_0.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v4_0", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midPointTai", "type": "double"}, 8 | {"name": "filterName", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/lsst.v4_0.diaSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v4_0", 3 | "name": "diaSource", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "diaSourceId", "type": "long"}, 7 | {"name": "ccdVisitId", "type": "long"}, 8 | {"name": "diaObjectId", "type": ["null", "long"], "default": null}, 9 | {"name": "ssObjectId", "type": ["null", "long"], "default": null}, 10 | {"name": "parentDiaSourceId", "type": ["null", "long"], "default": null}, 11 | {"name": "midPointTai", "type": "double"}, 12 | {"name": "filterName", "type": "string"}, 13 | {"name": "programId", "type": "int"}, 14 | {"name": "ra", "type": "double"}, 15 | {"name": "decl", "type": "double"}, 16 | {"name": "raErr", "type": ["null", "float"], "default": null}, 17 | {"name": "declErr", "type": ["null", "float"], "default": null}, 18 | {"name": "ra_decl_Cov", "type": ["null", "float"], "default": null}, 19 | {"name": "x", "type": "float"}, 20 | {"name": "y", "type": "float"}, 21 | {"name": "xErr", "type": ["null", "float"], "default": null}, 22 | {"name": "yErr", "type": ["null", "float"], "default": null}, 23 | {"name": "x_y_Cov", "type": ["null", "float"], "default": null}, 24 | {"name": "apFlux", "type": "float"}, 25 | {"name": "apFluxErr", "type": "float"}, 26 | {"name": "snr", "type": "float"}, 27 | {"name": "psFlux", "type": "float"}, 28 | {"name": "psFluxErr", "type": "float"}, 29 | {"name": "psRa", "type": ["null", "double"], "default": null}, 30 | {"name": "psDecl", "type": ["null", "double"], "default": null}, 31 | {"name": "psRaErr", "type": ["null", "float"], "default": null}, 32 | {"name": "psDeclErr", "type": ["null", "float"], "default": null}, 33 | {"name": "psFlux_psRa_Cov", "type": ["null", "float"], "default": null}, 34 | {"name": "psFlux_psDecl_Cov", "type": ["null", "float"], "default": null}, 35 | {"name": "psRa_psDecl_Cov", "type": ["null", "float"], "default": null}, 36 | {"name": "psLnL", "type": ["null", "float"], "default": null}, 37 | {"name": "psChi2", "type": ["null", "float"], "default": null}, 38 | {"name": "psNdata", "type": ["null", "int"], "default": null}, 39 | {"name": "trailFlux", "type": ["null", "float"], "default": null}, 40 | {"name": "trailRa", "type": ["null", "double"], "default": null}, 41 | {"name": "trailDecl", "type": ["null", "double"], "default": null}, 42 | {"name": "trailLength", "type": ["null", "float"], "default": null}, 43 | {"name": "trailAngle", "type": ["null", "float"], "default": null}, 44 | {"name": "trailFluxErr", "type": ["null", "float"], "default": null}, 45 | {"name": "trailRaErr", "type": ["null", "float"], "default": null}, 46 | {"name": "trailDeclErr", "type": ["null", "float"], "default": null}, 47 | {"name": "trailLengthErr", "type": ["null", "float"], "default": null}, 48 | {"name": "trailAngleErr", "type": ["null", "float"], "default": null}, 49 | {"name": "trailFlux_trailRa_Cov", "type": ["null", "float"], "default": null}, 50 | {"name": "trailFlux_trailDecl_Cov", "type": ["null", "float"], "default": null}, 51 | {"name": "trailFlux_trailLength_Cov", "type": ["null", "float"], "default": null}, 52 | {"name": "trailFlux_trailAngle_Cov", "type": ["null", "float"], "default": null}, 53 | {"name": "trailRa_trailDecl_Cov", "type": ["null", "float"], "default": null}, 54 | {"name": "trailRa_trailLength_Cov", "type": ["null", "float"], "default": null}, 55 | {"name": "trailRa_trailAngle_Cov", "type": ["null", "float"], "default": null}, 56 | {"name": "trailDecl_trailLength_Cov", "type": ["null", "float"], "default": null}, 57 | {"name": "trailDecl_trailAngle_Cov", "type": ["null", "float"], "default": null}, 58 | {"name": "trailLength_trailAngle_Cov", "type": ["null", "float"], "default": null}, 59 | {"name": "trailLnL", "type": ["null", "float"], "default": null}, 60 | {"name": "trailChi2", "type": ["null", "float"], "default": null}, 61 | {"name": "trailNdata", "type": ["null", "int"], "default": null}, 62 | {"name": "dipMeanFlux", "type": ["null", "float"], "default": null}, 63 | {"name": "dipFluxDiff", "type": ["null", "float"], "default": null}, 64 | {"name": "dipRa", "type": ["null", "double"], "default": null}, 65 | {"name": "dipDecl", "type": ["null", "double"], "default": null}, 66 | {"name": "dipLength", "type": ["null", "float"], "default": null}, 67 | {"name": "dipAngle", "type": ["null", "float"], "default": null}, 68 | {"name": "dipMeanFluxErr", "type": ["null", "float"], "default": null}, 69 | {"name": "dipFluxDiffErr", "type": ["null", "float"], "default": null}, 70 | {"name": "dipRaErr", "type": ["null", "float"], "default": null}, 71 | {"name": "dipDeclErr", "type": ["null", "float"], "default": null}, 72 | {"name": "dipLengthErr", "type": ["null", "float"], "default": null}, 73 | {"name": "dipAngleErr", "type": ["null", "float"], "default": null}, 74 | {"name": "dipMeanFlux_dipFluxDiff_Cov", "type": ["null", "float"], "default": null}, 75 | {"name": "dipMeanFlux_dipRa_Cov", "type": ["null", "float"], "default": null}, 76 | {"name": "dipMeanFlux_dipDecl_Cov", "type": ["null", "float"], "default": null}, 77 | {"name": "dipMeanFlux_dipLength_Cov", "type": ["null", "float"], "default": null}, 78 | {"name": "dipMeanFlux_dipAngle_Cov", "type": ["null", "float"], "default": null}, 79 | {"name": "dipFluxDiff_dipRa_Cov", "type": ["null", "float"], "default": null}, 80 | {"name": "dipFluxDiff_dipDecl_Cov", "type": ["null", "float"], "default": null}, 81 | {"name": "dipFluxDiff_dipLength_Cov", "type": ["null", "float"], "default": null}, 82 | {"name": "dipFluxDiff_dipAngle_Cov", "type": ["null", "float"], "default": null}, 83 | {"name": "dipRa_dipDecl_Cov", "type": ["null", "float"], "default": null}, 84 | {"name": "dipRa_dipLength_Cov", "type": ["null", "float"], "default": null}, 85 | {"name": "dipRa_dipAngle_Cov", "type": ["null", "float"], "default": null}, 86 | {"name": "dipDecl_dipLength_Cov", "type": ["null", "float"], "default": null}, 87 | {"name": "dipDecl_dipAngle_Cov", "type": ["null", "float"], "default": null}, 88 | {"name": "dipLength_dipAngle_Cov", "type": ["null", "float"], "default": null}, 89 | {"name": "dipLnL", "type": ["null", "float"], "default": null}, 90 | {"name": "dipChi2", "type": ["null", "float"], "default": null}, 91 | {"name": "dipNdata", "type": ["null", "int"], "default": null}, 92 | {"name": "totFlux", "aliases": ["fpFlux"], "type": ["null", "float"], "default": null}, 93 | {"name": "totFluxErr", "type": ["null", "float"], "default": null}, 94 | {"name": "diffFlux", "type": ["null", "float"], "default": null}, 95 | {"name": "diffFluxErr", "type": ["null", "float"], "default": null}, 96 | {"name": "fpBkgd", "aliases": ["fpSky"], "type": ["null", "float"], "default": null}, 97 | {"name": "fpBkgdErr", "type": ["null", "float"], "default": null}, 98 | {"name": "ixx", "type": ["null", "float"], "default": null}, 99 | {"name": "iyy", "type": ["null", "float"], "default": null}, 100 | {"name": "ixy", "type": ["null", "float"], "default": null}, 101 | {"name": "ixxErr", "type": ["null", "float"], "default": null}, 102 | {"name": "iyyErr", "type": ["null", "float"], "default": null}, 103 | {"name": "ixyErr", "type": ["null", "float"], "default": null}, 104 | {"name": "ixx_iyy_Cov", "type": ["null", "float"], "default": null}, 105 | {"name": "ixx_ixy_Cov", "type": ["null", "float"], "default": null}, 106 | {"name": "iyy_ixy_Cov", "type": ["null", "float"], "default": null}, 107 | {"name": "ixxPSF", "type": ["null", "float"], "default": null}, 108 | {"name": "iyyPSF", "type": ["null", "float"], "default": null}, 109 | {"name": "ixyPSF", "type": ["null", "float"], "default": null}, 110 | {"name": "extendedness", "type": ["null", "float"], "default": null}, 111 | {"name": "spuriousness", "type": ["null", "float"], "default": null}, 112 | {"name": "flags", "type": "long"} 113 | ] 114 | } 115 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "ccdVisitId": 111111, 8 | "midPointTai": 1480360995, 9 | "filterName": "r", 10 | "programId": 1, 11 | "ra": 351.570546978, 12 | "decl": 0.126243049656, 13 | "raSigma": 0.00028, 14 | "declSigma": 0.00028, 15 | "ra_decl_Cov": 0.00029, 16 | "x": 112.1, 17 | "y": 121.1, 18 | "xSigma": 1.2, 19 | "ySigma": 1.1, 20 | "x_y_Cov": 1.2, 21 | "apFlux": 1241.0, 22 | "apFluxErr": 12.0, 23 | "snr": 41.1, 24 | "psFlux": 1241.0, 25 | "psFluxErr": 12.0, 26 | "flags": 0 27 | }, 28 | "prvDiaSources": [ 29 | { 30 | "diaSourceId": 281323062375219198, 31 | "ccdVisitId": 111111, 32 | "midPointTai": 1480360995, 33 | "filterName": "r", 34 | "programId": 1, 35 | "ra": 351.570546978, 36 | "decl": 0.126243049656, 37 | "raSigma": 0.00028, 38 | "declSigma": 0.00028, 39 | "ra_decl_Cov": 0.00029, 40 | "x": 112.1, 41 | "y": 121.1, 42 | "xSigma": 1.2, 43 | "ySigma": 1.1, 44 | "x_y_Cov": 1.2, 45 | "apFlux": 1241.0, 46 | "apFluxErr": 12.0, 47 | "snr": 41.1, 48 | "psFlux": 1241.0, 49 | "psFluxErr": 12.0, 50 | "flags": 0 51 | }, 52 | { 53 | "diaSourceId": 281323062375219199, 54 | "ccdVisitId": 111111, 55 | "midPointTai": 1480360995, 56 | "filterName": "r", 57 | "programId": 1, 58 | "ra": 351.570546978, 59 | "decl": 0.126243049656, 60 | "raSigma": 0.00028, 61 | "declSigma": 0.00028, 62 | "ra_decl_Cov": 0.00029, 63 | "x": 112.1, 64 | "y": 121.1, 65 | "xSigma": 1.2, 66 | "ySigma": 1.1, 67 | "x_y_Cov": 1.2, 68 | "apFlux": 1241.0, 69 | "apFluxErr": 12.0, 70 | "snr": 41.1, 71 | "psFlux": 1241.0, 72 | "psFluxErr": 12.0, 73 | "flags": 0 74 | } 75 | ], 76 | "diaObject": { 77 | "diaObjectId": 281323062375219201, 78 | "ra": 351.570546978, 79 | "decl": 0.126243049656, 80 | "raSigma": 0.00028, 81 | "declSigma": 0.00028, 82 | "ra_decl_Cov": 0.00029, 83 | "radecTai": 1480360995, 84 | "pmRa": 0.00013, 85 | "pmDecl": 0.00014, 86 | "parallax": 2.124124, 87 | "pmRaSigma": 0.00013, 88 | "pmDeclSigma": 0.00013, 89 | "parallaxSigma": 0.00013, 90 | "pmRa_pmDecl_Cov": 0.00013, 91 | "pmRa_parallax_Cov": 0.00013, 92 | "pmDecl_parallax_Cov": 0.00013, 93 | "pmParallaxLnL": 0.00013, 94 | "pmParallaxChi2": 0.00013, 95 | "pmParallaxNdata": 1214, 96 | "flags": 0 97 | }, 98 | "ssObject": { 99 | "ssObjectId":5364546, 100 | "numObs": 10, 101 | "flags": 0 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/4/0/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/4/0/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | 5 | with open("alert.json", "r") as f: 6 | data = json.load(f) 7 | 8 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v4_0.alert").get_by_version("4.0") 9 | with open("fakeAlert.avro", "wb") as f: 10 | schema.store_alerts(f, [data]) 11 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/lsst.v5_0.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v5_0", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "sample avro alert schema v5.0", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v5_0.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v5_0.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v5_0.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v5_0.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v5_0.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v5_0.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/lsst.v5_0.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v5_0", 3 | "name": "diaForcedSource", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "diaForcedSourceId", "type": "long"}, 7 | {"name": "ccdVisitId", "type": "long"}, 8 | {"name": "diaObjectId", "type": "long"}, 9 | {"name": "midpointMjdTai", "type": "double"}, 10 | {"name": "band", "type": "string"}, 11 | {"name": "psfFlux", "type": "float"}, 12 | {"name": "psfFluxErr", "type": "float"}, 13 | {"name": "scienceFlux", "type": "float"}, 14 | {"name": "scienceFluxErr", "type":"float"} 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/lsst.v5_0.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v5_0", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/lsst.v5_0.diaSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v5_0", 3 | "name": "diaSource", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "diaSourceId", "type": "long"}, 7 | {"name": "ccdVisitId", "type": "long"}, 8 | {"name": "diaObjectId", "type": ["null", "long"], "default": null}, 9 | {"name": "ssObjectId", "type": ["null", "long"], "default": null}, 10 | {"name": "parentDiaSourceId", "type": ["null", "long"], "default": null}, 11 | {"name": "midpointMjdTai", "type": "double"}, 12 | {"name": "band", "type": "string"}, 13 | {"name": "programId", "type": "int"}, 14 | {"name": "ra", "type": "double"}, 15 | {"name": "dec", "type": "double"}, 16 | {"name": "raErr", "type": ["null", "float"], "default": null}, 17 | {"name": "decErr", "type": ["null", "float"], "default": null}, 18 | {"name": "ra_dec_Cov", "type": ["null", "float"], "default": null}, 19 | {"name": "x", "type": "float"}, 20 | {"name": "y", "type": "float"}, 21 | {"name": "xErr", "type": ["null", "float"], "default": null}, 22 | {"name": "yErr", "type": ["null", "float"], "default": null}, 23 | {"name": "x_y_Cov", "type": ["null", "float"], "default": null}, 24 | {"name": "apFlux", "type": "float"}, 25 | {"name": "apFluxErr", "type": "float"}, 26 | {"name": "snr", "type": "float"}, 27 | {"name": "psfFlux", "type": "float"}, 28 | {"name": "psfFluxErr", "type": "float"}, 29 | {"name": "psfRa", "type": ["null", "double"], "default": null}, 30 | {"name": "psfDec", "type": ["null", "double"], "default": null}, 31 | {"name": "psfRaErr", "type": ["null", "float"], "default": null}, 32 | {"name": "psfDecErr", "type": ["null", "float"], "default": null}, 33 | {"name": "psfFlux_psfRa_Cov", "type": ["null", "float"], "default": null}, 34 | {"name": "psfFlux_psfDec_Cov", "type": ["null", "float"], "default": null}, 35 | {"name": "psfRa_psfDec_Cov", "type": ["null", "float"], "default": null}, 36 | {"name": "psfLnL", "type": ["null", "float"], "default": null}, 37 | {"name": "psfChi2", "type": ["null", "float"], "default": null}, 38 | {"name": "psfNdata", "type": ["null", "int"], "default": null}, 39 | {"name": "trailFlux", "type": ["null", "float"], "default": null}, 40 | {"name": "trailRa", "type": ["null", "double"], "default": null}, 41 | {"name": "trailDec", "type": ["null", "double"], "default": null}, 42 | {"name": "trailLength", "type": ["null", "float"], "default": null}, 43 | {"name": "trailAngle", "type": ["null", "float"], "default": null}, 44 | {"name": "trailFluxErr", "type": ["null", "float"], "default": null}, 45 | {"name": "trailRaErr", "type": ["null", "float"], "default": null}, 46 | {"name": "trailDecErr", "type": ["null", "float"], "default": null}, 47 | {"name": "trailLengthErr", "type": ["null", "float"], "default": null}, 48 | {"name": "trailAngleErr", "type": ["null", "float"], "default": null}, 49 | {"name": "trailFlux_trailRa_Cov", "type": ["null", "float"], "default": null}, 50 | {"name": "trailFlux_trailDec_Cov", "type": ["null", "float"], "default": null}, 51 | {"name": "trailFlux_trailLength_Cov", "type": ["null", "float"], "default": null}, 52 | {"name": "trailFlux_trailAngle_Cov", "type": ["null", "float"], "default": null}, 53 | {"name": "trailRa_trailDec_Cov", "type": ["null", "float"], "default": null}, 54 | {"name": "trailRa_trailLength_Cov", "type": ["null", "float"], "default": null}, 55 | {"name": "trailRa_trailAngle_Cov", "type": ["null", "float"], "default": null}, 56 | {"name": "trailDec_trailLength_Cov", "type": ["null", "float"], "default": null}, 57 | {"name": "trailDec_trailAngle_Cov", "type": ["null", "float"], "default": null}, 58 | {"name": "trailLength_trailAngle_Cov", "type": ["null", "float"], "default": null}, 59 | {"name": "trailLnL", "type": ["null", "float"], "default": null}, 60 | {"name": "trailChi2", "type": ["null", "float"], "default": null}, 61 | {"name": "trailNdata", "type": ["null", "int"], "default": null}, 62 | {"name": "dipoleMeanFlux", "type": ["null", "float"], "default": null}, 63 | {"name": "dipoleFluxDiff", "type": ["null", "float"], "default": null}, 64 | {"name": "dipoleRa", "type": ["null", "double"], "default": null}, 65 | {"name": "dipoleDec", "type": ["null", "double"], "default": null}, 66 | {"name": "dipoleLength", "type": ["null", "float"], "default": null}, 67 | {"name": "dipoleAngle", "type": ["null", "float"], "default": null}, 68 | {"name": "dipoleMeanFluxErr", "type": ["null", "float"], "default": null}, 69 | {"name": "dipoleFluxDiffErr", "type": ["null", "float"], "default": null}, 70 | {"name": "dipoleRaErr", "type": ["null", "float"], "default": null}, 71 | {"name": "dipoleDecErr", "type": ["null", "float"], "default": null}, 72 | {"name": "dipoleLengthErr", "type": ["null", "float"], "default": null}, 73 | {"name": "dipoleAngleErr", "type": ["null", "float"], "default": null}, 74 | {"name": "dipoleMeanFlux_dipoleFluxDiff_Cov", "type": ["null", "float"], "default": null}, 75 | {"name": "dipoleMeanFlux_dipoleRa_Cov", "type": ["null", "float"], "default": null}, 76 | {"name": "dipoleMeanFlux_dipoleDec_Cov", "type": ["null", "float"], "default": null}, 77 | {"name": "dipoleMeanFlux_dipoleLength_Cov", "type": ["null", "float"], "default": null}, 78 | {"name": "dipoleMeanFlux_dipoleAngle_Cov", "type": ["null", "float"], "default": null}, 79 | {"name": "dipoleFluxDiff_dipoleRa_Cov", "type": ["null", "float"], "default": null}, 80 | {"name": "dipoleFluxDiff_dipoleDec_Cov", "type": ["null", "float"], "default": null}, 81 | {"name": "dipoleFluxDiff_dipoleLength_Cov", "type": ["null", "float"], "default": null}, 82 | {"name": "dipoleFluxDiff_dipoleAngle_Cov", "type": ["null", "float"], "default": null}, 83 | {"name": "dipoleRa_dipoleDec_Cov", "type": ["null", "float"], "default": null}, 84 | {"name": "dipoleRa_dipoleLength_Cov", "type": ["null", "float"], "default": null}, 85 | {"name": "dipoleRa_dipoleAngle_Cov", "type": ["null", "float"], "default": null}, 86 | {"name": "dipoleDec_dipoleLength_Cov", "type": ["null", "float"], "default": null}, 87 | {"name": "dipoleDec_dipoleAngle_Cov", "type": ["null", "float"], "default": null}, 88 | {"name": "dipoleLength_dipoleAngle_Cov", "type": ["null", "float"], "default": null}, 89 | {"name": "dipoleLnL", "type": ["null", "float"], "default": null}, 90 | {"name": "dipoleChi2", "type": ["null", "float"], "default": null}, 91 | {"name": "dipoleNdata", "type": ["null", "int"], "default": null}, 92 | {"name": "scienceFlux", "aliases": ["fpFlux"], "type": ["null", "float"], "default": null}, 93 | {"name": "scienceFluxErr", "type": ["null", "float"], "default": null}, 94 | {"name": "snapDiffFlux", "type": ["null", "float"], "default": null}, 95 | {"name": "snapDiffFluxErr", "type": ["null", "float"], "default": null}, 96 | {"name": "fpBkgd", "aliases": ["fpSky"], "type": ["null", "float"], "default": null}, 97 | {"name": "fpBkgdErr", "type": ["null", "float"], "default": null}, 98 | {"name": "ixx", "type": ["null", "float"], "default": null}, 99 | {"name": "iyy", "type": ["null", "float"], "default": null}, 100 | {"name": "ixy", "type": ["null", "float"], "default": null}, 101 | {"name": "ixxErr", "type": ["null", "float"], "default": null}, 102 | {"name": "iyyErr", "type": ["null", "float"], "default": null}, 103 | {"name": "ixyErr", "type": ["null", "float"], "default": null}, 104 | {"name": "ixx_iyy_Cov", "type": ["null", "float"], "default": null}, 105 | {"name": "ixx_ixy_Cov", "type": ["null", "float"], "default": null}, 106 | {"name": "iyy_ixy_Cov", "type": ["null", "float"], "default": null}, 107 | {"name": "ixxPSF", "type": ["null", "float"], "default": null}, 108 | {"name": "iyyPSF", "type": ["null", "float"], "default": null}, 109 | {"name": "ixyPSF", "type": ["null", "float"], "default": null}, 110 | {"name": "extendedness", "type": ["null", "float"], "default": null}, 111 | {"name": "reliability", "type": ["null", "float"], "default": null}, 112 | {"name": "flags", "type": "long"} 113 | ] 114 | } 115 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "ccdVisitId": 111111, 8 | "midpointMjdTai": 1480360995, 9 | "band": "r", 10 | "programId": 1, 11 | "ra": 351.570546978, 12 | "dec": 0.126243049656, 13 | "raSigma": 0.00028, 14 | "decSigma": 0.00028, 15 | "ra_dec_Cov": 0.00029, 16 | "x": 112.1, 17 | "y": 121.1, 18 | "xSigma": 1.2, 19 | "ySigma": 1.1, 20 | "x_y_Cov": 1.2, 21 | "apFlux": 1241.0, 22 | "apFluxErr": 12.0, 23 | "snr": 41.1, 24 | "psfFlux": 1241.0, 25 | "psfFluxErr": 12.0, 26 | "flags": 0 27 | }, 28 | "prvDiaSources": [ 29 | { 30 | "diaSourceId": 281323062375219198, 31 | "ccdVisitId": 111111, 32 | "midpointMjdTai": 1480360995, 33 | "band": "r", 34 | "programId": 1, 35 | "ra": 351.570546978, 36 | "dec": 0.126243049656, 37 | "raSigma": 0.00028, 38 | "decSigma": 0.00028, 39 | "ra_dec_Cov": 0.00029, 40 | "x": 112.1, 41 | "y": 121.1, 42 | "xSigma": 1.2, 43 | "ySigma": 1.1, 44 | "x_y_Cov": 1.2, 45 | "apFlux": 1241.0, 46 | "apFluxErr": 12.0, 47 | "snr": 41.1, 48 | "psfFlux": 1241.0, 49 | "psfFluxErr": 12.0, 50 | "flags": 0 51 | }, 52 | { 53 | "diaSourceId": 281323062375219199, 54 | "ccdVisitId": 111111, 55 | "midpointMjdTai": 1480360995, 56 | "band": "r", 57 | "programId": 1, 58 | "ra": 351.570546978, 59 | "dec": 0.126243049656, 60 | "raSigma": 0.00028, 61 | "decSigma": 0.00028, 62 | "ra_dec_Cov": 0.00029, 63 | "x": 112.1, 64 | "y": 121.1, 65 | "xSigma": 1.2, 66 | "ySigma": 1.1, 67 | "x_y_Cov": 1.2, 68 | "apFlux": 1241.0, 69 | "apFluxErr": 12.0, 70 | "snr": 41.1, 71 | "psfFlux": 1241.0, 72 | "psfFluxErr": 12.0, 73 | "flags": 0 74 | } 75 | ], 76 | "diaObject": { 77 | "diaObjectId": 281323062375219201, 78 | "ra": 351.570546978, 79 | "dec": 0.126243049656, 80 | "raSigma": 0.00028, 81 | "decSigma": 0.00028, 82 | "ra_dec_Cov": 0.00029, 83 | "radecMjdTai": 1480360995, 84 | "pmRa": 0.00013, 85 | "pmDec": 0.00014, 86 | "parallax": 2.124124, 87 | "pmRaSigma": 0.00013, 88 | "pmDecSigma": 0.00013, 89 | "parallaxSigma": 0.00013, 90 | "pmRa_pmDec_Cov": 0.00013, 91 | "pmRa_parallax_Cov": 0.00013, 92 | "pmDec_parallax_Cov": 0.00013, 93 | "pmParallaxLnL": 0.00013, 94 | "pmParallaxChi2": 0.00013, 95 | "pmParallaxNdata": 1214, 96 | "flags": 0 97 | }, 98 | "ssObject": { 99 | "ssObjectId":5364546, 100 | "numObs": 10, 101 | "flags": 0 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/5/0/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/5/0/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | 5 | with open("alert.json", "r") as f: 6 | data = json.load(f) 7 | 8 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v5_0.alert").get_by_version("5.0") 9 | with open("fakeAlert.avro", "wb") as f: 10 | schema.store_alerts(f, [data]) 11 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/0/lsst.v6_0.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v6_0", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "sample avro alert schema v6.0", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v6_0.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v6_0.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v6_0.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v6_0.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v6_0.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v6_0.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v6_0", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Id of the ccdVisit where this forcedSource was measured.", 18 | "name": "ccdVisitId", 19 | "type": "long" 20 | }, 21 | { 22 | "doc": "Point Source model flux.", 23 | "default": null, 24 | "name": "psfFlux", 25 | "type": [ 26 | "null", 27 | "float" 28 | ] 29 | }, 30 | { 31 | "doc": "Uncertainty of psfFlux.", 32 | "default": null, 33 | "name": "psfFluxErr", 34 | "type": [ 35 | "null", 36 | "float" 37 | ] 38 | }, 39 | { 40 | "doc": "x position at which psfFlux has been measured.", 41 | "default": null, 42 | "name": "x", 43 | "type": [ 44 | "null", 45 | "float" 46 | ] 47 | }, 48 | { 49 | "doc": "y position at which psfFlux has been measured.", 50 | "default": null, 51 | "name": "y", 52 | "type": [ 53 | "null", 54 | "float" 55 | ] 56 | }, 57 | { 58 | "doc": "Flags, bitwise OR tbd", 59 | "default": null, 60 | "name": "flags", 61 | "type": [ 62 | "null", 63 | "long" 64 | ] 65 | }, 66 | { 67 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 68 | "name": "midpointMjdTai", 69 | "type": "double" 70 | }, 71 | { 72 | "doc": "Filter band this source was observed with.", 73 | "default": null, 74 | "name": "band", 75 | "type": [ 76 | "null", 77 | "string" 78 | ] 79 | } 80 | ] 81 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v6_0", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/0/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "ccdVisitId": 111111, 9 | "midpointMjdTai": 1480360995, 10 | "band": "r", 11 | "programId": 1, 12 | "ra": 351.570546978, 13 | "dec": 0.126243049656, 14 | "raErr": 0.00028, 15 | "decErr": 0.00028, 16 | "ra_dec_Cov": 0.00029, 17 | "x": 112.1, 18 | "y": 121.1, 19 | "xErr": 1.2, 20 | "yErr": 1.1, 21 | "x_y_Cov": 1.2, 22 | "apFlux": 1241.0, 23 | "apFluxErr": 12.0, 24 | "snr": 41.1, 25 | "trailNdata": 5, 26 | "psfFlux": 1241.0, 27 | "psfFluxErr": 12.0, 28 | "flags": 0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "ccdVisitId": 111111, 35 | "midpointMjdTai": 1480360995, 36 | "band": "r", 37 | "programId": 1, 38 | "ra": 351.570546978, 39 | "dec": 0.126243049656, 40 | "raErr": 0.00028, 41 | "decErr": 0.00028, 42 | "ra_dec_Cov": 0.00029, 43 | "x": 112.1, 44 | "y": 121.1, 45 | "xErr": 1.2, 46 | "yErr": 1.1, 47 | "x_y_Cov": 1.2, 48 | "apFlux": 1241.0, 49 | "apFluxErr": 12.0, 50 | "snr": 41.1, 51 | "trailNdata": 5, 52 | "psfFlux": 1241.0, 53 | "psfFluxErr": 12.0, 54 | "flags": 0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "ccdVisitId": 111111, 60 | "midpointMjdTai": 1480360995, 61 | "band": "r", 62 | "programId": 1, 63 | "ra": 351.570546978, 64 | "dec": 0.126243049656, 65 | "raErr": 0.00028, 66 | "decErr": 0.00028, 67 | "ra_dec_Cov": 0.00029, 68 | "x": 112.1, 69 | "y": 121.1, 70 | "xErr": 1.2, 71 | "yErr": 1.1, 72 | "x_y_Cov": 1.2, 73 | "apFlux": 1241.0, 74 | "apFluxErr": 12.0, 75 | "snr": 41.1, 76 | "trailNdata": 5, 77 | "psfFlux": 1241.0, 78 | "psfFluxErr": 12.0, 79 | "flags": 0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214, 104 | "flags": 0 105 | }, 106 | "ssObject": { 107 | "ssObjectId":5364546, 108 | "numObs": 10, 109 | "flags": 0 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/0/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/6/0/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/0/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v6_0.alert").get_by_version("6.0") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/1/lsst.v6_1.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v6_1", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "Rubin Avro alert schema v6.1", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v6_1.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v6_1.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v6_1.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v6_1.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v6_1.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v6_1.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutScience", "type": ["null", "bytes"], "default": null}, 22 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/1/lsst.v6_1.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v6_1", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Id of the ccdVisit where this forcedSource was measured.", 18 | "name": "ccdVisitId", 19 | "type": "long" 20 | }, 21 | { 22 | "doc": "Point Source model flux.", 23 | "default": null, 24 | "name": "psfFlux", 25 | "type": [ 26 | "null", 27 | "float" 28 | ] 29 | }, 30 | { 31 | "doc": "Uncertainty of psfFlux.", 32 | "default": null, 33 | "name": "psfFluxErr", 34 | "type": [ 35 | "null", 36 | "float" 37 | ] 38 | }, 39 | { 40 | "doc": "x position at which psfFlux has been measured.", 41 | "default": null, 42 | "name": "x", 43 | "type": [ 44 | "null", 45 | "float" 46 | ] 47 | }, 48 | { 49 | "doc": "y position at which psfFlux has been measured.", 50 | "default": null, 51 | "name": "y", 52 | "type": [ 53 | "null", 54 | "float" 55 | ] 56 | }, 57 | { 58 | "doc": "Flags, bitwise OR tbd", 59 | "default": null, 60 | "name": "flags", 61 | "type": [ 62 | "null", 63 | "long" 64 | ] 65 | }, 66 | { 67 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 68 | "name": "midpointMjdTai", 69 | "type": "double" 70 | }, 71 | { 72 | "doc": "Filter band this source was observed with.", 73 | "default": null, 74 | "name": "band", 75 | "type": [ 76 | "null", 77 | "string" 78 | ] 79 | } 80 | ] 81 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/1/lsst.v6_1.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v6_1", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/1/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "ccdVisitId": 111111, 9 | "midpointMjdTai": 1480360995, 10 | "band": "r", 11 | "programId": 1, 12 | "ra": 351.570546978, 13 | "dec": 0.126243049656, 14 | "raErr": 0.00028, 15 | "decErr": 0.00028, 16 | "ra_dec_Cov": 0.00029, 17 | "x": 112.1, 18 | "y": 121.1, 19 | "xErr": 1.2, 20 | "yErr": 1.1, 21 | "x_y_Cov": 1.2, 22 | "apFlux": 1241.0, 23 | "apFluxErr": 12.0, 24 | "snr": 41.1, 25 | "trailNdata": 5, 26 | "psfFlux": 1241.0, 27 | "psfFluxErr": 12.0, 28 | "flags": 0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "ccdVisitId": 111111, 35 | "midpointMjdTai": 1480360995, 36 | "band": "r", 37 | "programId": 1, 38 | "ra": 351.570546978, 39 | "dec": 0.126243049656, 40 | "raErr": 0.00028, 41 | "decErr": 0.00028, 42 | "ra_dec_Cov": 0.00029, 43 | "x": 112.1, 44 | "y": 121.1, 45 | "xErr": 1.2, 46 | "yErr": 1.1, 47 | "x_y_Cov": 1.2, 48 | "apFlux": 1241.0, 49 | "apFluxErr": 12.0, 50 | "snr": 41.1, 51 | "trailNdata": 5, 52 | "psfFlux": 1241.0, 53 | "psfFluxErr": 12.0, 54 | "flags": 0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "ccdVisitId": 111111, 60 | "midpointMjdTai": 1480360995, 61 | "band": "r", 62 | "programId": 1, 63 | "ra": 351.570546978, 64 | "dec": 0.126243049656, 65 | "raErr": 0.00028, 66 | "decErr": 0.00028, 67 | "ra_dec_Cov": 0.00029, 68 | "x": 112.1, 69 | "y": 121.1, 70 | "xErr": 1.2, 71 | "yErr": 1.1, 72 | "x_y_Cov": 1.2, 73 | "apFlux": 1241.0, 74 | "apFluxErr": 12.0, 75 | "snr": 41.1, 76 | "trailNdata": 5, 77 | "psfFlux": 1241.0, 78 | "psfFluxErr": 12.0, 79 | "flags": 0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214, 104 | "flags": 0 105 | }, 106 | "ssObject": { 107 | "ssObjectId":5364546, 108 | "numObs": 10, 109 | "flags": 0 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/1/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/6/1/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/6/1/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v6_1.alert").get_by_version("6.1") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/lsst.v7_0.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_0", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "Rubin Avro alert schema v7.0", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v7_0.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v7_0.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v7_0.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v7_0.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v7_0.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v7_0.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutScience", "type": ["null", "bytes"], "default": null}, 22 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/lsst.v7_0.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_0", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Id of the visit where this forcedSource was measured.", 18 | "name": "visit", 19 | "type": "long" 20 | }, 21 | { 22 | "doc": "Id of the detector where this forcedSource was measured.", 23 | "name": "detector", 24 | "type": "int" 25 | }, 26 | { 27 | "default": null, 28 | "doc": "Point Source model flux.", 29 | "name": "psfFlux", 30 | "type": [ 31 | "null", 32 | "float" 33 | ] 34 | }, 35 | { 36 | "default": null, 37 | "doc": "Uncertainty of psfFlux.", 38 | "name": "psfFluxErr", 39 | "type": [ 40 | "null", 41 | "float" 42 | ] 43 | }, 44 | { 45 | "default": null, 46 | "doc": "x position at which psfFlux has been measured.", 47 | "name": "x", 48 | "type": [ 49 | "null", 50 | "float" 51 | ] 52 | }, 53 | { 54 | "default": null, 55 | "doc": "y position at which psfFlux has been measured.", 56 | "name": "y", 57 | "type": [ 58 | "null", 59 | "float" 60 | ] 61 | }, 62 | { 63 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 64 | "name": "midpointMjdTai", 65 | "type": "double" 66 | }, 67 | { 68 | "default": null, 69 | "doc": "Filter band this source was observed with.", 70 | "name": "band", 71 | "type": [ 72 | "null", 73 | "string" 74 | ] 75 | } 76 | ] 77 | } 78 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/lsst.v7_0.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_0", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/lsst.v7_0.ssObject.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_0", 4 | "name": "ssObject", 5 | "fields": [ 6 | { 7 | "doc": "Unique identifier.", 8 | "name": "ssObjectId", 9 | "type": "long" 10 | }, 11 | { 12 | "default": null, 13 | "doc": "The date the LSST first linked and submitted the discovery observations to the MPC. May be NULL if not an LSST discovery. The date format will follow general LSST conventions (MJD TAI, at the moment).", 14 | "name": "discoverySubmissionDate", 15 | "type": [ 16 | "null", 17 | "double" 18 | ] 19 | }, 20 | { 21 | "default": null, 22 | "doc": "The time of the first LSST observation of this object (could be precovered) as Modified Julian Date, International Atomic Time.", 23 | "name": "firstObservationDate", 24 | "type": [ 25 | "null", 26 | "double" 27 | ] 28 | }, 29 | { 30 | "default": null, 31 | "doc": "Arc of LSST observations.", 32 | "name": "arc", 33 | "type": [ 34 | "null", 35 | "float" 36 | ] 37 | }, 38 | { 39 | "default": null, 40 | "doc": "Number of LSST observations of this object.", 41 | "name": "numObs", 42 | "type": [ 43 | "null", 44 | "int" 45 | ] 46 | }, 47 | { 48 | "default": null, 49 | "doc": "Minimum orbit intersection distance to Earth.", 50 | "name": "MOID", 51 | "type": [ 52 | "null", 53 | "float" 54 | ] 55 | }, 56 | { 57 | "default": null, 58 | "doc": "True anomaly of the MOID point.", 59 | "name": "MOIDTrueAnomaly", 60 | "type": [ 61 | "null", 62 | "float" 63 | ] 64 | }, 65 | { 66 | "default": null, 67 | "doc": "Ecliptic longitude of the MOID point.", 68 | "name": "MOIDEclipticLongitude", 69 | "type": [ 70 | "null", 71 | "float" 72 | ] 73 | }, 74 | { 75 | "default": null, 76 | "doc": "DeltaV at the MOID point.", 77 | "name": "MOIDDeltaV", 78 | "type": [ 79 | "null", 80 | "float" 81 | ] 82 | }, 83 | { 84 | "default": null, 85 | "doc": "Best fit absolute magnitude (u band).", 86 | "name": "u_H", 87 | "type": [ 88 | "null", 89 | "float" 90 | ] 91 | }, 92 | { 93 | "default": null, 94 | "doc": "Best fit G12 slope parameter (u band).", 95 | "name": "u_G12", 96 | "type": [ 97 | "null", 98 | "float" 99 | ] 100 | }, 101 | { 102 | "default": null, 103 | "doc": "Uncertainty of H (u band).", 104 | "name": "u_HErr", 105 | "type": [ 106 | "null", 107 | "float" 108 | ] 109 | }, 110 | { 111 | "default": null, 112 | "doc": "Uncertainty of G12 (u band).", 113 | "name": "u_G12Err", 114 | "type": [ 115 | "null", 116 | "float" 117 | ] 118 | }, 119 | { 120 | "default": null, 121 | "doc": "H-G12 covariance (u band).", 122 | "name": "u_H_u_G12_Cov", 123 | "type": [ 124 | "null", 125 | "float" 126 | ] 127 | }, 128 | { 129 | "default": null, 130 | "doc": "Chi^2 statistic of the phase curve fit (u band).", 131 | "name": "u_Chi2", 132 | "type": [ 133 | "null", 134 | "float" 135 | ] 136 | }, 137 | { 138 | "default": null, 139 | "doc": "The number of data points used to fit the phase curve (u band).", 140 | "name": "u_Ndata", 141 | "type": [ 142 | "null", 143 | "int" 144 | ] 145 | }, 146 | { 147 | "default": null, 148 | "doc": "Best fit absolute magnitude (g band).", 149 | "name": "g_H", 150 | "type": [ 151 | "null", 152 | "float" 153 | ] 154 | }, 155 | { 156 | "default": null, 157 | "doc": "Best fit G12 slope parameter (g band).", 158 | "name": "g_G12", 159 | "type": [ 160 | "null", 161 | "float" 162 | ] 163 | }, 164 | { 165 | "default": null, 166 | "doc": "Uncertainty of H (g band).", 167 | "name": "g_HErr", 168 | "type": [ 169 | "null", 170 | "float" 171 | ] 172 | }, 173 | { 174 | "default": null, 175 | "doc": "Uncertainty of G12 (g band).", 176 | "name": "g_G12Err", 177 | "type": [ 178 | "null", 179 | "float" 180 | ] 181 | }, 182 | { 183 | "default": null, 184 | "doc": "H-G12 covariance (g band).", 185 | "name": "g_H_g_G12_Cov", 186 | "type": [ 187 | "null", 188 | "float" 189 | ] 190 | }, 191 | { 192 | "default": null, 193 | "doc": "Chi^2 statistic of the phase curve fit (g band).", 194 | "name": "g_Chi2", 195 | "type": [ 196 | "null", 197 | "float" 198 | ] 199 | }, 200 | { 201 | "default": null, 202 | "doc": "The number of data points used to fit the phase curve (g band).", 203 | "name": "g_Ndata", 204 | "type": [ 205 | "null", 206 | "int" 207 | ] 208 | }, 209 | { 210 | "default": null, 211 | "doc": "Best fit absolute magnitude (r band).", 212 | "name": "r_H", 213 | "type": [ 214 | "null", 215 | "float" 216 | ] 217 | }, 218 | { 219 | "default": null, 220 | "doc": "Best fit G12 slope parameter (r band).", 221 | "name": "r_G12", 222 | "type": [ 223 | "null", 224 | "float" 225 | ] 226 | }, 227 | { 228 | "default": null, 229 | "doc": "Uncertainty of H (r band).", 230 | "name": "r_HErr", 231 | "type": [ 232 | "null", 233 | "float" 234 | ] 235 | }, 236 | { 237 | "default": null, 238 | "doc": "Uncertainty of G12 (r band).", 239 | "name": "r_G12Err", 240 | "type": [ 241 | "null", 242 | "float" 243 | ] 244 | }, 245 | { 246 | "default": null, 247 | "doc": "H-G12 covariance (r band).", 248 | "name": "r_H_r_G12_Cov", 249 | "type": [ 250 | "null", 251 | "float" 252 | ] 253 | }, 254 | { 255 | "default": null, 256 | "doc": "Chi^2 statistic of the phase curve fit (r band).", 257 | "name": "r_Chi2", 258 | "type": [ 259 | "null", 260 | "float" 261 | ] 262 | }, 263 | { 264 | "default": null, 265 | "doc": "The number of data points used to fit the phase curve (r band).", 266 | "name": "r_Ndata", 267 | "type": [ 268 | "null", 269 | "int" 270 | ] 271 | }, 272 | { 273 | "default": null, 274 | "doc": "Best fit absolute magnitude (i band).", 275 | "name": "i_H", 276 | "type": [ 277 | "null", 278 | "float" 279 | ] 280 | }, 281 | { 282 | "default": null, 283 | "doc": "Best fit G12 slope parameter (i band).", 284 | "name": "i_G12", 285 | "type": [ 286 | "null", 287 | "float" 288 | ] 289 | }, 290 | { 291 | "default": null, 292 | "doc": "Uncertainty of H (i band).", 293 | "name": "i_HErr", 294 | "type": [ 295 | "null", 296 | "float" 297 | ] 298 | }, 299 | { 300 | "default": null, 301 | "doc": "Uncertainty of G12 (i band).", 302 | "name": "i_G12Err", 303 | "type": [ 304 | "null", 305 | "float" 306 | ] 307 | }, 308 | { 309 | "default": null, 310 | "doc": "H-G12 covariance (i band).", 311 | "name": "i_H_i_G12_Cov", 312 | "type": [ 313 | "null", 314 | "float" 315 | ] 316 | }, 317 | { 318 | "default": null, 319 | "doc": "Chi^2 statistic of the phase curve fit (i band).", 320 | "name": "i_Chi2", 321 | "type": [ 322 | "null", 323 | "float" 324 | ] 325 | }, 326 | { 327 | "default": null, 328 | "doc": "The number of data points used to fit the phase curve (i band).", 329 | "name": "i_Ndata", 330 | "type": [ 331 | "null", 332 | "int" 333 | ] 334 | }, 335 | { 336 | "default": null, 337 | "doc": "Best fit absolute magnitude (z band).", 338 | "name": "z_H", 339 | "type": [ 340 | "null", 341 | "float" 342 | ] 343 | }, 344 | { 345 | "default": null, 346 | "doc": "Best fit G12 slope parameter (z band).", 347 | "name": "z_G12", 348 | "type": [ 349 | "null", 350 | "float" 351 | ] 352 | }, 353 | { 354 | "default": null, 355 | "doc": "Uncertainty of H (z band).", 356 | "name": "z_HErr", 357 | "type": [ 358 | "null", 359 | "float" 360 | ] 361 | }, 362 | { 363 | "default": null, 364 | "doc": "Uncertainty of G12 (z band).", 365 | "name": "z_G12Err", 366 | "type": [ 367 | "null", 368 | "float" 369 | ] 370 | }, 371 | { 372 | "default": null, 373 | "doc": "H-G12 covariance (z band).", 374 | "name": "z_H_z_G12_Cov", 375 | "type": [ 376 | "null", 377 | "float" 378 | ] 379 | }, 380 | { 381 | "default": null, 382 | "doc": "Chi^2 statistic of the phase curve fit (z band).", 383 | "name": "z_Chi2", 384 | "type": [ 385 | "null", 386 | "float" 387 | ] 388 | }, 389 | { 390 | "default": null, 391 | "doc": "The number of data points used to fit the phase curve (z band).", 392 | "name": "z_Ndata", 393 | "type": [ 394 | "null", 395 | "int" 396 | ] 397 | }, 398 | { 399 | "default": null, 400 | "doc": "Best fit absolute magnitude (y band).", 401 | "name": "y_H", 402 | "type": [ 403 | "null", 404 | "float" 405 | ] 406 | }, 407 | { 408 | "default": null, 409 | "doc": "Best fit G12 slope parameter (y band).", 410 | "name": "y_G12", 411 | "type": [ 412 | "null", 413 | "float" 414 | ] 415 | }, 416 | { 417 | "default": null, 418 | "doc": "Uncertainty of H (y band).", 419 | "name": "y_HErr", 420 | "type": [ 421 | "null", 422 | "float" 423 | ] 424 | }, 425 | { 426 | "default": null, 427 | "doc": "Uncertainty of G12 (y band).", 428 | "name": "y_G12Err", 429 | "type": [ 430 | "null", 431 | "float" 432 | ] 433 | }, 434 | { 435 | "default": null, 436 | "doc": "H-G12 covariance (y band).", 437 | "name": "y_H_y_G12_Cov", 438 | "type": [ 439 | "null", 440 | "float" 441 | ] 442 | }, 443 | { 444 | "default": null, 445 | "doc": "Chi^2 statistic of the phase curve fit (y band).", 446 | "name": "y_Chi2", 447 | "type": [ 448 | "null", 449 | "float" 450 | ] 451 | }, 452 | { 453 | "default": null, 454 | "doc": "The number of data points used to fit the phase curve (y band).", 455 | "name": "y_Ndata", 456 | "type": [ 457 | "null", 458 | "int" 459 | ] 460 | }, 461 | { 462 | "default": null, 463 | "doc": "median `extendedness` value from the DIASource.", 464 | "name": "medianExtendedness", 465 | "type": [ 466 | "null", 467 | "float" 468 | ] 469 | } 470 | ] 471 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "visit": 1, 9 | "detector": 42, 10 | "midpointMjdTai": 1480360995, 11 | "band": "r", 12 | "programId": 1, 13 | "ra": 351.570546978, 14 | "dec": 0.126243049656, 15 | "raErr": 0.00028, 16 | "decErr": 0.00028, 17 | "ra_dec_Cov": 0.00029, 18 | "x": 112.1, 19 | "y": 121.1, 20 | "xErr": 1.2, 21 | "yErr": 1.1, 22 | "x_y_Cov": 1.2, 23 | "apFlux": 1241.0, 24 | "apFluxErr": 12.0, 25 | "snr": 41.1, 26 | "trailNdata": 5, 27 | "psfFlux": 1241.0, 28 | "psfFluxErr": 12.0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "visit": 2, 35 | "detector": 43, 36 | "midpointMjdTai": 1480360995, 37 | "band": "r", 38 | "programId": 1, 39 | "ra": 351.570546978, 40 | "dec": 0.126243049656, 41 | "raErr": 0.00028, 42 | "decErr": 0.00028, 43 | "ra_dec_Cov": 0.00029, 44 | "x": 112.1, 45 | "y": 121.1, 46 | "xErr": 1.2, 47 | "yErr": 1.1, 48 | "x_y_Cov": 1.2, 49 | "apFlux": 1241.0, 50 | "apFluxErr": 12.0, 51 | "snr": 41.1, 52 | "trailNdata": 5, 53 | "psfFlux": 1241.0, 54 | "psfFluxErr": 12.0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "visit": 1, 60 | "detector": 42, 61 | "midpointMjdTai": 1480360995, 62 | "band": "r", 63 | "programId": 1, 64 | "ra": 351.570546978, 65 | "dec": 0.126243049656, 66 | "raErr": 0.00028, 67 | "decErr": 0.00028, 68 | "ra_dec_Cov": 0.00029, 69 | "x": 112.1, 70 | "y": 121.1, 71 | "xErr": 1.2, 72 | "yErr": 1.1, 73 | "x_y_Cov": 1.2, 74 | "apFlux": 1241.0, 75 | "apFluxErr": 12.0, 76 | "snr": 41.1, 77 | "trailNdata": 5, 78 | "psfFlux": 1241.0, 79 | "psfFluxErr": 12.0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214 104 | }, 105 | "ssObject": { 106 | "ssObjectId":5364546, 107 | "numObs": 10 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/7/0/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/0/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v7_0.alert").get_by_version("7.0") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/lsst.v7_1.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_1", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "Rubin Avro alert schema v7.1", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v7_1.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v7_1.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v7_1.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v7_1.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v7_1.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v7_1.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutScience", "type": ["null", "bytes"], "default": null}, 22 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/lsst.v7_1.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_1", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Right ascension coordinate of the position of the object at time radecMjdTai.", 18 | "name": "ra", 19 | "type": "double" 20 | }, 21 | { 22 | "doc": "Declination coordinate of the position of the object at time radecMjdTai.", 23 | "name": "dec", 24 | "type": "double" 25 | }, 26 | { 27 | "doc": "Id of the visit where this forcedSource was measured.", 28 | "name": "visit", 29 | "type": "long" 30 | }, 31 | { 32 | "doc": "Id of the detector where this forcedSource was measured. Datatype short instead of byte because of DB concerns about unsigned bytes.", 33 | "name": "detector", 34 | "type": "int" 35 | }, 36 | { 37 | "default": null, 38 | "doc": "Point Source model flux.", 39 | "name": "psfFlux", 40 | "type": [ 41 | "null", 42 | "float" 43 | ] 44 | }, 45 | { 46 | "default": null, 47 | "doc": "Uncertainty of psfFlux.", 48 | "name": "psfFluxErr", 49 | "type": [ 50 | "null", 51 | "float" 52 | ] 53 | }, 54 | { 55 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 56 | "name": "midpointMjdTai", 57 | "type": "double" 58 | }, 59 | { 60 | "default": null, 61 | "doc": "Filter band this source was observed with.", 62 | "name": "band", 63 | "type": [ 64 | "null", 65 | "string" 66 | ] 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/lsst.v7_1.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_1", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/lsst.v7_1.ssObject.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_1", 4 | "name": "ssObject", 5 | "fields": [ 6 | { 7 | "doc": "Unique identifier.", 8 | "name": "ssObjectId", 9 | "type": "long" 10 | }, 11 | { 12 | "default": null, 13 | "doc": "The date the LSST first linked and submitted the discovery observations to the MPC. May be NULL if not an LSST discovery. The date format will follow general LSST conventions (MJD TAI, at the moment).", 14 | "name": "discoverySubmissionDate", 15 | "type": [ 16 | "null", 17 | "double" 18 | ] 19 | }, 20 | { 21 | "default": null, 22 | "doc": "The time of the first LSST observation of this object (could be precovered) as Modified Julian Date, International Atomic Time.", 23 | "name": "firstObservationDate", 24 | "type": [ 25 | "null", 26 | "double" 27 | ] 28 | }, 29 | { 30 | "default": null, 31 | "doc": "Arc of LSST observations.", 32 | "name": "arc", 33 | "type": [ 34 | "null", 35 | "float" 36 | ] 37 | }, 38 | { 39 | "default": null, 40 | "doc": "Number of LSST observations of this object.", 41 | "name": "numObs", 42 | "type": [ 43 | "null", 44 | "int" 45 | ] 46 | }, 47 | { 48 | "default": null, 49 | "doc": "Minimum orbit intersection distance to Earth.", 50 | "name": "MOID", 51 | "type": [ 52 | "null", 53 | "float" 54 | ] 55 | }, 56 | { 57 | "default": null, 58 | "doc": "True anomaly of the MOID point.", 59 | "name": "MOIDTrueAnomaly", 60 | "type": [ 61 | "null", 62 | "float" 63 | ] 64 | }, 65 | { 66 | "default": null, 67 | "doc": "Ecliptic longitude of the MOID point.", 68 | "name": "MOIDEclipticLongitude", 69 | "type": [ 70 | "null", 71 | "float" 72 | ] 73 | }, 74 | { 75 | "default": null, 76 | "doc": "DeltaV at the MOID point.", 77 | "name": "MOIDDeltaV", 78 | "type": [ 79 | "null", 80 | "float" 81 | ] 82 | }, 83 | { 84 | "default": null, 85 | "doc": "Best fit absolute magnitude (u band).", 86 | "name": "u_H", 87 | "type": [ 88 | "null", 89 | "float" 90 | ] 91 | }, 92 | { 93 | "default": null, 94 | "doc": "Best fit G12 slope parameter (u band).", 95 | "name": "u_G12", 96 | "type": [ 97 | "null", 98 | "float" 99 | ] 100 | }, 101 | { 102 | "default": null, 103 | "doc": "Uncertainty of H (u band).", 104 | "name": "u_HErr", 105 | "type": [ 106 | "null", 107 | "float" 108 | ] 109 | }, 110 | { 111 | "default": null, 112 | "doc": "Uncertainty of G12 (u band).", 113 | "name": "u_G12Err", 114 | "type": [ 115 | "null", 116 | "float" 117 | ] 118 | }, 119 | { 120 | "default": null, 121 | "doc": "H-G12 covariance (u band).", 122 | "name": "u_H_u_G12_Cov", 123 | "type": [ 124 | "null", 125 | "float" 126 | ] 127 | }, 128 | { 129 | "default": null, 130 | "doc": "Chi^2 statistic of the phase curve fit (u band).", 131 | "name": "u_Chi2", 132 | "type": [ 133 | "null", 134 | "float" 135 | ] 136 | }, 137 | { 138 | "default": null, 139 | "doc": "The number of data points used to fit the phase curve (u band).", 140 | "name": "u_Ndata", 141 | "type": [ 142 | "null", 143 | "int" 144 | ] 145 | }, 146 | { 147 | "default": null, 148 | "doc": "Best fit absolute magnitude (g band).", 149 | "name": "g_H", 150 | "type": [ 151 | "null", 152 | "float" 153 | ] 154 | }, 155 | { 156 | "default": null, 157 | "doc": "Best fit G12 slope parameter (g band).", 158 | "name": "g_G12", 159 | "type": [ 160 | "null", 161 | "float" 162 | ] 163 | }, 164 | { 165 | "default": null, 166 | "doc": "Uncertainty of H (g band).", 167 | "name": "g_HErr", 168 | "type": [ 169 | "null", 170 | "float" 171 | ] 172 | }, 173 | { 174 | "default": null, 175 | "doc": "Uncertainty of G12 (g band).", 176 | "name": "g_G12Err", 177 | "type": [ 178 | "null", 179 | "float" 180 | ] 181 | }, 182 | { 183 | "default": null, 184 | "doc": "H-G12 covariance (g band).", 185 | "name": "g_H_g_G12_Cov", 186 | "type": [ 187 | "null", 188 | "float" 189 | ] 190 | }, 191 | { 192 | "default": null, 193 | "doc": "Chi^2 statistic of the phase curve fit (g band).", 194 | "name": "g_Chi2", 195 | "type": [ 196 | "null", 197 | "float" 198 | ] 199 | }, 200 | { 201 | "default": null, 202 | "doc": "The number of data points used to fit the phase curve (g band).", 203 | "name": "g_Ndata", 204 | "type": [ 205 | "null", 206 | "int" 207 | ] 208 | }, 209 | { 210 | "default": null, 211 | "doc": "Best fit absolute magnitude (r band).", 212 | "name": "r_H", 213 | "type": [ 214 | "null", 215 | "float" 216 | ] 217 | }, 218 | { 219 | "default": null, 220 | "doc": "Best fit G12 slope parameter (r band).", 221 | "name": "r_G12", 222 | "type": [ 223 | "null", 224 | "float" 225 | ] 226 | }, 227 | { 228 | "default": null, 229 | "doc": "Uncertainty of H (r band).", 230 | "name": "r_HErr", 231 | "type": [ 232 | "null", 233 | "float" 234 | ] 235 | }, 236 | { 237 | "default": null, 238 | "doc": "Uncertainty of G12 (r band).", 239 | "name": "r_G12Err", 240 | "type": [ 241 | "null", 242 | "float" 243 | ] 244 | }, 245 | { 246 | "default": null, 247 | "doc": "H-G12 covariance (r band).", 248 | "name": "r_H_r_G12_Cov", 249 | "type": [ 250 | "null", 251 | "float" 252 | ] 253 | }, 254 | { 255 | "default": null, 256 | "doc": "Chi^2 statistic of the phase curve fit (r band).", 257 | "name": "r_Chi2", 258 | "type": [ 259 | "null", 260 | "float" 261 | ] 262 | }, 263 | { 264 | "default": null, 265 | "doc": "The number of data points used to fit the phase curve (r band).", 266 | "name": "r_Ndata", 267 | "type": [ 268 | "null", 269 | "int" 270 | ] 271 | }, 272 | { 273 | "default": null, 274 | "doc": "Best fit absolute magnitude (i band).", 275 | "name": "i_H", 276 | "type": [ 277 | "null", 278 | "float" 279 | ] 280 | }, 281 | { 282 | "default": null, 283 | "doc": "Best fit G12 slope parameter (i band).", 284 | "name": "i_G12", 285 | "type": [ 286 | "null", 287 | "float" 288 | ] 289 | }, 290 | { 291 | "default": null, 292 | "doc": "Uncertainty of H (i band).", 293 | "name": "i_HErr", 294 | "type": [ 295 | "null", 296 | "float" 297 | ] 298 | }, 299 | { 300 | "default": null, 301 | "doc": "Uncertainty of G12 (i band).", 302 | "name": "i_G12Err", 303 | "type": [ 304 | "null", 305 | "float" 306 | ] 307 | }, 308 | { 309 | "default": null, 310 | "doc": "H-G12 covariance (i band).", 311 | "name": "i_H_i_G12_Cov", 312 | "type": [ 313 | "null", 314 | "float" 315 | ] 316 | }, 317 | { 318 | "default": null, 319 | "doc": "Chi^2 statistic of the phase curve fit (i band).", 320 | "name": "i_Chi2", 321 | "type": [ 322 | "null", 323 | "float" 324 | ] 325 | }, 326 | { 327 | "default": null, 328 | "doc": "The number of data points used to fit the phase curve (i band).", 329 | "name": "i_Ndata", 330 | "type": [ 331 | "null", 332 | "int" 333 | ] 334 | }, 335 | { 336 | "default": null, 337 | "doc": "Best fit absolute magnitude (z band).", 338 | "name": "z_H", 339 | "type": [ 340 | "null", 341 | "float" 342 | ] 343 | }, 344 | { 345 | "default": null, 346 | "doc": "Best fit G12 slope parameter (z band).", 347 | "name": "z_G12", 348 | "type": [ 349 | "null", 350 | "float" 351 | ] 352 | }, 353 | { 354 | "default": null, 355 | "doc": "Uncertainty of H (z band).", 356 | "name": "z_HErr", 357 | "type": [ 358 | "null", 359 | "float" 360 | ] 361 | }, 362 | { 363 | "default": null, 364 | "doc": "Uncertainty of G12 (z band).", 365 | "name": "z_G12Err", 366 | "type": [ 367 | "null", 368 | "float" 369 | ] 370 | }, 371 | { 372 | "default": null, 373 | "doc": "H-G12 covariance (z band).", 374 | "name": "z_H_z_G12_Cov", 375 | "type": [ 376 | "null", 377 | "float" 378 | ] 379 | }, 380 | { 381 | "default": null, 382 | "doc": "Chi^2 statistic of the phase curve fit (z band).", 383 | "name": "z_Chi2", 384 | "type": [ 385 | "null", 386 | "float" 387 | ] 388 | }, 389 | { 390 | "default": null, 391 | "doc": "The number of data points used to fit the phase curve (z band).", 392 | "name": "z_Ndata", 393 | "type": [ 394 | "null", 395 | "int" 396 | ] 397 | }, 398 | { 399 | "default": null, 400 | "doc": "Best fit absolute magnitude (y band).", 401 | "name": "y_H", 402 | "type": [ 403 | "null", 404 | "float" 405 | ] 406 | }, 407 | { 408 | "default": null, 409 | "doc": "Best fit G12 slope parameter (y band).", 410 | "name": "y_G12", 411 | "type": [ 412 | "null", 413 | "float" 414 | ] 415 | }, 416 | { 417 | "default": null, 418 | "doc": "Uncertainty of H (y band).", 419 | "name": "y_HErr", 420 | "type": [ 421 | "null", 422 | "float" 423 | ] 424 | }, 425 | { 426 | "default": null, 427 | "doc": "Uncertainty of G12 (y band).", 428 | "name": "y_G12Err", 429 | "type": [ 430 | "null", 431 | "float" 432 | ] 433 | }, 434 | { 435 | "default": null, 436 | "doc": "H-G12 covariance (y band).", 437 | "name": "y_H_y_G12_Cov", 438 | "type": [ 439 | "null", 440 | "float" 441 | ] 442 | }, 443 | { 444 | "default": null, 445 | "doc": "Chi^2 statistic of the phase curve fit (y band).", 446 | "name": "y_Chi2", 447 | "type": [ 448 | "null", 449 | "float" 450 | ] 451 | }, 452 | { 453 | "default": null, 454 | "doc": "The number of data points used to fit the phase curve (y band).", 455 | "name": "y_Ndata", 456 | "type": [ 457 | "null", 458 | "int" 459 | ] 460 | }, 461 | { 462 | "default": null, 463 | "doc": "median `extendedness` value from the DIASource.", 464 | "name": "medianExtendedness", 465 | "type": [ 466 | "null", 467 | "float" 468 | ] 469 | } 470 | ] 471 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "visit": 1, 9 | "detector": 42, 10 | "midpointMjdTai": 1480360995, 11 | "band": "r", 12 | "programId": 1, 13 | "ra": 351.570546978, 14 | "dec": 0.126243049656, 15 | "raErr": 0.00028, 16 | "decErr": 0.00028, 17 | "ra_dec_Cov": 0.00029, 18 | "x": 112.1, 19 | "y": 121.1, 20 | "xErr": 1.2, 21 | "yErr": 1.1, 22 | "x_y_Cov": 1.2, 23 | "apFlux": 1241.0, 24 | "apFluxErr": 12.0, 25 | "snr": 41.1, 26 | "trailNdata": 5, 27 | "psfFlux": 1241.0, 28 | "psfFluxErr": 12.0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "visit": 2, 35 | "detector": 43, 36 | "midpointMjdTai": 1480360995, 37 | "band": "r", 38 | "programId": 1, 39 | "ra": 351.570546978, 40 | "dec": 0.126243049656, 41 | "raErr": 0.00028, 42 | "decErr": 0.00028, 43 | "ra_dec_Cov": 0.00029, 44 | "x": 112.1, 45 | "y": 121.1, 46 | "xErr": 1.2, 47 | "yErr": 1.1, 48 | "x_y_Cov": 1.2, 49 | "apFlux": 1241.0, 50 | "apFluxErr": 12.0, 51 | "snr": 41.1, 52 | "trailNdata": 5, 53 | "psfFlux": 1241.0, 54 | "psfFluxErr": 12.0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "visit": 1, 60 | "detector": 42, 61 | "midpointMjdTai": 1480360995, 62 | "band": "r", 63 | "programId": 1, 64 | "ra": 351.570546978, 65 | "dec": 0.126243049656, 66 | "raErr": 0.00028, 67 | "decErr": 0.00028, 68 | "ra_dec_Cov": 0.00029, 69 | "x": 112.1, 70 | "y": 121.1, 71 | "xErr": 1.2, 72 | "yErr": 1.1, 73 | "x_y_Cov": 1.2, 74 | "apFlux": 1241.0, 75 | "apFluxErr": 12.0, 76 | "snr": 41.1, 77 | "trailNdata": 5, 78 | "psfFlux": 1241.0, 79 | "psfFluxErr": 12.0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214 104 | }, 105 | "ssObject": { 106 | "ssObjectId":5364546, 107 | "numObs": 10 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/7/1/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/1/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v7_1.alert").get_by_version("7.1") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/2/lsst.v7_2.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_2", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "Rubin Avro alert schema v7.2", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v7_2.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v7_2.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v7_2.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v7_2.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v7_2.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v7_2.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutScience", "type": ["null", "bytes"], "default": null}, 22 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/2/lsst.v7_2.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_2", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Right ascension coordinate of the position of the DiaObject at time radecMjdTai.", 18 | "name": "ra", 19 | "type": "double" 20 | }, 21 | { 22 | "doc": "Declination coordinate of the position of the DiaObject at time radecMjdTai.", 23 | "name": "dec", 24 | "type": "double" 25 | }, 26 | { 27 | "doc": "Id of the visit where this forcedSource was measured.", 28 | "name": "visit", 29 | "type": "long" 30 | }, 31 | { 32 | "doc": "Id of the detector where this forcedSource was measured. Datatype short instead of byte because of DB concerns about unsigned bytes.", 33 | "name": "detector", 34 | "type": "int" 35 | }, 36 | { 37 | "default": null, 38 | "doc": "Point Source model flux.", 39 | "name": "psfFlux", 40 | "type": [ 41 | "null", 42 | "float" 43 | ] 44 | }, 45 | { 46 | "default": null, 47 | "doc": "Uncertainty of psfFlux.", 48 | "name": "psfFluxErr", 49 | "type": [ 50 | "null", 51 | "float" 52 | ] 53 | }, 54 | { 55 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 56 | "name": "midpointMjdTai", 57 | "type": "double" 58 | }, 59 | { 60 | "default": null, 61 | "doc": "Filter band this source was observed with.", 62 | "name": "band", 63 | "type": [ 64 | "null", 65 | "string" 66 | ] 67 | } 68 | ] 69 | } 70 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/2/lsst.v7_2.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_2", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/2/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "visit": 1, 9 | "detector": 42, 10 | "midpointMjdTai": 1480360995, 11 | "band": "r", 12 | "programId": 1, 13 | "ra": 351.570546978, 14 | "dec": 0.126243049656, 15 | "raErr": 0.00028, 16 | "decErr": 0.00028, 17 | "ra_dec_Cov": 0.00029, 18 | "x": 112.1, 19 | "y": 121.1, 20 | "xErr": 1.2, 21 | "yErr": 1.1, 22 | "x_y_Cov": 1.2, 23 | "apFlux": 1241.0, 24 | "apFluxErr": 12.0, 25 | "snr": 41.1, 26 | "trailNdata": 5, 27 | "psfFlux": 1241.0, 28 | "psfFluxErr": 12.0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "visit": 2, 35 | "detector": 43, 36 | "midpointMjdTai": 1480360995, 37 | "band": "r", 38 | "programId": 1, 39 | "ra": 351.570546978, 40 | "dec": 0.126243049656, 41 | "raErr": 0.00028, 42 | "decErr": 0.00028, 43 | "ra_dec_Cov": 0.00029, 44 | "x": 112.1, 45 | "y": 121.1, 46 | "xErr": 1.2, 47 | "yErr": 1.1, 48 | "x_y_Cov": 1.2, 49 | "apFlux": 1241.0, 50 | "apFluxErr": 12.0, 51 | "snr": 41.1, 52 | "trailNdata": 5, 53 | "psfFlux": 1241.0, 54 | "psfFluxErr": 12.0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "visit": 1, 60 | "detector": 42, 61 | "midpointMjdTai": 1480360995, 62 | "band": "r", 63 | "programId": 1, 64 | "ra": 351.570546978, 65 | "dec": 0.126243049656, 66 | "raErr": 0.00028, 67 | "decErr": 0.00028, 68 | "ra_dec_Cov": 0.00029, 69 | "x": 112.1, 70 | "y": 121.1, 71 | "xErr": 1.2, 72 | "yErr": 1.1, 73 | "x_y_Cov": 1.2, 74 | "apFlux": 1241.0, 75 | "apFluxErr": 12.0, 76 | "snr": 41.1, 77 | "trailNdata": 5, 78 | "psfFlux": 1241.0, 79 | "psfFluxErr": 12.0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214 104 | }, 105 | "ssObject": { 106 | "ssObjectId":5364546, 107 | "numObs": 10 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/2/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/7/2/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/2/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v7_2.alert").get_by_version("7.2") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/lsst.v7_3.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_3", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "Rubin Avro alert schema v7.3", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v7_3.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v7_3.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v7_3.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v7_3.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v7_3.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v7_3.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutScience", "type": ["null", "bytes"], "default": null}, 22 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/lsst.v7_3.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_3", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Right ascension coordinate of the position of the DiaObject at time radecMjdTai.", 18 | "name": "ra", 19 | "type": "double" 20 | }, 21 | { 22 | "doc": "Declination coordinate of the position of the DiaObject at time radecMjdTai.", 23 | "name": "dec", 24 | "type": "double" 25 | }, 26 | { 27 | "doc": "Id of the visit where this forcedSource was measured.", 28 | "name": "visit", 29 | "type": "long" 30 | }, 31 | { 32 | "doc": "Id of the detector where this forcedSource was measured. Datatype short instead of byte because of DB concerns about unsigned bytes.", 33 | "name": "detector", 34 | "type": "int" 35 | }, 36 | { 37 | "doc": "Point Source model flux.", 38 | "default": null, 39 | "name": "psfFlux", 40 | "type": [ 41 | "null", 42 | "float" 43 | ] 44 | }, 45 | { 46 | "doc": "Uncertainty of psfFlux.", 47 | "default": null, 48 | "name": "psfFluxErr", 49 | "type": [ 50 | "null", 51 | "float" 52 | ] 53 | }, 54 | { 55 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 56 | "name": "midpointMjdTai", 57 | "type": "double" 58 | }, 59 | { 60 | "doc": "Filter band this source was observed with.", 61 | "default": null, 62 | "name": "band", 63 | "type": [ 64 | "null", 65 | "string" 66 | ] 67 | } 68 | ] 69 | } 70 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/lsst.v7_3.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_3", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/lsst.v7_3.ssObject.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_3", 4 | "name": "ssObject", 5 | "fields": [ 6 | { 7 | "doc": "Unique identifier.", 8 | "name": "ssObjectId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "The date the LSST first linked and submitted the discovery observations to the MPC. May be NULL if not an LSST discovery. The date format will follow general LSST conventions (MJD TAI, at the moment).", 13 | "default": null, 14 | "name": "discoverySubmissionDate", 15 | "type": [ 16 | "null", 17 | "double" 18 | ] 19 | }, 20 | { 21 | "doc": "The time of the first LSST observation of this object (could be precovered) as Modified Julian Date, International Atomic Time.", 22 | "default": null, 23 | "name": "firstObservationDate", 24 | "type": [ 25 | "null", 26 | "double" 27 | ] 28 | }, 29 | { 30 | "doc": "Arc of LSST observations.", 31 | "default": null, 32 | "name": "arc", 33 | "type": [ 34 | "null", 35 | "float" 36 | ] 37 | }, 38 | { 39 | "doc": "Number of LSST observations of this object.", 40 | "default": null, 41 | "name": "numObs", 42 | "type": [ 43 | "null", 44 | "int" 45 | ] 46 | }, 47 | { 48 | "doc": "Minimum orbit intersection distance to Earth.", 49 | "default": null, 50 | "name": "MOID", 51 | "type": [ 52 | "null", 53 | "float" 54 | ] 55 | }, 56 | { 57 | "doc": "True anomaly of the MOID point.", 58 | "default": null, 59 | "name": "MOIDTrueAnomaly", 60 | "type": [ 61 | "null", 62 | "float" 63 | ] 64 | }, 65 | { 66 | "doc": "Ecliptic longitude of the MOID point.", 67 | "default": null, 68 | "name": "MOIDEclipticLongitude", 69 | "type": [ 70 | "null", 71 | "float" 72 | ] 73 | }, 74 | { 75 | "doc": "DeltaV at the MOID point.", 76 | "default": null, 77 | "name": "MOIDDeltaV", 78 | "type": [ 79 | "null", 80 | "float" 81 | ] 82 | }, 83 | { 84 | "doc": "Best fit absolute magnitude (u band).", 85 | "default": null, 86 | "name": "u_H", 87 | "type": [ 88 | "null", 89 | "float" 90 | ] 91 | }, 92 | { 93 | "doc": "Best fit G12 slope parameter (u band).", 94 | "default": null, 95 | "name": "u_G12", 96 | "type": [ 97 | "null", 98 | "float" 99 | ] 100 | }, 101 | { 102 | "doc": "Uncertainty of H (u band).", 103 | "default": null, 104 | "name": "u_HErr", 105 | "type": [ 106 | "null", 107 | "float" 108 | ] 109 | }, 110 | { 111 | "doc": "Uncertainty of G12 (u band).", 112 | "default": null, 113 | "name": "u_G12Err", 114 | "type": [ 115 | "null", 116 | "float" 117 | ] 118 | }, 119 | { 120 | "doc": "H-G12 covariance (u band).", 121 | "default": null, 122 | "name": "u_H_u_G12_Cov", 123 | "type": [ 124 | "null", 125 | "float" 126 | ] 127 | }, 128 | { 129 | "doc": "Chi^2 statistic of the phase curve fit (u band).", 130 | "default": null, 131 | "name": "u_Chi2", 132 | "type": [ 133 | "null", 134 | "float" 135 | ] 136 | }, 137 | { 138 | "doc": "The number of data points used to fit the phase curve (u band).", 139 | "default": null, 140 | "name": "u_Ndata", 141 | "type": [ 142 | "null", 143 | "int" 144 | ] 145 | }, 146 | { 147 | "doc": "Best fit absolute magnitude (g band).", 148 | "default": null, 149 | "name": "g_H", 150 | "type": [ 151 | "null", 152 | "float" 153 | ] 154 | }, 155 | { 156 | "doc": "Best fit G12 slope parameter (g band).", 157 | "default": null, 158 | "name": "g_G12", 159 | "type": [ 160 | "null", 161 | "float" 162 | ] 163 | }, 164 | { 165 | "doc": "Uncertainty of H (g band).", 166 | "default": null, 167 | "name": "g_HErr", 168 | "type": [ 169 | "null", 170 | "float" 171 | ] 172 | }, 173 | { 174 | "doc": "Uncertainty of G12 (g band).", 175 | "default": null, 176 | "name": "g_G12Err", 177 | "type": [ 178 | "null", 179 | "float" 180 | ] 181 | }, 182 | { 183 | "doc": "H-G12 covariance (g band).", 184 | "default": null, 185 | "name": "g_H_g_G12_Cov", 186 | "type": [ 187 | "null", 188 | "float" 189 | ] 190 | }, 191 | { 192 | "doc": "Chi^2 statistic of the phase curve fit (g band).", 193 | "default": null, 194 | "name": "g_Chi2", 195 | "type": [ 196 | "null", 197 | "float" 198 | ] 199 | }, 200 | { 201 | "doc": "The number of data points used to fit the phase curve (g band).", 202 | "default": null, 203 | "name": "g_Ndata", 204 | "type": [ 205 | "null", 206 | "int" 207 | ] 208 | }, 209 | { 210 | "doc": "Best fit absolute magnitude (r band).", 211 | "default": null, 212 | "name": "r_H", 213 | "type": [ 214 | "null", 215 | "float" 216 | ] 217 | }, 218 | { 219 | "doc": "Best fit G12 slope parameter (r band).", 220 | "default": null, 221 | "name": "r_G12", 222 | "type": [ 223 | "null", 224 | "float" 225 | ] 226 | }, 227 | { 228 | "doc": "Uncertainty of H (r band).", 229 | "default": null, 230 | "name": "r_HErr", 231 | "type": [ 232 | "null", 233 | "float" 234 | ] 235 | }, 236 | { 237 | "doc": "Uncertainty of G12 (r band).", 238 | "default": null, 239 | "name": "r_G12Err", 240 | "type": [ 241 | "null", 242 | "float" 243 | ] 244 | }, 245 | { 246 | "doc": "H-G12 covariance (r band).", 247 | "default": null, 248 | "name": "r_H_r_G12_Cov", 249 | "type": [ 250 | "null", 251 | "float" 252 | ] 253 | }, 254 | { 255 | "doc": "Chi^2 statistic of the phase curve fit (r band).", 256 | "default": null, 257 | "name": "r_Chi2", 258 | "type": [ 259 | "null", 260 | "float" 261 | ] 262 | }, 263 | { 264 | "doc": "The number of data points used to fit the phase curve (r band).", 265 | "default": null, 266 | "name": "r_Ndata", 267 | "type": [ 268 | "null", 269 | "int" 270 | ] 271 | }, 272 | { 273 | "doc": "Best fit absolute magnitude (i band).", 274 | "default": null, 275 | "name": "i_H", 276 | "type": [ 277 | "null", 278 | "float" 279 | ] 280 | }, 281 | { 282 | "doc": "Best fit G12 slope parameter (i band).", 283 | "default": null, 284 | "name": "i_G12", 285 | "type": [ 286 | "null", 287 | "float" 288 | ] 289 | }, 290 | { 291 | "doc": "Uncertainty of H (i band).", 292 | "default": null, 293 | "name": "i_HErr", 294 | "type": [ 295 | "null", 296 | "float" 297 | ] 298 | }, 299 | { 300 | "doc": "Uncertainty of G12 (i band).", 301 | "default": null, 302 | "name": "i_G12Err", 303 | "type": [ 304 | "null", 305 | "float" 306 | ] 307 | }, 308 | { 309 | "doc": "H-G12 covariance (i band).", 310 | "default": null, 311 | "name": "i_H_i_G12_Cov", 312 | "type": [ 313 | "null", 314 | "float" 315 | ] 316 | }, 317 | { 318 | "doc": "Chi^2 statistic of the phase curve fit (i band).", 319 | "default": null, 320 | "name": "i_Chi2", 321 | "type": [ 322 | "null", 323 | "float" 324 | ] 325 | }, 326 | { 327 | "doc": "The number of data points used to fit the phase curve (i band).", 328 | "default": null, 329 | "name": "i_Ndata", 330 | "type": [ 331 | "null", 332 | "int" 333 | ] 334 | }, 335 | { 336 | "doc": "Best fit absolute magnitude (z band).", 337 | "default": null, 338 | "name": "z_H", 339 | "type": [ 340 | "null", 341 | "float" 342 | ] 343 | }, 344 | { 345 | "doc": "Best fit G12 slope parameter (z band).", 346 | "default": null, 347 | "name": "z_G12", 348 | "type": [ 349 | "null", 350 | "float" 351 | ] 352 | }, 353 | { 354 | "doc": "Uncertainty of H (z band).", 355 | "default": null, 356 | "name": "z_HErr", 357 | "type": [ 358 | "null", 359 | "float" 360 | ] 361 | }, 362 | { 363 | "doc": "Uncertainty of G12 (z band).", 364 | "default": null, 365 | "name": "z_G12Err", 366 | "type": [ 367 | "null", 368 | "float" 369 | ] 370 | }, 371 | { 372 | "doc": "H-G12 covariance (z band).", 373 | "default": null, 374 | "name": "z_H_z_G12_Cov", 375 | "type": [ 376 | "null", 377 | "float" 378 | ] 379 | }, 380 | { 381 | "doc": "Chi^2 statistic of the phase curve fit (z band).", 382 | "default": null, 383 | "name": "z_Chi2", 384 | "type": [ 385 | "null", 386 | "float" 387 | ] 388 | }, 389 | { 390 | "doc": "The number of data points used to fit the phase curve (z band).", 391 | "default": null, 392 | "name": "z_Ndata", 393 | "type": [ 394 | "null", 395 | "int" 396 | ] 397 | }, 398 | { 399 | "doc": "Best fit absolute magnitude (y band).", 400 | "default": null, 401 | "name": "y_H", 402 | "type": [ 403 | "null", 404 | "float" 405 | ] 406 | }, 407 | { 408 | "doc": "Best fit G12 slope parameter (y band).", 409 | "default": null, 410 | "name": "y_G12", 411 | "type": [ 412 | "null", 413 | "float" 414 | ] 415 | }, 416 | { 417 | "doc": "Uncertainty of H (y band).", 418 | "default": null, 419 | "name": "y_HErr", 420 | "type": [ 421 | "null", 422 | "float" 423 | ] 424 | }, 425 | { 426 | "doc": "Uncertainty of G12 (y band).", 427 | "default": null, 428 | "name": "y_G12Err", 429 | "type": [ 430 | "null", 431 | "float" 432 | ] 433 | }, 434 | { 435 | "doc": "H-G12 covariance (y band).", 436 | "default": null, 437 | "name": "y_H_y_G12_Cov", 438 | "type": [ 439 | "null", 440 | "float" 441 | ] 442 | }, 443 | { 444 | "doc": "Chi^2 statistic of the phase curve fit (y band).", 445 | "default": null, 446 | "name": "y_Chi2", 447 | "type": [ 448 | "null", 449 | "float" 450 | ] 451 | }, 452 | { 453 | "doc": "The number of data points used to fit the phase curve (y band).", 454 | "default": null, 455 | "name": "y_Ndata", 456 | "type": [ 457 | "null", 458 | "int" 459 | ] 460 | }, 461 | { 462 | "doc": "median `extendedness` value from the DIASource.", 463 | "default": null, 464 | "name": "medianExtendedness", 465 | "type": [ 466 | "null", 467 | "float" 468 | ] 469 | } 470 | ] 471 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "visit": 1, 9 | "detector": 42, 10 | "midpointMjdTai": 1480360995, 11 | "band": "r", 12 | "programId": 1, 13 | "ra": 351.570546978, 14 | "dec": 0.126243049656, 15 | "raErr": 0.00028, 16 | "decErr": 0.00028, 17 | "ra_dec_Cov": 0.00029, 18 | "x": 112.1, 19 | "y": 121.1, 20 | "xErr": 1.2, 21 | "yErr": 1.1, 22 | "x_y_Cov": 1.2, 23 | "apFlux": 1241.0, 24 | "apFluxErr": 12.0, 25 | "snr": 41.1, 26 | "trailNdata": 5, 27 | "psfFlux": 1241.0, 28 | "psfFluxErr": 12.0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "visit": 2, 35 | "detector": 43, 36 | "midpointMjdTai": 1480360995, 37 | "band": "r", 38 | "programId": 1, 39 | "ra": 351.570546978, 40 | "dec": 0.126243049656, 41 | "raErr": 0.00028, 42 | "decErr": 0.00028, 43 | "ra_dec_Cov": 0.00029, 44 | "x": 112.1, 45 | "y": 121.1, 46 | "xErr": 1.2, 47 | "yErr": 1.1, 48 | "x_y_Cov": 1.2, 49 | "apFlux": 1241.0, 50 | "apFluxErr": 12.0, 51 | "snr": 41.1, 52 | "trailNdata": 5, 53 | "psfFlux": 1241.0, 54 | "psfFluxErr": 12.0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "visit": 1, 60 | "detector": 42, 61 | "midpointMjdTai": 1480360995, 62 | "band": "r", 63 | "programId": 1, 64 | "ra": 351.570546978, 65 | "dec": 0.126243049656, 66 | "raErr": 0.00028, 67 | "decErr": 0.00028, 68 | "ra_dec_Cov": 0.00029, 69 | "x": 112.1, 70 | "y": 121.1, 71 | "xErr": 1.2, 72 | "yErr": 1.1, 73 | "x_y_Cov": 1.2, 74 | "apFlux": 1241.0, 75 | "apFluxErr": 12.0, 76 | "snr": 41.1, 77 | "trailNdata": 5, 78 | "psfFlux": 1241.0, 79 | "psfFluxErr": 12.0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214 104 | }, 105 | "ssObject": { 106 | "ssObjectId":5364546, 107 | "numObs": 10 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/7/3/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/3/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v7_3.alert").get_by_version("7.3") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/4/lsst.v7_4.alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_4", 3 | "type": "record", 4 | "name": "alert", 5 | "doc": "Rubin Avro alert schema v7.4", 6 | "fields": [ 7 | {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, 8 | {"name": "diaSource", "type": "lsst.v7_4.diaSource"}, 9 | {"name": "prvDiaSources", "type": ["null", { 10 | "type": "array", 11 | "items": "lsst.v7_4.diaSource"}], "default": null}, 12 | {"name": "prvDiaForcedSources", "type": ["null", { 13 | "type": "array", 14 | "items": "lsst.v7_4.diaForcedSource"}], "default": null}, 15 | {"name": "prvDiaNondetectionLimits", "type": ["null", { 16 | "type": "array", 17 | "items": "lsst.v7_4.diaNondetectionLimit"}], "default": null}, 18 | {"name": "diaObject", "type": ["null", "lsst.v7_4.diaObject"], "default": null}, 19 | {"name": "ssObject", "type": ["null", "lsst.v7_4.ssObject"], "default": null}, 20 | {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, 21 | {"name": "cutoutScience", "type": ["null", "bytes"], "default": null}, 22 | {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/4/lsst.v7_4.diaForcedSource.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "lsst.v7_4", 4 | "name": "diaForcedSource", 5 | "fields": [ 6 | { 7 | "doc": "Unique id.", 8 | "name": "diaForcedSourceId", 9 | "type": "long" 10 | }, 11 | { 12 | "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", 13 | "name": "diaObjectId", 14 | "type": "long" 15 | }, 16 | { 17 | "doc": "Right ascension coordinate of the position of the DiaObject at time radecMjdTai.", 18 | "name": "ra", 19 | "type": "double" 20 | }, 21 | { 22 | "doc": "Declination coordinate of the position of the DiaObject at time radecMjdTai.", 23 | "name": "dec", 24 | "type": "double" 25 | }, 26 | { 27 | "doc": "Id of the visit where this forcedSource was measured.", 28 | "name": "visit", 29 | "type": "long" 30 | }, 31 | { 32 | "doc": "Id of the detector where this forcedSource was measured. Datatype short instead of byte because of DB concerns about unsigned bytes.", 33 | "name": "detector", 34 | "type": "int" 35 | }, 36 | { 37 | "doc": "Point Source model flux.", 38 | "default": null, 39 | "name": "psfFlux", 40 | "type": [ 41 | "null", 42 | "float" 43 | ] 44 | }, 45 | { 46 | "doc": "Uncertainty of psfFlux.", 47 | "default": null, 48 | "name": "psfFluxErr", 49 | "type": [ 50 | "null", 51 | "float" 52 | ] 53 | }, 54 | { 55 | "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", 56 | "name": "midpointMjdTai", 57 | "type": "double" 58 | }, 59 | { 60 | "doc": "Filter band this source was observed with.", 61 | "default": null, 62 | "name": "band", 63 | "type": [ 64 | "null", 65 | "string" 66 | ] 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/4/lsst.v7_4.diaNondetectionLimit.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.v7_4", 3 | "name": "diaNondetectionLimit", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "ccdVisitId", "type": "long"}, 7 | {"name": "midpointMjdTai", "type": "double"}, 8 | {"name": "band", "type": "string"}, 9 | {"name": "diaNoise", "type": "float"} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/4/sample_data/alert.json: -------------------------------------------------------------------------------- 1 | { 2 | "alertId": 1231321321, 3 | "l1dbId": 222222222, 4 | "diaSource": 5 | { 6 | "diaSourceId": 281323062375219200, 7 | "time_processed": 0, 8 | "visit": 1, 9 | "detector": 42, 10 | "midpointMjdTai": 1480360995, 11 | "band": "r", 12 | "programId": 1, 13 | "ra": 351.570546978, 14 | "dec": 0.126243049656, 15 | "raErr": 0.00028, 16 | "decErr": 0.00028, 17 | "ra_dec_Cov": 0.00029, 18 | "x": 112.1, 19 | "y": 121.1, 20 | "xErr": 1.2, 21 | "yErr": 1.1, 22 | "x_y_Cov": 1.2, 23 | "apFlux": 1241.0, 24 | "apFluxErr": 12.0, 25 | "snr": 41.1, 26 | "trailNdata": 5, 27 | "psfFlux": 1241.0, 28 | "psfFluxErr": 12.0 29 | }, 30 | "prvDiaSources": [ 31 | { 32 | "diaSourceId": 281323062375219198, 33 | "time_processed": 0, 34 | "visit": 2, 35 | "detector": 43, 36 | "midpointMjdTai": 1480360995, 37 | "band": "r", 38 | "programId": 1, 39 | "ra": 351.570546978, 40 | "dec": 0.126243049656, 41 | "raErr": 0.00028, 42 | "decErr": 0.00028, 43 | "ra_dec_Cov": 0.00029, 44 | "x": 112.1, 45 | "y": 121.1, 46 | "xErr": 1.2, 47 | "yErr": 1.1, 48 | "x_y_Cov": 1.2, 49 | "apFlux": 1241.0, 50 | "apFluxErr": 12.0, 51 | "snr": 41.1, 52 | "trailNdata": 5, 53 | "psfFlux": 1241.0, 54 | "psfFluxErr": 12.0 55 | }, 56 | { 57 | "diaSourceId": 281323062375219199, 58 | "time_processed": 0, 59 | "visit": 1, 60 | "detector": 42, 61 | "midpointMjdTai": 1480360995, 62 | "band": "r", 63 | "programId": 1, 64 | "ra": 351.570546978, 65 | "dec": 0.126243049656, 66 | "raErr": 0.00028, 67 | "decErr": 0.00028, 68 | "ra_dec_Cov": 0.00029, 69 | "x": 112.1, 70 | "y": 121.1, 71 | "xErr": 1.2, 72 | "yErr": 1.1, 73 | "x_y_Cov": 1.2, 74 | "apFlux": 1241.0, 75 | "apFluxErr": 12.0, 76 | "snr": 41.1, 77 | "trailNdata": 5, 78 | "psfFlux": 1241.0, 79 | "psfFluxErr": 12.0 80 | } 81 | ], 82 | "diaObject": { 83 | "diaObjectId": 281323062375219201, 84 | "validityStart": 0, 85 | "ra": 351.570546978, 86 | "dec": 0.126243049656, 87 | "raErr": 0.00028, 88 | "decErr": 0.00028, 89 | "ra_dec_Cov": 0.00029, 90 | "radecMjdTai": 1480360995, 91 | "pmRa": 0.00013, 92 | "pmDec": 0.00014, 93 | "parallax": 2.124124, 94 | "pmRaErr": 0.00013, 95 | "pmDecErr": 0.00013, 96 | "pmParallaxNdata": 0, 97 | "parallaxErr": 0.00013, 98 | "pmRa_pmDec_Cov": 0.00013, 99 | "pmRa_parallax_Cov": 0.00013, 100 | "pmDec_parallax_Cov": 0.00013, 101 | "pmParallaxLnL": 0.00013, 102 | "pmParallaxChi2": 0.00013, 103 | "pmParallaxNdata": 1214 104 | }, 105 | "ssObject": { 106 | "ssObjectId":5364546, 107 | "numObs": 10 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/4/sample_data/fakeAlert.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lsst/alert_packet/a9fcad5d897aa656264e80be13695ef9ac89f070/python/lsst/alert/packet/schema/7/4/sample_data/fakeAlert.avro -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/7/4/sample_data/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import lsst.alert.packet as packet 4 | from datetime import datetime 5 | import pandas as pd 6 | 7 | with open("alert.json", "r") as f: 8 | data = json.load(f) 9 | 10 | data_time = datetime.now() 11 | 12 | data['diaSource']['time_processed'] = pd.Timestamp(data_time) 13 | data['diaObject']['validityStart'] = pd.Timestamp(data_time) 14 | 15 | schema = packet.SchemaRegistry.from_filesystem(schema_root="lsst.v7_4.alert").get_by_version("7.4") 16 | with open("fakeAlert.avro", "wb") as f: 17 | schema.store_alerts(f, [data]) 18 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/examples/sample_data/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "diaSourceId":111111, 3 | "visit": 1, 4 | "detector": 42, 5 | "midpointMjdTai":12314.142412, 6 | "band":"my favorite filter", 7 | "radec": { 8 | "ra": 124142.12414, 9 | "dec": 1214141.121 10 | }, 11 | "raVar": 5.1, 12 | "decVar": 10.1, 13 | "ra_dec_Cov": 1.241, 14 | "x": 112.1, 15 | "y": 121.1, 16 | "xVar": 1.2, 17 | "yVar": 12.1, 18 | "x_y_Cov": 11.2, 19 | "snr": 41.1, 20 | "psfFlux": 1241.0 21 | } 22 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/examples/sample_data/simplest.json: -------------------------------------------------------------------------------- 1 | { 2 | "diaSourceId":111111, 3 | "visit": 1, 4 | "detector": 42, 5 | "diaObjectId":111111, 6 | "midpointMjdTai":12314.142412, 7 | "band":"my favorite filter", 8 | "radec": { 9 | "ra": 124142.12414, 10 | "dec": 1214141.121 11 | }, 12 | "raVar": 5.1, 13 | "decVar": 10.1, 14 | "ra_dec_Cov": 1.241, 15 | "x": 112.1, 16 | "y": 121.1, 17 | "xVar": 1.2, 18 | "yVar": 12.1, 19 | "x_y_Cov": 11.2, 20 | "snr": 41.1, 21 | "psfFlux": 1241.0 22 | } 23 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/examples/simple.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.alert", 3 | "type": "record", 4 | "name": "diaSource", 5 | "fields": [ 6 | {"name": "diaSourceId", "type": "long"}, 7 | {"name": "visit", "type": "long"}, 8 | {"name": "detector", "type": "int"}, 9 | {"name": "diaObjectId", "type": ["long", "null"], "default": null}, 10 | {"name": "ssObjectId", "type": ["long", "null"], "default": null}, 11 | {"name": "parentSourceId", "type": ["long", "null"], "default": null}, 12 | {"name": "midpointMjdTai", "type": "double"}, 13 | {"name": "band", "type": "string"}, 14 | {"name": "radec", "type": [{"type": "record", 15 | "name": "Radec", "fields": [ 16 | {"name": "ra", "type": "double"}, 17 | {"name": "dec", "type": "double"} 18 | ]}]}, 19 | {"name": "raVar", "type": "float"}, 20 | {"name": "decVar", "type": "float"}, 21 | {"name": "ra_dec_Cov", "type": "float"}, 22 | {"name": "x", "type": "float"}, 23 | {"name": "y", "type": "float"}, 24 | {"name": "xVar", "type": "float"}, 25 | {"name": "yVar", "type": "float"}, 26 | {"name": "x_y_Cov", "type": "float"}, 27 | {"name": "apMeanSb01", "aliases": ["base_CircularApertureFlux_3_0_flux"], "type": ["float", "null"], "default": null}, 28 | {"name": "apMeanSb02", "aliases": ["base_CircularApertureFlux_4_5_flux"], "type": ["float", "null"], "default": null}, 29 | {"name": "apMeanSb03", "aliases": ["base_CircularApertureFlux_6_0_flux"], "type": ["float", "null"], "default": null}, 30 | {"name": "apMeanSb04", "aliases": ["base_CircularApertureFlux_9_0_flux"], "type": ["float", "null"], "default": null}, 31 | {"name": "apMeanSb05", "aliases": ["base_CircularApertureFlux_12_0_flux"], "type": ["float", "null"], "default": null}, 32 | {"name": "apMeanSb06", "aliases": ["base_CircularApertureFlux_17_0_flux"], "type": ["float", "null"], "default": null}, 33 | {"name": "apMeanSb07", "aliases": ["base_CircularApertureFlux_25_0_flux"], "type": ["float", "null"], "default": null}, 34 | {"name": "apMeanSb08", "aliases": ["base_CircularApertureFlux_35_0_flux"], "type": ["float", "null"], "default": null}, 35 | {"name": "apMeanSb09", "aliases": ["base_CircularApertureFlux_50_0_flux"], "type": ["float", "null"], "default": null}, 36 | {"name": "apMeanSb10", "aliases": ["base_CircularApertureFlux_70_0_flux"], "type": ["float", "null"], "default": null}, 37 | {"name": "apMeanSb01Sigma", "aliases": ["base_CircularApertureFlux_3_0_fluxSigma"], "type": ["float", "null"], "default": null}, 38 | {"name": "apMeanSb02Sigma", "aliases": ["base_CircularApertureFlux_4_5_fluxSigma"], "type": ["float", "null"], "default": null}, 39 | {"name": "apMeanSb03Sigma", "aliases": ["base_CircularApertureFlux_6_0_fluxSigma"], "type": ["float", "null"], "default": null}, 40 | {"name": "apMeanSb04Sigma", "aliases": ["base_CircularApertureFlux_9_0_fluxSigma"], "type": ["float", "null"], "default": null}, 41 | {"name": "apMeanSb05Sigma", "aliases": ["base_CircularApertureFlux_12_0_fluxSigma"], "type": ["float", "null"], "default": null}, 42 | {"name": "apMeanSb06Sigma", "aliases": ["base_CircularApertureFlux_17_0_fluxSigma"], "type": ["float", "null"], "default": null}, 43 | {"name": "apMeanSb07Sigma", "aliases": ["base_CircularApertureFlux_25_0_fluxSigma"], "type": ["float", "null"], "default": null}, 44 | {"name": "apMeanSb08Sigma", "aliases": ["base_CircularApertureFlux_35_0_fluxSigma"], "type": ["float", "null"], "default": null}, 45 | {"name": "apMeanSb09Sigma", "aliases": ["base_CircularApertureFlux_50_0_fluxSigma"], "type": ["float", "null"], "default": null}, 46 | {"name": "apMeanSb10Sigma", "aliases": ["base_CircularApertureFlux_70_0_fluxSigma"], "type": ["float", "null"], "default": null}, 47 | {"name": "snr", "type": "float"}, 48 | {"name": "psfFlux", "type": "float"} 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/examples/simplest.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "lsst.alert", 3 | "type": "record", 4 | "name": "diaSource", 5 | "fields": [ 6 | {"name": "diaSourceId", "type": "long"}, 7 | {"name": "visit", "type": "long"}, 8 | {"name": "detector", "type": "int"}, 9 | {"name": "diaObjectId", "type": ["long", "null"], "default": null}, 10 | {"name": "midpointMjdTai", "type": "double"}, 11 | {"name": "band", "type": "string"}, 12 | {"name": "radec", "type": [{"type": "record", 13 | "name": "Radec", "fields": [ 14 | {"name": "ra", "type": "double"}, 15 | {"name": "dec", "type": "double"} 16 | ]}]}, 17 | {"name": "raVar", "type": "float"}, 18 | {"name": "decVar", "type": "float"}, 19 | {"name": "ra_dec_Cov", "type": "float"}, 20 | {"name": "x", "type": "float"}, 21 | {"name": "y", "type": "float"}, 22 | {"name": "xVar", "type": "float"}, 23 | {"name": "yVar", "type": "float"}, 24 | {"name": "x_y_Cov", "type": "float"}, 25 | {"name": "snr", "type": "float"}, 26 | {"name": "psfFlux", "type": "float"} 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schema/latest.txt: -------------------------------------------------------------------------------- 1 | 7.4 2 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/schemaRegistry.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | """Provide a lookup table for versioned alert schemas. 23 | """ 24 | 25 | import os 26 | 27 | __all__ = ["SchemaRegistry"] 28 | 29 | 30 | class SchemaRegistry: 31 | """A registry for alert schemas. 32 | 33 | When a schema is registered, the registry allocates it an ID. We can 34 | subsequently retrieve the schema by either the ID or by a version number. 35 | """ 36 | def __init__(self): 37 | self._version_to_id = {} 38 | self._id_to_schema = {} 39 | self._ids = [] 40 | 41 | def register_schema(self, schema, version): 42 | """Register a new schema in the registry. 43 | 44 | If an existing schema has the same ID, it is replaced. 45 | 46 | If an existing schema has the same version, the new schema replaces 47 | the old one under that version, but the old schema can still be 48 | accessed by ID. 49 | 50 | Parameters 51 | ---------- 52 | schema : `lsst.alert.packet.Schema` 53 | Alert schema to register. 54 | version : `str` 55 | Version of the schema being registered. By convention, this is of 56 | the form ``MAJOR.MINOR``, but any string can be used. 57 | 58 | Returns 59 | ------- 60 | schema_id : `int` 61 | The ID that has been allocated to the schema. 62 | """ 63 | schema_id = schema.get_schema_id() 64 | self._version_to_id[version] = schema_id 65 | self._ids.append(schema_id) 66 | self._id_to_schema[schema_id] = schema 67 | return schema_id 68 | 69 | def get_by_id(self, schema_id): 70 | """Return the schema corresponding to the given ID. 71 | 72 | Paramters 73 | --------- 74 | schema_id : `int` 75 | The ID by which to look up the schema. 76 | 77 | Returns 78 | ------- 79 | schema : `lsst.alert.packet.Schema` 80 | The corresponding schema. 81 | """ 82 | return self._id_to_schema[schema_id] 83 | 84 | def get_by_version(self, version): 85 | """Return the schema corresponding to the given version. 86 | 87 | Paramters 88 | --------- 89 | version : `str` 90 | The version by which to look up the schema. Corresponds to the 91 | version specified when the schema was registered. 92 | 93 | Returns 94 | ------- 95 | schema : `lsst.alert.packet.Schema` 96 | The corresponding schema. 97 | """ 98 | return self._id_to_schema[self._version_to_id[version]] 99 | 100 | @property 101 | def known_versions(self): 102 | """Return all the schema versions tracked by this registry. 103 | 104 | Returns 105 | ------- 106 | schemas : `set` of `str` 107 | Set of schema versions. 108 | """ 109 | return set(self._version_to_id) 110 | 111 | @property 112 | def known_ids(self): 113 | """Return all the schema ids tracked by this registry. 114 | 115 | Returns 116 | ------- 117 | schemas : `list` of `int` 118 | List of schema ids. 119 | """ 120 | return set(self._ids) 121 | 122 | @classmethod 123 | def from_filesystem(cls, root=None, schema_root="lsst.v7_4.alert"): 124 | """Populate a schema registry based on the filesystem. 125 | 126 | Walk the directory tree from the root provided, locating files named 127 | according to `schema_root_file`, and load the corresponding schemas 128 | into the registry. 129 | """ 130 | from .schema import Schema 131 | from .schema import get_schema_root 132 | if not root: 133 | root = get_schema_root() 134 | registry = cls() 135 | schema_root_file = schema_root + ".avsc" 136 | for root, dirs, files in os.walk(root, followlinks=False): 137 | if schema_root_file in files: 138 | schema = Schema.from_file(os.path.join(root, schema_root_file)) 139 | version = ".".join(root.split("/")[-2:]) 140 | registry.register_schema(schema, version) 141 | return registry 142 | 143 | @classmethod 144 | def all_schemas_from_filesystem(cls, root=None): 145 | """Populate a schema registry based on the filesystem. 146 | 147 | Walk the directory tree from the root provided, locating all schemas. 148 | """ 149 | from .schema import Schema 150 | from .schema import get_schema_root 151 | if not root: 152 | root = get_schema_root() 153 | registry = cls() 154 | for root, dirs, files in os.walk(root, followlinks=False): 155 | for file in files: 156 | if "alert.avsc" in file: 157 | schema = Schema.from_file(os.path.join(root, file)) 158 | version = ".".join(root.split("/")[-2:]) 159 | registry.register_schema(schema, version) 160 | return registry 161 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/simulate.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | """Generation of simulated Avro alrt packets. 23 | """ 24 | import random 25 | import string 26 | import numpy 27 | 28 | __all__ = ["simulate_alert"] 29 | 30 | 31 | def randomNull(): 32 | """Provide a random value of the Avro `null` type. 33 | """ 34 | return None 35 | 36 | 37 | def randomBoolean(): 38 | """Provide a random value of the Avro `boolean` type. 39 | """ 40 | return random.choice([True, False]) 41 | 42 | 43 | def randomInt(): 44 | """Provide a random value of the Avro (32 bit, signed) `int` type. 45 | """ 46 | return int(numpy.random.randint(-2**31, 2**31 - 1, dtype=numpy.int32)) 47 | 48 | 49 | def randomLong(): 50 | """Provide a random value of the Avro (64 bit, signed) `long` type. 51 | """ 52 | return int(numpy.random.randint(-2**63, 2**63 - 1, dtype=numpy.int64)) 53 | 54 | 55 | def randomFloat(): 56 | """Provide a random value of the Avro (32) bit `float` type. 57 | """ 58 | return float(numpy.float32(numpy.random.random())) 59 | 60 | 61 | def randomDouble(): 62 | """Provide a random value of the Avro `double` type. 63 | """ 64 | return float(numpy.float64(numpy.random.random())) 65 | 66 | 67 | def randomString(): 68 | """Provide a random value of the Avro `string` type. 69 | 70 | A string of length between 0 and 10 is returned. 71 | """ 72 | return ''.join(random.choice(string.ascii_letters) 73 | for _ in range(random.randint(0, 10))) 74 | 75 | 76 | def randomBytes(max_bytes=1000): 77 | """Provide a random value of the Avro `bytes` type. 78 | 79 | Up to `max_bytes` bytes are returned. 80 | """ 81 | return numpy.random.bytes(random.randint(0, max_bytes)) 82 | 83 | 84 | randomizerFunctionsByType = { 85 | 'null': randomNull, 86 | 'boolean': randomBoolean, 87 | 'int': randomInt, 88 | 'long': randomLong, 89 | 'float': randomFloat, 90 | 'double': randomDouble, 91 | 'string': randomString, 92 | 'bytes': randomBytes 93 | } 94 | 95 | 96 | def simulate_alert(schema, keepNull=None, arrayCount=None): 97 | """Parse the schema and generate a compliant alert with random contents. 98 | 99 | Parameters 100 | ---------- 101 | schema : `dict` 102 | Schema to which to conform. Should be fully resolved (i.e., not 103 | contain any references to special LSST types). 104 | keepNull : {`list` of `str`, `None`} 105 | Schema keys for which to output null values. 106 | arrayCount : {`dict`, `None`} 107 | Number of array items to randomly generate for each provided schema 108 | key. 109 | 110 | Returns 111 | ------- 112 | output : `dict` 113 | Packet with random values corresponding to provided schema. 114 | 115 | Notes 116 | ----- 117 | `keepNull` and `arrayCount` expect schema keys without namespaces 118 | (e.g., `'diaSourceId'`, not `'lsst.alert.diaSource.diaSourceId'`. 119 | This is sufficient because our schemas have unique keys but is 120 | not fully general. 121 | 122 | Todo 123 | ---- 124 | This should accept an instance of `lsst.alert.Schema` and do whatever 125 | resolution is necessary internally. 126 | """ 127 | 128 | output = {} 129 | 130 | if keepNull is None: 131 | keepNull = [] 132 | 133 | if arrayCount is None: 134 | arrayCount = {} 135 | 136 | if type(schema['type']) is list: 137 | # potentially nullable 138 | if ('null' in schema['type']) and (schema['name'] in keepNull): 139 | return {schema['name']: None} 140 | else: 141 | # inferring the type like this this is not general but works 142 | # for our application 143 | schema['type'] = schema['type'][0] 144 | 145 | if type(schema['type']) is dict: 146 | # either an array, a record, or a nested type 147 | if schema['type']['type'] == 'array': 148 | if schema['name'] in arrayCount: 149 | output_array = [] 150 | for i in range(arrayCount[schema['name']]): 151 | output_array.append(simulate_alert( 152 | schema['type']['items'], keepNull=keepNull, arrayCount=arrayCount)) 153 | return {schema['name']: output_array} 154 | else: 155 | return {schema['name']: None} 156 | elif schema['type'] == 'record': 157 | for field in schema['fields']: 158 | output.update(simulate_alert(field, keepNull=keepNull, arrayCount=arrayCount)) 159 | return output 160 | else: 161 | # a nested type 162 | output.update({schema['name']: simulate_alert( 163 | schema['type'], keepNull=keepNull, arrayCount=arrayCount)}) 164 | return output 165 | 166 | if schema['type'] == 'record': 167 | for field in schema['fields']: 168 | output.update(simulate_alert(field, keepNull=keepNull, arrayCount=arrayCount)) 169 | return output 170 | elif schema['type'] in randomizerFunctionsByType.keys(): 171 | return {schema['name']: randomizerFunctionsByType[schema['type']]()} 172 | 173 | raise ValueError('Parsing broke...') 174 | -------------------------------------------------------------------------------- /python/lsst/alert/packet/updateSchema.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | import argparse 23 | import os 24 | import fastavro 25 | import yaml 26 | import json 27 | 28 | 29 | __all__ = ['generate_schema'] 30 | 31 | 32 | def write_schema(schema, path): 33 | 34 | if not os.path.exists(path): 35 | if not os.path.exists(path[0:-2]): 36 | os.mkdir(path[0:-2]) 37 | os.mkdir(path) 38 | else: 39 | os.mkdir(path) 40 | 41 | updated_schema_name = schema['namespace'] + '.' + schema['name'] + '.avsc' 42 | 43 | with open(path + '/' + updated_schema_name, "w") as f: 44 | json.dump(schema, f, indent=2) 45 | 46 | 47 | def add_namespace(schema): 48 | 49 | schema_names = schema['name'].split(".") 50 | schema['name'] = schema_names[2] 51 | schema['namespace'] = schema_names[0] + "." + schema_names[1] 52 | schema = dict(sorted(schema.items(), reverse=True)) 53 | 54 | return schema 55 | 56 | 57 | def populate_fields(apdb_table): 58 | """Make a dictionary of fields to populate the avro schema. At present, a 59 | number of rules must be included to ensure certain fields are excluded. 60 | 61 | Parameters 62 | ---------- 63 | apdb_table: `dict` 64 | The dictionary used to generate the avro schema. 65 | """ 66 | 67 | field_dictionary_array = [] 68 | for column in apdb_table['columns']: 69 | # We are still finalizing the time series feature names. 70 | 71 | excluded_fields = ['validityStart', 'validityEnd', 'Periodic', 'max', 'min', 72 | 'Science', 'Percentile', 'Max', 'Min', 'science', 'LowzGal', 73 | 'MAD', 'Skew', 'Intercept', 'Slope', 'Stetson', 'lastNonForcedSource', 74 | 'nDiaSources', 'ExtObj', 'isDipole', 'bboxSize', 'Time', 'time_'] 75 | exclude = False 76 | for excluded_field in excluded_fields: 77 | if excluded_field in column['name']: 78 | exclude = True 79 | 80 | if not exclude: 81 | if 'char' in column['datatype']: 82 | column['datatype'] = 'string' 83 | 84 | if 'short' in column['datatype']: 85 | column['datatype'] = 'int' 86 | 87 | if 'timestamp' in column['datatype']: 88 | column['datatype'] = {'type': 'long', 'logicalType': 'timestamp-micros'} 89 | else: 90 | column['datatype'] = str(column['datatype']) 91 | 92 | # Check if a column is nullable. If it is, it needs a default. 93 | if 'nullable' in column: 94 | if column['nullable'] is False: 95 | # Check if a column has a description, if so, include "doc" 96 | if 'description' in column: 97 | field = {'name': column['name'], 98 | 'type': column['datatype'], 99 | 'doc': column['description']} 100 | else: 101 | field = {'name': column['name'], 102 | 'type': column['datatype'], 'doc': ''} 103 | else: # nullable == True 104 | if 'description' in column: 105 | field = {'name': column['name'], 106 | 'type': ['null', column['datatype']], 107 | 'doc': column['description'], 'default': None} 108 | else: 109 | field = {'name': column['name'], 110 | 'type': ['null', column['datatype']], 111 | 'doc': '', 'default': None} 112 | else: # nullable not in columns (nullable == True) 113 | if 'description' in column: 114 | field = {'name': column['name'], 115 | 'type': ['null', column['datatype']], 116 | 'doc': column['description'], 'default': None} 117 | 118 | else: 119 | field = {"name": column['name'], 120 | "type": ["null", column["datatype"]], 121 | "doc": "", "default": None} 122 | 123 | field_dictionary_array.append(field) 124 | 125 | return field_dictionary_array 126 | 127 | 128 | def create_schema(name, field_dictionary_list, version): 129 | """ Create a schema using a field dictionary. fastavro will automatically 130 | take the name and namespace and put them as one, so the name should just be 131 | the schema name and the namespace needs to be created separately. The 132 | fastavro keys also need to be removed from the schema. 133 | 134 | Parameters 135 | ---------- 136 | name: `string` 137 | The name of the schema as a string. (e.g., `'diaSource'`). 138 | 139 | field_dictionary_list: 'list' 140 | A list containing dictionary entries for the individual fields. 141 | 142 | version: 'string' 143 | The version number of the schema. 144 | """ 145 | name = name[0:2].lower() + name[2:] 146 | schema = fastavro.parse_schema({ 147 | "name": name, 148 | "type": "record", 149 | "fields": field_dictionary_list 150 | }) 151 | 152 | schema['namespace'] = 'lsst.v' + version 153 | fastavro_keys = list(schema.keys()) 154 | for key in fastavro_keys: 155 | if '__' in key and '__len__' not in key: 156 | schema.pop(key) 157 | 158 | schema = dict(sorted(schema.items(), reverse=True)) 159 | 160 | return schema 161 | 162 | 163 | def generate_schema(apdb_filepath, schema_path, schema_version): 164 | """Generate avro schemas using an apdb.yaml file. 165 | 166 | Using a provided path to the apdb.yaml file and schema folder, 167 | generate a new schema, using a provided version number. 168 | 169 | Parameters 170 | ---------- 171 | apdb_filepath: `string` 172 | Input path to the apdb.yaml file which contains the information 173 | used to generate the new schemas. 174 | Example: path/to/sdm_schemas/yml/apdb.yaml 175 | 176 | schema_path: `string` 177 | Input path to the schema folder where the new schemas will 178 | be added. 179 | Example: /path/to/alert_packet/python/lsst/alert/packet/schema 180 | 181 | schema_version: 'string' 182 | Provide the version number of the schema as a string. 183 | Example: "5.1" 184 | 185 | """ 186 | 187 | path = os.path.join(schema_path, *schema_version.split(".")) 188 | 189 | with open(apdb_filepath, 'r') as file: 190 | apdb = yaml.safe_load(file) 191 | 192 | version_name = schema_version.split(".")[0] + "_" + schema_version.split(".")[1] 193 | 194 | table_names = ['DiaForcedSource', 'DiaObject', 'DiaSource', 'SSObject'] 195 | for name in table_names: 196 | 197 | for table in apdb['tables']: 198 | if table['name'] == name: 199 | field_dictionary = populate_fields(table) 200 | schema = create_schema(name, field_dictionary, version_name) 201 | write_schema(schema, path) 202 | 203 | 204 | if __name__ == '__main__': 205 | 206 | parser = argparse.ArgumentParser(description='Generate a schema using an apdb.yaml as the source' 207 | 'of truth and include a desired alert version number' 208 | 'Example input:' 209 | 'python3 updateSchema.py ' 210 | 'Path/To/Yaml/sdm_schemas/yml/apdb.yaml ' 211 | 'Path/To/alert_packet/lsst/alert/packet/schema "6.0"') 212 | parser.add_argument('apdb_filepath') 213 | parser.add_argument('schema_path') 214 | parser.add_argument('schema_version') 215 | 216 | args = parser.parse_args() 217 | 218 | generate_schema(args.apdb_filepath, args.schema_path, args.schema_version) 219 | -------------------------------------------------------------------------------- /schema: -------------------------------------------------------------------------------- 1 | ./python/lsst/alert/packet/schema -------------------------------------------------------------------------------- /script/docker-tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Determine the tag for Docker images. Takes the Git ref as its only 4 | # argument. 5 | 6 | set -eo pipefail 7 | 8 | if [ -n "$GITHUB_HEAD_REF" ]; then 9 | # For pull requests 10 | echo ${GITHUB_HEAD_REF} | sed -E 's,/,-,g' 11 | else 12 | # For push events 13 | echo ${GITHUB_REF} | sed -E 's,refs/(heads|tags)/,,' | sed -E 's,/,-,g' 14 | fi 15 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = lsst-alert-packet 3 | version = 0.3.0 4 | description = Code for interacting with Vera C. Rubin Observatory alert packets 5 | url = https://github.com/lsst/alert_packet 6 | classifiers = 7 | Programming Language :: Python :: 3 8 | License :: OSI Approved :: GNU General Public License v3 (GPLv3) 9 | Development Status :: 3 - Alpha 10 | author = Spencer Nelson 11 | author_email = swnelson@uw.edu 12 | license = GPLv3 13 | 14 | [options] 15 | install_requires = 16 | fastavro 17 | numpy 18 | requests 19 | lsst-resources 20 | packages = 21 | lsst.alert.packet 22 | lsst.alert.packet.bin 23 | package_dir = 24 | =python 25 | 26 | [options.package_data] 27 | lsst.alert.packet = 28 | schema 29 | schema/* 30 | schema/*/* 31 | schema/*/*/* 32 | schema/*/*/sample_data/* 33 | 34 | [options.entry_points] 35 | console_scripts = 36 | validateAvroRoundTrip.py = lsst.alert.packet.bin.validateAvroRoundTrip:main 37 | simulateAlerts.py = lsst.alert.packet.bin.simulateAlerts:main 38 | syncAllSchemasToRegistry.py = lsst.alert.packet.bin.syncAllSchemasToRegistry:main 39 | 40 | [flake8] 41 | max-line-length = 110 42 | max-doc-length = 79 43 | ignore = E133, E226, E228, N802, N803, N806, N812, N813, N815, N816, W503 44 | exclude = 45 | __init__.py 46 | doc/conf.py 47 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | # All configuration is in setup.cfg. 4 | setup() 5 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | -------------------------------------------------------------------------------- /test/test_io.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | from contextlib import contextmanager 23 | import io 24 | import unittest 25 | import tempfile 26 | import posixpath 27 | import fastavro # noqa: F401 28 | import json 29 | from lsst.alert.packet.io import retrieve_alerts 30 | from lsst.alert.packet.schema import Schema 31 | from lsst.alert.packet import get_latest_schema_version, get_schema_path 32 | 33 | 34 | class RetrieveAlertsTestCase(unittest.TestCase): 35 | def setUp(self): 36 | """Prepare the test with sample alerts. 37 | 38 | It's important to use the actual alert schema here because the 39 | retrieve_alerts function calls Schema.__init__, which resets fastavro's 40 | SCHEMA_DEFS cache. That cache only gets used for complex schemas which 41 | use named references to types, so a simple mock record type is not 42 | sufficient. 43 | """ 44 | self.test_schema_version = get_latest_schema_version() 45 | self.test_schema = Schema.from_file() 46 | sample_json_path = posixpath.join( 47 | get_schema_path(*self.test_schema_version), "sample_data", "alert.json", 48 | ) 49 | with open(sample_json_path, "r") as f: 50 | self.sample_alert = json.load(f) 51 | 52 | def _mock_alerts(self, n): 53 | """Return a list of alerts with mock values, matching 54 | self.sample_alert. 55 | """ 56 | alerts = [] 57 | for i in range(n): 58 | alert = self.sample_alert.copy() 59 | alert["alertId"] = i 60 | alerts.append(alert) 61 | return alerts 62 | 63 | def assert_alert_lists_equal(self, have_alerts, want_alerts): 64 | """Assert that two lists of mock alerts are equal - or at least, 65 | equal enough. 66 | 67 | We can't naively do `self.assertEqual(have_alerts, want_alerts)` 68 | because fastavro will explicitly populate an alert with "None" for 69 | every optional field when deserializing it. The sample alert.json files 70 | don't have those explicit Nones, and constructing them automatically 71 | seems complex. 72 | 73 | A simple check is just that the two lists have the same length and that 74 | the alertIds match. alertId is the only field that differs in a batch 75 | of mock data created with self._mock_alerts, so this is probably 76 | sufficient. 77 | """ 78 | self.assertEqual(len(have_alerts), len(want_alerts)) 79 | for i in range(len(have_alerts)): 80 | self.assertEqual( 81 | have_alerts[i]["alertId"], want_alerts[i]["alertId"], 82 | f"alert idx={i} has mismatched IDs", 83 | ) 84 | 85 | @contextmanager 86 | def _temp_alert_file(self, alerts): 87 | with tempfile.TemporaryFile(mode="w+b") as alert_file: 88 | self.test_schema.store_alerts(alert_file, alerts) 89 | alert_file.seek(0) 90 | yield alert_file 91 | 92 | def test_bad_read(self): 93 | """Check that we throw a useful exception on failure. 94 | """ 95 | # This should work both if the stream has a name... 96 | self.assertRaises(RuntimeError, retrieve_alerts, 97 | tempfile.NamedTemporaryFile()) 98 | 99 | # ...and if it doesn't. 100 | self.assertRaises(RuntimeError, retrieve_alerts, io.IOBase()) 101 | 102 | def test_retrieve_alerts(self): 103 | """Write some alerts to a file. They should be readable back out. 104 | """ 105 | alerts = self._mock_alerts(5) 106 | with self._temp_alert_file(alerts) as alert_file: 107 | have_schema, have_alerts_iterable = retrieve_alerts(alert_file, self.test_schema) 108 | have_alerts = list(have_alerts_iterable) 109 | 110 | self.assert_alert_lists_equal(alerts, list(have_alerts)) 111 | 112 | fastavro_keys = list(self.test_schema.definition.keys()) 113 | for key in fastavro_keys: 114 | if '__' in key and '__len__' not in key: 115 | self.test_schema.definition.pop(key) 116 | 117 | self.assertEqual(self.test_schema.definition, have_schema.definition) 118 | 119 | def test_alert_file_with_one_alert(self): 120 | """Write a single alert to a file. It should be readable back out. 121 | """ 122 | alerts = self._mock_alerts(1) 123 | 124 | with self._temp_alert_file(alerts) as alert_file: 125 | have_schema, have_alerts_iterable = retrieve_alerts(alert_file, self.test_schema) 126 | have_alerts = list(have_alerts_iterable) 127 | 128 | self.assert_alert_lists_equal(alerts, list(have_alerts)) 129 | 130 | fastavro_keys = list(self.test_schema.definition.keys()) 131 | for key in fastavro_keys: 132 | if '__' in key and '__len__' not in key: 133 | self.test_schema.definition.pop(key) 134 | 135 | self.assertEqual(self.test_schema.definition, have_schema.definition) 136 | 137 | def test_alert_file_with_no_alerts(self): 138 | """Write an alert file that contains no alerts at all. It should be 139 | readable. 140 | """ 141 | alerts = [] 142 | 143 | with self._temp_alert_file(alerts) as alert_file: 144 | have_schema, have_alerts_iterable = retrieve_alerts(alert_file, self.test_schema) 145 | have_alerts = list(have_alerts_iterable) 146 | 147 | self.assert_alert_lists_equal(alerts, list(have_alerts)) 148 | 149 | fastavro_keys = list(self.test_schema.definition.keys()) 150 | for key in fastavro_keys: 151 | if '__' in key and '__len__' not in key: 152 | self.test_schema.definition.pop(key) 153 | 154 | self.assertEqual(self.test_schema.definition, have_schema.definition) 155 | -------------------------------------------------------------------------------- /test/test_schema.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | import os.path 23 | import unittest 24 | 25 | import fastavro 26 | 27 | from lsst.alert.packet import get_schema_root, Schema, get_path_to_latest_schema 28 | 29 | 30 | class SchemaRootTestCase(unittest.TestCase): 31 | """Test for get_schema_root(). 32 | """ 33 | 34 | def test_get_schema_root(self): 35 | self.assertTrue(os.path.isdir(get_schema_root())) 36 | 37 | 38 | class PathLatestSchemTestCase(unittest.TestCase): 39 | """Test for get_path_to_latest_schema(). 40 | """ 41 | 42 | def test_path_latest_schema(self): 43 | self.assertTrue(os.path.isfile(get_path_to_latest_schema())) 44 | 45 | 46 | class ResolveTestCase(unittest.TestCase): 47 | """Test for schema resolution. 48 | """ 49 | 50 | def test_recursive_resolve(self): 51 | """Check that resolution of nested schemas gives the expected result. 52 | """ 53 | # Definition of schemas in terms of each other. 54 | named_schemas = {} 55 | sub_sub_schema = fastavro.parse_schema({ # noqa: F841 56 | "name": "subsub", 57 | "namespace": "lsst", 58 | "type": "record", 59 | "fields": [ 60 | {"name": "sub_sub_field", "type": "string"} 61 | ] 62 | }, named_schemas) 63 | 64 | sub_schema = fastavro.parse_schema({ # noqa: F841 65 | "name": "sub", 66 | "namespace": "lsst", 67 | "type": "record", 68 | "fields": [ 69 | {"name": "sub_field", "type": "lsst.subsub"}, 70 | {"name": "second_sub_field", "type": "lsst.subsub"} 71 | ] 72 | }, named_schemas) 73 | 74 | top_schema = fastavro.parse_schema({ 75 | "name": "top", 76 | "namespace": "lsst", 77 | "type": "record", 78 | "fields": [ 79 | {"name": "top_field", "type": "lsst.sub"}, 80 | {"name": "boring_field", "type": "int"} 81 | ] 82 | }, named_schemas) 83 | 84 | # Derived by substituting the above into each other by hand. 85 | model_resolved_schema = { 86 | "type": "record", 87 | "name": "lsst.top", 88 | "fields": [ 89 | { 90 | "name": "top_field", 91 | "type": { 92 | "type": "record", 93 | "name": "lsst.sub", 94 | "fields": [ 95 | { 96 | "name": "sub_field", 97 | "type": { 98 | "type": "record", 99 | "name": "lsst.subsub", 100 | "fields": [ 101 | { 102 | "name": "sub_sub_field", 103 | "type": "string" 104 | } 105 | ] 106 | } 107 | }, 108 | { 109 | "name": "second_sub_field", 110 | "type": { 111 | "type": "record", 112 | "name": "lsst.subsub", 113 | "fields": [ 114 | { 115 | "name": "sub_sub_field", 116 | "type": "string" 117 | } 118 | ] 119 | } 120 | } 121 | ] 122 | } 123 | }, 124 | { 125 | "name": "boring_field", 126 | "type": "int" 127 | } 128 | ] 129 | } 130 | 131 | resolved_schema = Schema(top_schema).definition 132 | partially_resolved_schema = fastavro.schema.expand_schema(resolved_schema) 133 | fully_resolved_schema = fastavro.schema.expand_schema(partially_resolved_schema) 134 | 135 | fastavro_keys = list(fully_resolved_schema.keys()) 136 | for key in fastavro_keys: 137 | if '__' in key and '__len__' not in key: 138 | fully_resolved_schema.pop(key) 139 | 140 | self.assertEqual(fully_resolved_schema, model_resolved_schema) 141 | -------------------------------------------------------------------------------- /test/test_schemaRegistry.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | import json 23 | import os 24 | import unittest 25 | from tempfile import TemporaryDirectory 26 | 27 | from lsst.alert.packet import SchemaRegistry 28 | 29 | 30 | def write_schema(root_dir, filename, version_major, version_minor): 31 | """Write a simple schema to a filesystem location based on its version. 32 | """ 33 | # Generate a new schema for each version to avoid ID collisions. 34 | schema = { 35 | "name": "example", 36 | "namespace": f"lsst.v{version_major}_{version_minor}", 37 | "type": "record", 38 | "fields": [ 39 | {"name": "field%s%s" % (version_major, version_minor), 40 | "type": "int"} 41 | ] 42 | } 43 | target_dir = os.path.join(root_dir, version_major, version_minor) 44 | os.makedirs(target_dir) 45 | with open(os.path.join(target_dir, filename), "w") as f: 46 | json.dump(schema, f) 47 | 48 | 49 | def create_filesystem_hierarchy(root_dir, root_file="lsst.example.avsc"): 50 | """Create a simple schema hierarchy on the filesystem. 51 | """ 52 | write_schema(root_dir, root_file, "1", "0") 53 | write_schema(root_dir, root_file, "2", "0") 54 | write_schema(root_dir, root_file, "2", "1") 55 | 56 | 57 | class FromFilesystemTestCase(unittest.TestCase): 58 | """Demonstrate that the SchemaRegistry can work with on-disk data. 59 | """ 60 | 61 | def test_from_filesystem(self): 62 | """Check priming a registry based on a simple filesystem hierarchy. 63 | """ 64 | versions = ("1.0", "2.0", "2.1") 65 | 66 | with TemporaryDirectory() as tempdir: 67 | create_filesystem_hierarchy(tempdir) 68 | registry = SchemaRegistry.from_filesystem(tempdir, 69 | schema_root="lsst.example") 70 | 71 | self.assertEqual(len(registry.known_versions), 3) 72 | for version in versions: 73 | self.assertIn(version, registry.known_versions) 74 | 75 | for version in versions: 76 | registry.get_by_version(version) 77 | self.assertRaises(KeyError, registry.get_by_version, "2.2") 78 | 79 | for id in registry.known_ids: 80 | registry.get_by_id(id) 81 | self.assertRaises(KeyError, registry.get_by_id, "202") 82 | -------------------------------------------------------------------------------- /test/test_schemas.py: -------------------------------------------------------------------------------- 1 | # This file is part of alert_packet. 2 | # 3 | # Developed for the LSST Data Management System. 4 | # This product includes software developed by the LSST Project 5 | # (https://www.lsst.org). 6 | # See the COPYRIGHT file at the top-level directory of this distribution 7 | # for details of code ownership. 8 | # 9 | # This program is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | import os.path 23 | import unittest 24 | 25 | import json 26 | 27 | from lsst.alert.packet import get_schema_root, SchemaRegistry 28 | 29 | 30 | def path_to_sample_data(schema_root, version, filename): 31 | return os.path.join(schema_root, *version.split("."), 32 | "sample_data", filename) 33 | 34 | 35 | class SchemaValidityTestCase(unittest.TestCase): 36 | """Check that schemas can read and write sample alert data. 37 | """ 38 | 39 | def setUp(self): 40 | self.registry = SchemaRegistry.from_filesystem() 41 | 42 | def test_example_json(self): 43 | """Test that example data in JSON format can be loaded by the schema. 44 | """ 45 | no_data = ("1.0",) # No example data is available. 46 | 47 | for version in self.registry.known_versions: 48 | path = path_to_sample_data(get_schema_root(), version, "alert.json") 49 | schema = self.registry.get_by_version(version) # noqa: F841 50 | if version in no_data: 51 | self.assertFalse(os.path.exists(path)) 52 | else: 53 | with open(path, "r") as f: 54 | data = json.load(f) 55 | self.assertTrue(self.registry.get_by_version(version).validate(data)) 56 | 57 | def test_example_avro(self): 58 | """Test that example data in Avro format can be loaded by the schema. 59 | """ 60 | no_data = ("1.0",) # No example data is available. 61 | bad_versions = ("2.0",) # This data is known not to parse. 62 | 63 | for version in self.registry.known_versions: 64 | path = path_to_sample_data(get_schema_root(), version, 65 | "fakeAlert.avro") 66 | schema = self.registry.get_by_version(version) 67 | 68 | if version in no_data: 69 | self.assertFalse(os.path.exists(path)) 70 | else: 71 | with open(path, "rb") as f: 72 | if version in bad_versions: 73 | with self.assertRaises(RuntimeError): 74 | schema.retrieve_alerts(f) 75 | else: 76 | retrieved_schema, alerts = schema.retrieve_alerts(f) 77 | 78 | fastavro_keys = list(schema.definition.keys()) 79 | for key in fastavro_keys: 80 | if '__' in key and '__len__' not in key: 81 | schema.definition.pop(key) 82 | 83 | self.assertEqual(retrieved_schema, schema, 84 | f"schema not equal on version={version}") 85 | for idx, alert in enumerate(alerts): 86 | self.assertTrue(schema.validate(alert), 87 | f"failed to validate version={version}, alert idx={idx}") 88 | -------------------------------------------------------------------------------- /test/test_wheel_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Test that the project can be build as a wheel, and that the wheel can be 4 | # installed, and that validateAvroRoundTrip exits cleanly after installing the 5 | # wheel. 6 | set -e 7 | 8 | REPO_ROOT=$(git rev-parse --show-toplevel) 9 | 10 | TEMP_VENV_DIR=$(mktemp -d) 11 | virtualenv $TEMP_VENV_DIR 12 | source $TEMP_VENV_DIR/bin/activate 13 | 14 | TEMP_DIST_DIR=$(mktemp -d) 15 | TEMP_BUILD_DIR=$(mktemp -d) 16 | python $REPO_ROOT/setup.py bdist_wheel --dist-dir=$TEMP_DIST_DIR --bdist-dir=$TEMP_BUILD_DIR 17 | 18 | pip install $TEMP_DIST_DIR/* 19 | validateAvroRoundTrip.py 20 | rm -rf $TEMP_DIST_DIR 21 | rm -rf $TEMP_BUILD_DIR 22 | 23 | deactivate 24 | rm -rf $TEMP_VENV_DIR 25 | -------------------------------------------------------------------------------- /ups/alert_packet.table: -------------------------------------------------------------------------------- 1 | envPrepend(PYTHONPATH, ${PRODUCT_DIR}/python) 2 | envPrepend(PATH, ${PRODUCT_DIR}/bin) 3 | 4 | # These are necessary to generate a new alert schema version, but the package 5 | # is stand-alone otherwise. 6 | setupOptional(resources) 7 | setupOptional(sdm_schemas) 8 | -------------------------------------------------------------------------------- /ups/eupspkg.cfg.sh: -------------------------------------------------------------------------------- 1 | build() { 2 | msg "No build needed; skipping default" 3 | } 4 | 5 | install() { 6 | # Do not auto-detect the build system; use eupspkg's "just copy 7 | # everything" approach. 8 | 9 | die_if_empty PRODUCT 10 | die_if_empty VERSION 11 | 12 | clean_old_install 13 | 14 | # just copy everything, except for the ups directory 15 | mkdir -p "$PREFIX" 16 | cp -a ./ "$PREFIX" 17 | rm -rf "$PREFIX/ups" 18 | msg "Copied the product into '$PREFIX'" 19 | 20 | cd "$reldir" 21 | 22 | install_ups 23 | } 24 | --------------------------------------------------------------------------------