├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── edge ├── README.md ├── artifacts │ ├── Edge scenario radar chart.png │ ├── Scope of Edge.png │ ├── onnx-edge-scope-and-profile-definition.md │ └── onnx-edge-wg-sow.md ├── discussion_materials │ ├── Edge scenario radar chart.png │ ├── ONNX Edge Execution Profile Compliance.md │ ├── ONNX Edge scope and profile definition.md │ ├── ONNX Profile Topology Mapping Proposal │ ├── ONNX Stationary IoT device with AV profile (prototype).md │ ├── README.md │ ├── Scope of Edge.png │ ├── onnx-edge-scenarios.md │ └── onnx-profile-topology-map.md ├── meetings │ ├── 20190404.md │ ├── 20190417.md │ ├── 20190501.md │ ├── 20190508.md │ ├── 20190515.md │ ├── 20190605.md │ ├── 20190612.md │ ├── 20190619.md │ ├── 20190626.md │ ├── 20190710.md │ ├── 20190724.md │ ├── 20190807.md │ ├── 20190821.md │ ├── 20190904.md │ ├── 20190925.md │ ├── 20191016.md │ ├── 20191030.md │ ├── 20191113.md │ ├── 20191115.md │ ├── 20191204.md │ ├── 20191218.md │ └── 20200108.md └── workshops │ ├── ONNX_WorkshopAug2019-Edge_WG_Session.pdf │ ├── ONNX_WorkshopMar2019-Edge&MLPerf.pdf │ ├── ONNX_WorkshopMar2019-Edge_WG_Session.pdf │ └── ONNX_WorkshopNov2019-Edge_WG_Session.pdf ├── foundation ├── LF AI - ONNX Community Discussion.pdf ├── LFAI-proposal.adoc ├── README.md ├── foundation-options.md └── whitepaper.md ├── generative-ai ├── ReadMe.md └── meetings │ ├── ReadMe.md │ ├── meeting_1_date_April_23.md │ ├── meeting_2_date_April_30.md │ ├── meeting_3_date_May_7.md │ └── meeting_kickoff_date_April_17.md ├── multi-device ├── README.md └── meetings │ ├── README.md │ ├── notes-09-13-2023.md │ ├── notes-10-11-2023.md │ ├── notes-11-08-2023.md │ └── notes-12-13-2023.md ├── preprocessing ├── README.md ├── meetings │ ├── 20210811.md │ ├── 20211013.md │ ├── 20211110.md │ ├── 20211208.md │ ├── 20220112.md │ ├── 20220309.md │ ├── 20220413.md │ ├── 20220511.md │ ├── 20220608.md │ ├── 20220713.md │ ├── 20221019.md │ ├── 20221109.md │ ├── 20230111.md │ ├── 20230308.md │ ├── 20230419.md │ ├── 20230510.md │ ├── 20230614.md │ ├── 20230712.md │ ├── 20231011.md │ ├── 20231108.md │ ├── 20240110.md │ └── slides │ │ ├── 20210811_slides.pdf │ │ ├── 20211013_slides.pdf │ │ ├── 20211110_slides.pdf │ │ ├── 20211208_slides.pdf │ │ ├── 20220112_slides.pdf │ │ ├── 20220309_slides.pdf │ │ ├── 20220413_slides.pdf │ │ ├── 20220511_slides.pdf │ │ ├── 20220608_slides.pdf │ │ ├── 20220713_slides.pdf │ │ ├── 20221019_slides.pdf │ │ ├── 20221109_slides.pdf │ │ ├── 20230111_slides.pdf │ │ ├── 20230308_slides.pdf │ │ ├── 20230419_slides.pdf │ │ ├── 20230510_slides.pdf │ │ ├── 20230614_slides.pdf │ │ ├── 20230712_slides.pdf │ │ └── 20231011_slides.pdf └── notebooks │ ├── batch-preprocessing │ ├── Batched_preprocessing.ipynb │ ├── loop-test.onnx.1.png │ ├── loop-test.onnx.2.png │ └── preprocessing.onnx.png │ ├── image_proc_functions │ └── ImageProcFunctions.ipynb │ ├── images │ ├── cute-4074304_224.jpg │ ├── grasshopper-4357903_1280.jpg │ ├── image_info.txt │ └── snail-4345504_1280.jpg │ ├── preprocessing-rn50 │ ├── onnx-preprocessing.png │ ├── resnet50_modelzoo_onnxruntime_inference_onnx_preprocessing_poc.ipynb │ └── rn50-preprocessing-resize.png │ ├── resize-antialias │ └── resize-antialias.ipynb │ ├── resize_flip_invariant │ └── onnx_flip_invariant_resize.ipynb │ ├── resnet_preproc_zoo │ ├── ResNet with preprocessing via ONNX hub.ipynb │ └── Resnet preprocessing model and data preparation.ipynb │ └── sequence_map │ └── SequenceMap.ipynb ├── release ├── docs │ └── onnx_release_verification.md └── meetings │ ├── 001-20200603.md │ ├── 002-20200624.md │ ├── 003-20200715.md │ ├── 004-20200729.md │ ├── 005-20200819.md │ ├── 006-20200902.md │ ├── 007-20200916.md │ ├── 008-20200923.md │ ├── 009-20200930.md │ ├── 010-20201007.md │ ├── 011-20201104.md │ └── 012-20201111.md ├── safety-related-profile ├── README.md ├── deliverables │ ├── README.md │ ├── issues │ │ └── issues.md │ ├── needs │ │ └── needs.md │ ├── reqs │ │ └── reqs.md │ └── scope │ │ └── scope.md ├── documents │ ├── Attic │ │ ├── ONNX_Op_classification.xlsx │ │ ├── ONNX_format_comments_v0.1.docx │ │ └── spec_conventions │ ├── README.md │ ├── analysis_of_standards │ │ ├── 2024-11-08 - Replication criteria.md │ │ ├── SONNX_requirements_draft1.docx │ │ └── questions_to_wg114.md │ ├── conv_specification_example │ │ ├── README.md │ │ ├── imgs │ │ │ ├── README.md │ │ │ ├── autopad.png │ │ │ ├── conv-dep-3-channels.png │ │ │ ├── conv-dep.png │ │ │ ├── conv-std-3 channels.png │ │ │ ├── conv-std-3-channels.png │ │ │ ├── conv-std.png │ │ │ ├── conv.png │ │ │ ├── conv_pad.png │ │ │ ├── conv_stride.png │ │ │ ├── dilation.png │ │ │ ├── grouped_convolution.png │ │ │ ├── onnx_conv.drawio │ │ │ └── why3_workflow.png │ │ ├── reviews │ │ │ ├── README.md │ │ │ ├── imgs │ │ │ │ ├── README.md │ │ │ │ ├── jl-img1.png │ │ │ │ └── jl-img2.png │ │ │ ├── jean-loup.md │ │ │ └── sebastian.md │ │ └── tests │ │ │ └── conv_onnx.ipynb │ ├── issues.md │ ├── mngt │ │ ├── README.md │ │ ├── WG_objectives_and_organisation.docx │ │ └── Workplan.xlsx │ ├── needs.md │ ├── procedures │ │ ├── ModifGithubONNX.pdf │ │ └── collaborative_docs.md │ ├── profile_graph │ │ └── graph.md │ ├── profile_opset │ │ ├── conv │ │ │ ├── conv.md │ │ │ └── imgs │ │ │ │ ├── README.md │ │ │ │ ├── autopad.png │ │ │ │ ├── conv-dep-3-channels.png │ │ │ │ ├── conv-dep.png │ │ │ │ ├── conv-std-3 channels.png │ │ │ │ ├── conv-std-3-channels.png │ │ │ │ ├── conv-std.png │ │ │ │ ├── conv.png │ │ │ │ ├── conv_pad.png │ │ │ │ ├── conv_stride.png │ │ │ │ ├── dilation.png │ │ │ │ ├── grouped_convolution.png │ │ │ │ ├── onnx_conv.drawio │ │ │ │ └── why3_workflow.png │ │ └── where │ │ │ ├── imgs │ │ │ ├── ONNX_WHERE.png │ │ │ └── README.md │ │ │ └── where.md │ ├── reqs.md │ ├── sow.md │ └── usecases.md ├── meetings │ ├── Other_meetings │ │ ├── 2024-10-25-DeepGreen.md │ │ └── 2024-11 - ISCLP meeting - SONNX.pptx │ ├── README.md │ ├── analysis_sub_wg │ │ ├── README.md │ │ ├── minutes.md │ │ └── slides-18-12.md │ ├── errors │ │ ├── 01_what_is_the_issue.md │ │ ├── 2024-11-06 - Numerical issues.pdf │ │ ├── README.md │ │ ├── error_specification.md │ │ ├── images │ │ │ └── ada_accuracy.png │ │ ├── slides-06-11.md │ │ └── slides-06-11.pdf │ ├── formal_methods │ │ ├── minutes.md │ │ ├── slides-29-11.md │ │ └── slides-29-11.pdf │ ├── general │ │ ├── 2024-06-28 - MEET-UP │ │ │ ├── 280624_ONNX_MeetUp.pdf │ │ │ └── README.md │ │ ├── 2024-09-25 - KOM │ │ │ ├── 2024-09-25 - SONNX KOM.pdf │ │ │ └── README.md │ │ ├── README.md │ │ └── slides │ │ │ ├── 2024-12-04-modifiedconv2Dinformalspec.pdf │ │ │ └── README.md │ ├── minutes.md │ ├── operator_spec_sub_wg │ │ ├── README.md │ │ ├── SONNX_Operator_List.xlsx │ │ └── worksharing.md │ ├── presentation_proposals.md │ ├── reqs_sub_wg │ │ ├── README.md │ │ ├── minutes.md │ │ └── slides-18-12.md │ ├── slides │ │ ├── 2024-12-18-Embedded_AI_Coder_Bosch_Boblest.pdf │ │ ├── 2025-01-15-Edoardo_Manino_SONNX_slides.pdf │ │ └── README.md │ └── team.md └── tools │ ├── LeNet5.onnx │ ├── README.md │ ├── onnx_depend.py │ ├── requirements.txt │ └── tasks.py └── training ├── README.md └── workshops └── August23_2019 ├── TrainingReadout.pdf ├── TrainingStory.pdf ├── TrainingUpdate.pdf ├── TrainingWorkshopStart.pdf └── onnx_2_tf_test.pdf /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Working Groups 4 | 5 | As described in the ONNX [governance](https://github.com/onnx/onnx/tree/main/community#wg---working-groups), Working Groups (WGs) are temporary groups formed to address issues that cross SIG boundaries. Working Groups have a have a clear goal measured through specific deliverables and disband after the goal is achieved. Working groups do not own artifacts long term; they create specifications, recommendations, and/or code implementations for submission to the relevant SIGs for approval and acceptance. 6 | 7 | ## Proposing a new working group 8 | New Working Groups are created when there is sufficient interest in a topic area and someone volunteers to be the chair for the group and submits a proposal to the steering committee. The chair facilitates the discussion and helps synthesize proposals and decisions. 9 | 10 | ## Joining a working group 11 | Working Groups have their discussions on Slack in dedicated channels. (Older working groups used Gitter rooms which are now archived) 12 | 13 | You can find the schedule of meetings on the [LF AI wiki](https://wiki.lfai.foundation/pages/viewpage.action?pageId=18481196) 14 | 15 | Working Groups store these artifacts, including meeting notes, in this repository. The working group leads have merge permissions on the repo but should only handle PRs related to their working group. 16 | 17 | ## Active working groups 18 | 19 | Contact e-mail addresses of WG leads can be found [here](https://wiki.lfaidata.foundation/pages/viewpage.action?pageId=18481196). 20 | 21 | | Working Group | Objectives | 22 | | ------------------ | ------------- | 23 | | [Preprocessing](https://lfaifoundation.slack.com/archives/C02AANGFBJB) | Data pre/post processing and featurization | 24 | | [Multi-device](https://lfaifoundation.slack.com/archives/C05JY32GCCS) | Multi-device support in ONNX | 25 | | Safety-Related-Profile | | 26 | 27 | ## Completed working groups 28 | 29 | | Working Group | Objectives | Status | 30 | | ------------------ | ------------- | ------ | 31 | | [Release](https://lfaifoundation.slack.com/archives/C018VGGJUGK) | Improve and document release process, enhance release verification, identify fixes for release builds, recommend release operational guidelines in communication, frequency, scope, dependencies, and schedule | Completed - release process enhanced and documented at https://github.com/onnx/onnx/blob/main/docs/OnnxReleases.md 32 | | [Control Flow and Loops](https://gitter.im/onnx/ControlFlowWG) | Enable dynamic control structures to enable advanced models for NLP, speech, and video/image processing | Completed - If, Loop, and Scan operators were added in ONNX 1.3 release. | 33 | | Quantization | Enhance ONNX to support quantized data types and operators on a variety of runtimes and hardware devices | Completed - added in ONNX 1.5 release. | 34 | | Foundation | Identify and evaluate non-profit foundation options for the ONNX consortium. Execute on best option. | Completed - ONNX joined LF AI in November 2019| 35 | 36 | ## Inactive working groups 37 | 38 | | Working Group | Objectives | Status | 39 | | ------------------ | ------------- | ------ | 40 | | Testing/Compliance | Create tools, tests and APIs to ensure models and backends comply with the ONNX specification | became part of the ArchInfra SIG for now | 41 | | Edge/Mobile | Enable deployment of ONNX models to edge platforms by identifying the ONNX operators that must be supported by mobile and IoT runtimes, defining tool chain requirements, and contributing to ONNX compatibility tests | Stopped as of January 2020 | 42 | | Data Pipelines | Define new operators for processing the data that goes in and comes out | No recent activity | 43 | | [Training](https://lfaifoundation.slack.com/archives/C018K560U14) | Expand ONNX to support training as well as inference | 44 | -------------------------------------------------------------------------------- /edge/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Edge Working Group 4 | 5 | This is artifacts repository where ONNX Edge working group will capture various artifacts and deliverables. 6 | Structure of the space will evolve over time. 7 | 8 | ## Working Group Status 9 | **ACTIVE** 10 | 11 | ## Contributors 12 | *Note: Contributors list will be updated as per participation and contributions.* 13 | * Milan Oljaca (Qualcomm) (co-chair) 14 | * Ofer Rosenberg (Qualcomm) (co-chair) 15 | * Yedong Liu (Huawei) 16 | * Saurabh Tangri (Intel) 17 | * Manash Goswami (Microsoft) 18 | 19 | ## Meeting logistics 20 | * WG chair(s) will drive and facilitate the meetings 21 | * Publish agenda, produce meeting notes, action items, etc. 22 | * Meetings are bi-weekly 23 | * ***Wednesdays 8:00am PST***, starting Apr 17, 2019. 24 | * Will post a message to [ONNX Edge Gitter Channel](https://gitter.im/onnx/edge) with agenda and meeting invite link 25 | * Up to 7 days before the meeting by WG chair, but proposals to agenda item updates are welcomed from all contributors via gitter 26 | * Meeting/Telepresence tool: Zoom 27 | * Meetings will be recorded and published to [ONNX YouTube channel](https://www.youtube.com/channel/UCIg4Cceqra3rtJEC5LPTdtw) 28 | 29 | ## Discussion mechanics 30 | * [ONNX Edge WG Gitter Channel](https://gitter.im/onnx/edge) 31 | * [ONNX Edge WG repo](https://github.com/onnx/working-groups/edge) 32 | * Document reviews 33 | * Will use Google doc for draft reviews until we get to final doc proposal, at which point it will get transcribed into .md and committed to [ONNX Edge WG repo](https://github.com/onnx/working-groups/edge). 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /edge/artifacts/Edge scenario radar chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/artifacts/Edge scenario radar chart.png -------------------------------------------------------------------------------- /edge/artifacts/Scope of Edge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/artifacts/Scope of Edge.png -------------------------------------------------------------------------------- /edge/discussion_materials/Edge scenario radar chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/discussion_materials/Edge scenario radar chart.png -------------------------------------------------------------------------------- /edge/discussion_materials/ONNX Edge Execution Profile Compliance.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Edge Execution Mode Proposal 4 | Author: Saurabh Tangri, Milan Oljaca, Yedong Liu, Ofer Rosenberg 5 | 6 | Version: 0.1 7 | 8 | ## Background 9 | Edge profiles are intended to provide guidelines and compliance procedures for solution vendors. Please refer [here](https://github.com/onnx/working-groups/blob/master/edge/discussion_materials/ONNX%20Edge%20scope%20and%20profile%20definition.md) for details on edge profiles. The edge profiles can be static or dynamic depending on the nature of edge device. A fixed function device such as a smart speaker supports a single static edge profile. For other multi-function edge devices(example PC) a profile can be viewed as a mode of operation. Such a device can support multiple ONNX edge profiles. At times when such a device is operating multiple profiles simultaneously ONNX Edge execution mode attributes provide hints to the platform. This establishes an execution contract between the application and the underlying platform which guarantees Quality of Service. 10 | 11 | ## Definition 12 | For multi-function devices such as a PC, ONNX Edge profile execution mode is an optional mechanism to express configuration needed to run a particular Machine Learning model. The collection of attributes contained in edge profile help ML frameworks and underlying runtimes allocate and manage resources prior to running the model. It is described in the model file and used by the runtime. 13 | 14 | ## Purpose/Objective 15 | To ensure unambiguous usage of ONNX edge execution modes and profile attributes during training and inference we need to describe runtime behavior. 16 | 17 | ## Scope 18 | ONNX execution modes describe the minimum deployment configuration needed to successfully run a particular model. It should describe the assumptions that were made when authoring the model and are needed to comply to the requirements of edge profiles. These can be viewed as a contract between author of a ML model and the executor of the model. 19 | 20 | | | Details | How to populate | Data Type | Example | 21 | |--------------|---------------------------------------------------------------------|--------------------------------------------------|----------------------------------------------------|---------| 22 | | Accuracy | What accuracy to expect when deploying the model with this profile. | Accuracy noted. (Post training) | DatasetName: Metric: | | 23 | | Memory | What memory footprint to expect when running in this profile. | Average steady state working set memory | Size | | 24 | | Latency | What minimum performance is expected by this profile | latency noted during model validation. | Batch Size First time latency Steady state latency | | 25 | | Power | What power performance is expected by this profile | What is minimum steady state throughput per watt | IPS/W | | 26 | | Data Locality | Is model intended to run on a network connected device? | What is Network QoS required to run the model | Yes/No or QoS | | 27 | 28 | Following are the implications of adding Edge Execution Modes 29 | 1. A producer of a ONNX model(Training frameworks or conversion tools) should be able to store profiles and attributes. 30 | 2. Visualization tools like(ex: Netron) should be able to show profile attributes. 31 | 32 | 33 | ## Compliance Policy 34 | In order to be compliant the executing entity:example a runtime or framework needs to run the model in a configuration that meets the requirements described by the attributes. 35 | 36 | ## Compliance Procedure 37 | 38 | ## Related Procedures 39 | 40 | ## References 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /edge/discussion_materials/ONNX Profile Topology Mapping Proposal: -------------------------------------------------------------------------------- 1 | # ONNX Profile Topology Mapping Proposal 2 | Author: Ofer Rosenberg, Milan Oljaca 3 | 4 | Version: 0.1 5 | 6 | ## Introduction 7 | 8 | A part of the Edge working group charter is to define a set of profiles for edge devices. The goal is for an edge device to be tested for compliance with a set of profiles. 9 | During the working group discussions turned out that there can be a few approaches to breaking down of the edge devices functionality space. 10 | Intent with such profile breakdown is to establish categorization of edge profiles based on use cases and profile attributes identified in [onnx-edge-scope-and-profile-definition.md](https://github.com/onnx/working-groups/blob/master/edge/artifacts/onnx-edge-scope-and-profile-definition.md) document. 11 | The goal of this document is to describe the few possible ways to do the topology mapping, and drive towards deciding on one of these. 12 | There are two decisions that need to be made : 13 | 1. Define 3-level topology or 2-level topology 14 | 2. Define the mapping type (sensor based, network based, etc.) 15 | 16 | ## Topology Level 17 | 18 | In defining the profiles, we can take two approaches : 19 | 1. A three-level topology mapping, where the top level categories are identified as per [Scope of Edge.png](https://github.com/onnx/working-groups/blob/master/edge/artifacts/Scope%20of%20Edge.png). 20 | 1. First level : Edge Infrastructure, Edge Device 21 | 1. Second level : Based on mapping type (see next section) 22 | 1. Third level : Complexity (Basic / Intermediate / Advanced) 23 | 2. A two-level topology mapping. Instead of top level categories, roll it into more complexity levels 24 | 1. First level : Based on mapping type (see next section) 25 | 1. Second level : Complexity (1-N) 26 | 27 | 28 | ## Mapping Type 29 | 30 | In defining the mapping type, we suggest the following two approaches : 31 | 1. Based on Sensor type : 2D Image, Audio, Voice, Text, ... 32 | 2. Based on Network type : Classification, SSD, Image Reconstruction, Speech Recognition, ... 33 | 34 | ## Selection Criteria: 35 | 36 | The selection criteria between the various mapping options will focus on 37 | 1. Defining a set of profiles which covers the range of target devices 38 | 2. Creating a clear profile definition that will serve for compliance purpose 39 | 40 | -------------------------------------------------------------------------------- /edge/discussion_materials/ONNX Stationary IoT device with AV profile (prototype).md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Stationary IoT device with AV profile (prototype) 4 | Author: Ofer Rosenberg 5 | 6 | Version: 0.1 7 | 8 | ## Introduction 9 | 10 | *This document is a prototyping effort to describe a profile using the method and definitions provided in "ONNX Edge scope and profile definition"* 11 | 12 | The profile described in this document is of a stationary IoT device with Audio/Video interface (input and output). 13 | The family of edge devices covered by this profile are devices which remain in the same location (fixed to a wall, lying on a table, etc), constantly connected to the web (wired ro wireless) and have audio and video interfaces - inputs, outputs or both. 14 | The next section provides a few examples for devices which fit into this profile. 15 | 16 | 17 | ## Profile Examples 18 | Here are a few examples for devices which fit this profile : 19 | 20 | 1. IP Security Camera : This is an IoT device which is stationary, usually fixed to a wall or placed in some fixed location. It has a video input, which may range from low-resolution to 4K images, and possibly an audio input. It also has web connectivity, used to report on events and sometimes send video sections or still images. It is constantly connected to a power source. In terms of processing, it usually processes the AV information on the device, to ensure low latency of event detection. 21 | 2. Smart Speaker : This is an IoT device which usually located on some table of shelf, has audio inputs and outputs, and has web connectivity (wired or wireless). It may be connected to a power source or runs on batteries. In terms of processing, it usually uses a hybrid model, where some preliminary processing runs on the device, and the rest runs in the cloud. 22 | 3. Smart Display : Similar to a smart speaker, but has a video display and a camera. In terms of connectivity, power supply and processing model, similar to the smart speaker. 23 | 24 | 25 | ## Profile attributes 26 | This sections describes the attribute values for Stationary IoT device with AV profile, referred in this section in short as "the device" 27 | 28 | ### 1. Accuracy 29 | 30 | The ONNX model zoo collects many wide-used neural network models, for use cases such as image classification, object detection, face recognition, image segmentation, etc. While the accuracy number varies, the top-5 accuracy seems acceptable in many cases; e.g. VGG, Resnet and MobileNet models all have top-5 accuracy above or close to 90%. Acceptable accuracy is very much use case or scenario dependent. Accuracy is critical for scenarios like smart city, face recognition and smart camera. Challenge in deploying these models on an edge device is to keep the accuracy still at acceptable level while employing various techniques to reduce model's computational complexity and size (e.g. via model compression or low-bit integer math computation). 31 | 32 | ### 2. Size 33 | 34 | The device should have at least 2GB of DRAM memory, and free 8GB of Storage room (as flash drive or else) 35 | 36 | ### 3. Latency 37 | 38 | In case of a Video input, the device should be able to process any of the defined classification networks in less than 33mSec (fitting 30FPS rate). 39 | In case of Audio input, the device should process the RNN based networks in less than 20mSec 40 | 41 | ### 4. Power consumption 42 | 43 | The device would be limited to a power consumption of 5W. The expectation is to have 60% of the use cases consume 3W or less. 44 | 45 | 46 | ### 5. Data locality 47 | The following set of video classification models are required to be supported fully on device, due to latency & privacy requirements : 48 | 49 | - List TBD 50 | The following set of audio models are expected to run in hybrid mode, where only part of the model run on the device : 51 | 52 | - List TBD 53 | 54 | ## Required Networks 55 | This section covers a list of networks required to be supported by the device 56 | 57 | ### Inception v3 58 | TBD 59 | ### MobileNet v2 60 | TBD 61 | 62 | ## ONNX Operations 63 | TBD 64 | -------------------------------------------------------------------------------- /edge/discussion_materials/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Folder to collect various WG discussion materials. 4 | -------------------------------------------------------------------------------- /edge/discussion_materials/Scope of Edge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/discussion_materials/Scope of Edge.png -------------------------------------------------------------------------------- /edge/discussion_materials/onnx-profile-topology-map.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Profile Topology Map 4 | Author: Milan Oljaca, Ofer Rosenberg, Yedong Liu, Saurabh Tangri 5 | 6 | Version: 0.1 7 | 8 | ## Introduction 9 | 10 | This document defines a hierarchical classification of ONNX edge profiles, i.e. a profile topology map. 11 | Intent with such profile breakdown is to establish categorization of edge profiles based on use cases and profile attributes identified in [onnx-edge-scope-and-profile-definition.md](https://github.com/onnx/working-groups/blob/master/edge/artifacts/onnx-edge-scope-and-profile-definition.md) document. 12 | 13 | ## Profile topology 14 | 15 | Top level categories are identified as per [Scope of Edge.png](https://github.com/onnx/working-groups/blob/master/edge/artifacts/Scope%20of%20Edge.png). 16 | 17 | ### Edge Infrastructure 18 | Considering edge infrastructure deployment aspects, e.g. stationary and wall-powered, the profile attributes of most relevance are accuracy and latency. 19 | 20 | #### Basic 2D Image profile 21 | Use cases: Computer Vison - Image Classification, Object Detection, Semantic Segmentation 22 | Models: Mobilenet, MobilenetSSD, Enet? 23 | Capability: Basic (e.g. meets top-5 accuracy, <= 15fps latency, input resolution <= 480p, ... ) 24 | 25 | #### Advanced 2D Image Profile 26 | Use cases: Computer Vison - Image Classification, Object Detection, Semantic Segmentation 27 | Models: .... 28 | Capability: Advanced (e.g. meets top-1 accuracy, <= 30fps latency, input resolution >= 1080p, ... ) 29 | 30 | 31 | #### Basic Audio Profile 32 | Use cases: ASR 33 | Models: .... 34 | Capability: Basic (e.g. .... ) 35 | 36 | #### Advanced Audio Profile 37 | Use cases: NLP, Translation 38 | Models: .... 39 | Capability: Advanced (e.g. .... ) 40 | 41 | 42 | ### Edge Devices 43 | Considering edge devices' deployment aspects, two sub-categoriese are considered: 44 | * Mobile (battery powered): profile attributes of most relevance are power consumption and size. 45 | * Stationary (wall-powered): profile attributes of most relevance are accuracy, latency and size. 46 | 47 | #### Basic 2D Image profile 48 | Use cases: Computer Vison - Image Classification, Object Detection, Semantic Segmentation 49 | Models: .... 50 | Capability: Basic (e.g. meets top-5 accuracy, <= ??fps latency, input resolution <= ??, ... ) 51 | Subtype: Data Locality: Yes or No 52 | 53 | #### Advanced 2D Image Profile 54 | Use cases: Computer Vison - Image Classification, Object Detection, Semantic Segmentation 55 | Models: .... 56 | Capability: Advanced (e.g. meets top-1 accuracy, <= 30fps latency, input resolution >= 1080p, ... ) 57 | Subtype: Data Locality: Yes or No 58 | 59 | #### Basic Audio Profile 60 | Use cases: ASR 61 | Models: .... 62 | Capability: Basic (e.g. ) 63 | Subtype: Data Locality: Yes or No 64 | 65 | #### Advanced Audio Profile 66 | Use cases: NLP, Translation 67 | Models: .... 68 | Capability: Advanced (e.g. ) 69 | Subtype: Data Locality: Yes or No 70 | 71 | 72 | ## NOTES 73 | * Accuracy could be mAP based as per selected reference models in the profile. 74 | * E.g. Basic profiles will have simpler models but lower accuracy. 75 | 76 | * Possible more formal profile naming convention to consider, more future proof. E.g. 77 | * 2D Image: Class A1 Profile ( = Advanced, data locality=yes) 78 | * 2D Image: Class A2 Profile ( = Advanced, data locality=no) 79 | * 2D Image: Class B1 Profile ( = Basic, data locality=yes) 80 | * 2D Image: Class B2 Profile ( = Basic, data locality=no) 81 | * 2D Image: Class B3 Profile ( = Basic, some future bounds go here: e.g. same as B1 but not more then 1% mAP drop ) 82 | 83 | 84 | -------------------------------------------------------------------------------- /edge/meetings/20190417.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Apr 17, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Statement of Work 7 | * Review / finalize proposal: [ONNX Edge Working Group SoW](https://docs.google.com/document/d/18M-i7yMfuMLnpsrUnyCCc5mosPIt7Q_lDNUQAQqZrBo/edit?usp=sharing) 8 | * Action items and goals for next meeting 9 | * Discussion 10 | 11 | ## Meeting Minutes 12 | 13 | ### Attendees 14 | * Milan Oljaca (Qualcomm) 15 | * Ofer Rosenberg (Qaulcomm) 16 | * Yedong Liu (Huawei) 17 | * Saurabh Tangri (Intel) 18 | * Bogdan (Software AG) 19 | 20 | ### Notes 21 | * **Milan**: Asked everybody to acknoledge recording of the meeting 22 | * **Milan**: made updates to [Edge README.md](https://github.com/onnx/working-groups/blob/master/edge/README.md). Please comment or create issue to update further. 23 | * [ONNX Edge Working Group SoW](https://docs.google.com/document/d/18M-i7yMfuMLnpsrUnyCCc5mosPIt7Q_lDNUQAQqZrBo/edit?usp=sharing) Review 24 | * **Goal is to complete and have final document ready in the next meeting.** 25 | * **Saurabh**: asked about realation to docs from Howard / Yedong. 26 | * **Milan**: PR from Howard is a summary of discusion that took place in Nov 2018 ONNX workshop. 27 | * **Yedong**: Created new PR [ONNX Edge Scenario Discussion](https://github.com/onnx/working-groups/pull/3), update to Howard's PR. 28 | * **Ofer**: Documents submitted by Yedong and Howard (Huawei) are great starting points for definiton of edge profiles. Once we define SoW. 29 | * **Ofer**: suggested creating "discussion materials" folder where docs like the one from Howard / Yedong could be placed into. There was no objection to this suggestion. 30 | * **Saurabh**: About MLPerf, are they comitted to ONNX as input format? Consider other benchmarking partners if MLPerf does not work out. 31 | * E.g. [TPC](http://www.tpc.org/), [AIBench](http://www.aibench.org/) 32 | * **Milan**: Bechmarking is related to compliance. We need to initiate contact with other ONNX groups as a heads up for cooperation. 33 | * **Ofer**: We could encourage benchmarking orgs to participate in ONNX WG if they want to influence or contribute with purpose of alignment with their work. 34 | * **Milan**: Working with Prasanth on logistics to publish meeting recordings to [ONNX youtube channel](https://www.youtube.com/channel/UCIg4Cceqra3rtJEC5LPTdtw) 35 | 36 | ### Action Items 37 | * **Milan**: Publish meeting minutes and recording. 38 | * Next meeting ***Wed May 1st 2019 at 8:00am PST***. 39 | * ONNX Edge Working Group SoW document review 40 | * **Everybody**: closing comments on the document. 41 | * Goal for next meeting is to finalize document, do acceptance vote and submit for approval to steering committee. 42 | * **Milan**: Handle PRs and create discussion_materials folder. 43 | 44 | 45 | -------------------------------------------------------------------------------- /edge/meetings/20190501.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed May 1, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Statement of Work 7 | * Review / finalize proposal: [ONNX Edge Working Group SoW](https://docs.google.com/document/d/18M-i7yMfuMLnpsrUnyCCc5mosPIt7Q_lDNUQAQqZrBo/edit?usp=sharing) 8 | * **Goal for the meeting is to finalize document, do acceptance vote and submit for approval to steering committee.** 9 | * Action items and goals for next meeting 10 | * Discussion 11 | 12 | ## Meeting Minutes 13 | * [Meeting recording](https://youtu.be/OZmURWO50Po) 14 | 15 | ### Attendees 16 | * Milan Oljaca (Qualcomm) 17 | * Yedong Liu (Huawei) 18 | 19 | ### Notes 20 | * We had a short meeting, due to low attendance. Could not accomplish goal set for the meeting. 21 | * **Milan**: Proposed setting up a short follow up meeting on Wed May 8, to complete set goal 22 | * **Yedong**: Suggested also to advertise on onnx/lobby gitter channel, in addition to onnx/edge channel for participation 23 | 24 | ### Action Items 25 | * **Milan**: Publish meeting minutes and recording. 26 | * Set up a follow up meeting for ***Wed May 8st 2019 at 8:30am PST***. 27 | * Ask on gitter channels for participation for SoW approval 28 | 29 | 30 | -------------------------------------------------------------------------------- /edge/meetings/20190508.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed May 8, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Follow up to May 1st meeting 7 | * Finalize [ONNX Edge Working Group SoW](https://docs.google.com/document/d/18M-i7yMfuMLnpsrUnyCCc5mosPIt7Q_lDNUQAQqZrBo/edit?usp=sharing) 8 | * Goal: Acceptance vote, make SoW official and ready for submission to ONNX steering committee 9 | * NOTE: if not able to attend, all contributors please vote by adding a comment to the document 10 | * Action items and goals for next meeting 11 | * Discussion 12 | 13 | ## Meeting Minutes 14 | * [Meeting recording](https://youtu.be/914T4qX7NcM) 15 | 16 | ### Attendees 17 | * Milan Oljaca (Qualcomm) 18 | * Yedong Liu (Huawei) 19 | * Ofer Rosenberg (Qaulcomm) 20 | 21 | ### Notes 22 | * **Milan**: Acceptance vote was carried, all attendees acknowledged SoW as ready and approved it for formal submission to ONNX steering committee 23 | * Howard Huang (Huawei) acknowledged SoW doc with gitter message 24 | * All agreed that under Deliverables section, Document1 and Document2 could be combined into one document 25 | * Document will be transcibed into .md and pushed into working-groups/edge github 26 | * **Milan**: Commented about Document1&2 (next WG deliverable as described in SoW), and overlap with [onnx-edge-scenarios.md](https://github.com/onnx/working-groups/blob/master/edge/discussion_materials/onnx-edge-scenarios.md). 27 | Asked Yedong if he could take ownership and create a draft for the review. Yedong agreed, and committed to have draft ready for review in May 29, 2019 meeting. 28 | * **Ofer**: Suggested to use May 15 meeting to discuss/brainstorm content for the document 29 | 30 | ### Action Items 31 | * **Milan**: Publish meeting minutes and recording. 32 | * Next meeting: ***Wed May 15 2019 at 8:00am PST*** 33 | * **Milan**: Publish SoW 34 | * Transcibe into .md and push into working-groups/edge github. 35 | * Submit to ONNX steering committee 36 | * **Yedong**: Prepare "Edge scope and profile definition" (working title, can change) document draft for review. 37 | 38 | 39 | -------------------------------------------------------------------------------- /edge/meetings/20190515.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed May 15, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Discussion 7 | * "Edge scope and profile definition" (working title, can change) document content 8 | * Action items and goals for next meeting 9 | 10 | ## Meeting Minutes 11 | * [Meeting recording](https://youtu.be/RvxEVsccOp8) 12 | 13 | ### Attendees 14 | * Milan Oljaca (Qualcomm) 15 | * Yedong Liu (Huawei) 16 | * Ofer Rosenberg (Qaulcomm) 17 | * Saurabh Tangri (Intel) 18 | 19 | ### Notes 20 | * Much of the discusison was in context of "What is Edge?" slide from [Mar2019 workshop - Edge WG session slides](https://github.com/onnx/working-groups/blob/master/edge/workshops/ONNX_WorkshopMar2019-Edge_WG_Session.pdf) (slide 20) 21 | * **Yedong**: Commented that he is working on "Edge scope and profile definition" document and is using this slide as reference 22 | * Significant discussion was about Secirity/Privacy as one of profile atributes that was listed, and what it means. 23 | **Milan**: Suggested using "Data Locality" instead 24 | 25 | ### Action Items 26 | * **Milan**: Publish meeting minutes and recording. 27 | * Next meeting: ***Wed May 29 2019 at 8:00am PST*** 28 | * **Yedong**: Prepare "Edge scope and profile definition" (working title, can change) document draft for review in May 29 meeting. 29 | Share draft to the group before the meeting if possible. 30 | 31 | -------------------------------------------------------------------------------- /edge/meetings/20190605.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jun 05, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * "Edge scope and profile definition" document review. 7 | * Yedong posted doc in [PR](https://github.com/onnx/working-groups/pull/6/) 8 | 9 | ## Meeting Minutes 10 | * [Meeting recording](https://youtu.be/ivL8pM6a9cY) 11 | 12 | ### Attendees 13 | * Milan Oljaca (Qualcomm) 14 | * Yedong Liu (Huawei) 15 | * Saurabh Tangri (Intel) 16 | 17 | ### Notes 18 | * Yedong walked us through the [document he posted for the review](https://github.com/onnx/working-groups/pull/6/) 19 | * **Saurabh**: In context of Edge scenario section, asked about PC scenario, as it was not obvious to him if it belongs to Edge. PC contains several subsystems (like camera and audio) that by itself can be considered edge profiles. 20 | * **Milan**: commented that considering that Edge scope is pretty much defined as !Cloud, PC is an Edge device. Brought up possibility to have a composite Edge profile. In this context the PC scenario is similar to Car or Phone scenario. 21 | * **Yedong**: will update section to clarify that PC as included in scenario examples. 22 | * **Milan**: In context of Edge Device section, asked for clarification about connectivity, and that we should not limit connectivity to internet only. 23 | This is to cover for scenarios where edge device is connected via other means like usb, or pcie. 24 | * **Saurabh**: Make sure we differentiate connectivity in context of state vs. capability. E.g. device can have connectivity capability, but not be in connected state. 25 | * In context of Edge profile metrics/attributes section, discussed if Accuracy should be an attribute. 26 | Is it data/math type what really matters, e.g. integer vs. float compute? Also if "Compute capabilities" or "Compute resource constraints" should be attribute, if we need to separate edge profiles based on it. Is Latency/Speed and Size sufficient? 27 | Data/math type compute capability could be covered though a subset of operations identified for an edge profile. 28 | There was no conclusion, this was an open discussion. Will continue in next meeting. 29 | 30 | ### Action Items 31 | * **Milan**: Publish meeting minutes and recording. 32 | * Next meeting: ***Wed Jun 12 2019 at 8:00am PST*** 33 | * **Yedong**: Make updates to the document based on discussion today. 34 | * **All**: Please post comments to document in [PR](https://github.com/onnx/working-groups/pull/6/). 35 | We will continue with review in next meeting. 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /edge/meetings/20190612.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jun 12, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Continue reviewing "Edge scope and profile definition" document. 7 | * Yedong updated the document based on comments 8 | * Updated document is [Here](https://github.com/onnx/working-groups/blob/54c5c7b7a7b56b04bcfe5f48c10b250e6d625d42/edge/discussion_materials/ONNX%20Edge%20scope%20and%20profile%20definition.md) 9 | 10 | ## Meeting Minutes 11 | * [Meeting recording](https://youtu.be/7CJeQdPq2mQ) 12 | 13 | ### Attendees 14 | * Ofer Rosenberg (Qualcomm) 15 | * Yedong Liu (Huawei) 16 | * Saurabh Tangri (Intel) 17 | 18 | ### Notes 19 | * Yedong continued to walk through the document 20 | * **Yedong**: as agreed on previous meeting, connectivity is a state that a device has, not an attribute. A device can be connected or not. 21 | * **Ofer**: Is there a scenario where a device doesn’t have any connectivity ? 22 | * **Yedong**: the assumption is that there’s always connectivity in the wider term. Not just internet, but Bluetooth/USB/etc. 23 | * **Saurabh**: How does having connectivity or not affects the edge devices ? 24 | * **Ofer**: It affects the use-cases that the device needs to support. In case of Hybrid Processing, network processing is split to two : part is processed on the device and part is in the cloud. As an example, take voice processing. A Hybrid scenario will have one network running on the device, passing it to the cloud for further processing. Without connectivity, all processing is done on the device. This implies different set of capabilities / processing power. 25 | * **Ofer**: Current way the document is written, it describes a single profile. It needs to be extended to multiple profiles 26 | * **Yedong**: We need to look at it from ONNX point of view. We said that “Edge” relates to quantized operators, and defines a subset of operators to be supported. 27 | * **Ofer**: Right, but each profile has different use cases. Based on the SoW we want to define conformance testing to validate use-cases, which in ONNX terms means set of models and validating the outputs. Trying to define one set that applies to all edge devices will either make this test set too small, or limit the types of devices that can pass it. That’s where profiles can help, by defining different test set per profile 28 | * **Ofer**: Looking at the SoW, we want to finish reviewing this document by end of this month. We have only one bi-weekly meeting scheduled by then, on June 26th. Shall we add another one on the June 19 th ? 29 | * **Yedong**: Agree. 30 | * **Saurabh**: Agree 31 | * **Ofer**: Good. Will take it with Milan, and update. 32 | ### Action Items 33 | * **Ofer**: Publish meeting minutes and recording. 34 | * Next meeting: We are considering adding one on Jun 19. We'll update on gitter. 35 | * **Yedong**: Make updates to the document based on discussion today. 36 | * **All**: Please post comments to document in [PR](https://github.com/onnx/working-groups/pull/6/). 37 | We will continue with review in next meeting. 38 | -------------------------------------------------------------------------------- /edge/meetings/20190619.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jun 19, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Continue reviewing "Edge scope and profile definition" document. 7 | * PR discussion: https://github.com/onnx/working-groups/pull/6/ 8 | 9 | ## Meeting Minutes 10 | * [Meeting recording](https://youtu.be/AR_RYjpZVKE) 11 | 12 | ### Attendees 13 | * Milan Oljaca (Qualcomm) 14 | * Yedong Liu (Huawei) 15 | * Ofer Rosenberg (Qualcomm) 16 | 17 | ### Notes 18 | * Yedong updated the document before the meeting based on comments from Ofer and Milan 19 | * Updated document is [Here](https://github.com/onnx/working-groups/pull/6/files#diff-5a9f20e897606cb780825b9766d9b172) 20 | * Yedong continued to walk through the document 21 | * All comments were pretty much accepted as is. 22 | * **Milan**: Suggested adding some diagrams into document, e.g. for depicting edge scope. Showed candidate diagram for "Scope of edge" section. 23 | * **Ofer**: Agreed, and proposed putting drawings into Google drive slides for ease of access and editing. 24 | * **Yedong**: Agreed. Also suggested putting drawing under "Edge profile section", e.g. radar chart from March workshop slides (slide 20). 25 | * **Milan**: Hard to know if currently identified profile attributes are sufficeint. We will not know until we finish defining edge profiles and corresponding scenarios. Should be ok, we can go back and update this document if we find a need for more attributes. 26 | * **Ofer**: suggested we test it, try to define 2-3 test profiles using existing attributes and see if they provide enough "resolutioin" to describe profiles. Took action item for next meeting. 27 | * **Ofer**: Should we use these attributes quantitatively or qualitatively? Should there be a range of values when defining profile? 28 | **Yedong**: Suggested we go with value ranges, that seems more practical. All agreed. We will test it when we start defining profiles. 29 | * We have one more meeting to go in June, set goal was to complete and publish this document in 2019Q2. 30 | * Goal is to achive that in next meeting, and complete review and have document ready to publish into artifacts 31 | 32 | ### Action Items 33 | * **Milan**: Publish meeting minutes and recording. 34 | * Next meeting: Jun 26, 2019. 35 | * **Yedong**: Make updates to the document based on discussion today. 36 | * **Ofer**: Define 2 test profiles using currently identified attributes. Will review in next meeting. 37 | * **Milan**: Publish block diagram drawings we discussed in the meeting to google drive doc. 38 | * **All**: Final review comments to document in [PR](https://github.com/onnx/working-groups/pull/6/). 39 | * Goal is to complete the review and have document ready to publish into artifacts. 40 | -------------------------------------------------------------------------------- /edge/meetings/20190626.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jun 26, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Finalize "Edge scope and profile definition" document review. 7 | * https://github.com/onnx/working-groups/pull/6/. All, please comment directly in PR. 8 | * Goal is to complete the review and have final document ready to publish into artifacts. 9 | * Miscellaneous 10 | * Operators SIG interaction 11 | 12 | ## Meeting Minutes 13 | * [Meeting recording](https://youtu.be/iNw9sVj9ZVQ) 14 | 15 | ### Attendees 16 | * Milan Oljaca (Qualcomm) 17 | * Yedong Liu (Huawei) 18 | * Ofer Rosenberg (Qualcomm) 19 | * Saurabh Tangri (Intel) 20 | 21 | ### Notes 22 | * We discussed next steps for finalizing [Edge scope and profile definition](https://github.com/onnx/working-groups/pull/6)document. 23 | * As per process we defined, document needs to be published to artifacts folder. 24 | * As per our SoW, goal to publish this document in 2019Q2. 25 | * We agreed to proceed as follows 26 | * Yedong to make one more update of the document, for new comments and add drawings. 27 | * Milan to merge PR afterwards. Document will show up in discussion_materials folder. 28 | * Continue review discussion (if needed) through an issue in github. We can do acceptance vote through the issue as well. 29 | * Create an PR for final document in "artifacts" folder. Merge once approved by the group. 30 | * Complete by Jun 30 31 | * **Milan**: Pointed to Operators SIG meetings. First meeting was 2 weeks ago. Another one today. 32 | See [Operators SIG gitter](https://gitter.im/onnx/operators). There is agenda item (organize operators by domain) which may have overlap with edge profile definition. 33 | Milan plans to attend today's meeting to bring it up. Saurabh will follow up with participants from Intel in that regard as well. 34 | * **Milan**: ONNX SIGs steering committee published [SIGs Best Practices](https://github.com/onnx/sigs/blob/master/best-practices.md) and asked if the same set of rules should be applied to WGs. Milan replied that it is fine, since that is already what Edge WG is doing. 35 | * Next meeting 36 | * Review a few "prototype" profiles (main purpose is to test profile definition through attributes) 37 | * Review and establish profile definition template. 38 | 39 | 40 | ### Action Items 41 | * **Milan**: Publish meeting minutes and recording. 42 | * Next meeting: Jul 10, 2019, 8:00am PST 43 | * **Yedong**: Make updates to the document PR based on discussion today. 44 | * **Milan**: Merge PR. Conduct acceptance vote and publish to artifacts. Will coordinate with Yedong. 45 | * **Ofer**: Define 2 test/prototype profiles using currently identified attributes. Will review in next meeting. 46 | * **Milan**: Profile definition template proposal 47 | 48 | -------------------------------------------------------------------------------- /edge/meetings/20190710.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jul 10, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | 7 | * Review a few "prototype" profiles (main purpose is to test profile definition through attributes) 8 | * Profile definition template proposal: discussion 9 | * Update about SIGs and their activity, and interaction we should take 10 | 11 | ## Meeting Minutes 12 | * [Meeting recording](https://youtu.be/FUwzJMuWaUY) 13 | 14 | ### Attendees 15 | * Milan Oljaca (Qualcomm) 16 | * Yedong Liu (Huawei) 17 | * Ofer Rosenberg (Qualcomm) 18 | 19 | ### Notes 20 | * Short meeting today. No review material for the agenda items. 21 | * Groups decided to have a follow up meeting next week. 22 | * **Milan**: Short update on acceptance status of "Edge scope and profile definition" document. As per replies in https://github.com/onnx/working-groups/issues/7 document is accepted. Did not receive reply from Saurabh yet, but majority of contributors approved it. 23 | Next step is to copy document to artifacts folder. 24 | * **Milan**: Dialed in to last Operators SIG meeting and brougth up Edge WG work in context of operator domains, to ensure we do not duplicate work. SIGs first reaction was that there is no overlap, but will see, now they are aware of Edge WG work. 25 | * **MIlan**: Other SIGs came alive in gitter and github as per defined process: Infra, Converters 26 | 27 | ### Action Items 28 | * **Milan**: Publish meeting minutes and recording. 29 | * Next meeting: Jul 17, 2019, 8:00am PST 30 | * **MIlan**: Copy "Edge scope and profile definition" document from discussion_materials to artifact folder. 31 | * **Ofer**: Define at least one "prototype" profile using currently identified attributes. Will review in next meeting. 32 | * Work with Milan to combine that with proposal for profile definiton template. 33 | 34 | -------------------------------------------------------------------------------- /edge/meetings/20190724.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jul 24, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | 7 | * Review a "prototype" profile 8 | * Test profile definition through attributes 9 | * Consider informaton and structure for profile definition template 10 | 11 | ## Meeting Minutes 12 | * [Meeting recording](https://youtu.be/_3S3TUyZI10) 13 | 14 | ### Attendees 15 | * Milan Oljaca (Qualcomm) 16 | * Yedong Liu (Huawei) 17 | * Ofer Rosenberg (Qualcomm) 18 | 19 | ### Notes 20 | * **Ofer** shared proposal for "Stationary IoT device with AV" profile. Group discussed and suggested some changes. In general, content and structure looked ok and could be used as a template. 21 | There was discussion around profiles' hierarchy or levels, organizing profiles based on their similarities. 22 | **Milan** suggested group should define top level profile breakdown map. 23 | 24 | ### Action Items 25 | * **Milan**: Publish meeting minutes and recording. 26 | * Next meeting: Aug 7, 2019, 8:00am PST. To be announced via gitter. 27 | * **Ofer**: Upload prototype profile we discussed to discussion_materials folder 28 | * **Milan**: Copy "Edge scope and profile definition" document from discussion_materials to artifact folder. 29 | * **Milan**: Propose top level profile breakdown map. Will review in next meeting. 30 | 31 | 32 | -------------------------------------------------------------------------------- /edge/meetings/20190807.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Aug 7, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | 7 | * Proposal for top level profile breakdown map 8 | * Aug 23 Workshop alignment 9 | 10 | ## Meeting Minutes 11 | * [Meeting recording](https://youtu.be/vV0ig2ZyOGQ) 12 | 13 | ### Attendees 14 | * Milan Oljaca (Qualcomm) 15 | * Saurabh Tangri (Intel) 16 | 17 | ### Notes 18 | * **Milan** Due to light attendance, decided not to discuss profile topology map proposal. 19 | * Will consider another meeting next week. 20 | * **Milan** Aug23 workshop - asking for input from contributors about info we should share. 21 | * Planning to review workshop update slides in WG Aug 21 meeting. 22 | * General discusson 23 | * Updated Saurabh about discusisons in last 2 meetings. 24 | * Discussed end goals of edge profiles. Compliance and interaction with Architecture/Infrastructure SIG. 25 | * **Saurabh** Proposed to put together "Profile Compliance Workflow" proposal. To be added to agenda for next meeting. 26 | 27 | ### Action Items 28 | * **Milan**: Publish meeting minutes and recording. 29 | * Next meeting: Aug 14 or 21, 2019, 8:00am PST. To be announced via gitter. 30 | * Follow up on gitter about having meeting next Wed 31 | * **Saurabh**: Prepare "Profile Compliance Workflow" proposal. 32 | 33 | 34 | -------------------------------------------------------------------------------- /edge/meetings/20190821.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Aug 21, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | 7 | * Review WG update slides for Aug23 workshop 8 | * Review proposal for top level profile breakdown map 9 | * Review "Profile Compliance Workflow" proposal 10 | 11 | ## Meeting Minutes 12 | * [Meeting recording](https://youtu.be/XLYhGEPW3mM) 13 | 14 | ### Attendees 15 | * Yedong Liu (Huawei) 16 | * Ofer Rosenberg (Qualcomm) 17 | * Saurabh Tangri (Intel) 18 | * Milan Oljaca (Qualcomm) 19 | 20 | ### Notes 21 | * **Milan** Updated everybody about communication initiated with Steering Committee, in preparation for the workshop, raised concerns about community participation and ultimatelly relevance of Edge WG. 22 | * Will bring it up in the workshop. 23 | * **Milan** Presented draft of slides for Aug23 workshop 24 | * Groups reviewed, we made some changes, but in general looked ok. 25 | * Ofer made coment we should flush out what edge is status update. 26 | * Saurabh indicated perhaps we should highlight that profiles are about execution/inference compliance 27 | * Need to add some slides for Breakout session as per agenda. Ofer - about prototype profile, Saurabh - Profile compliance workflow 28 | * Group agreed, Milan will share slides over the email for further review, and Ofer will schedule a follow up meeting tomorrow to finalize. 29 | * Milan is out on vacation starting tomorrow. Ofer will be master editor of the slides and presenter in the workshop. 30 | * Discussed proposal for profile topology map 31 | * https://github.com/onnx/working-groups/blob/master/edge/discussion_materials/onnx-profile-topology-map.md 32 | * Ofer - good approach, profile definiton can be tied to data sensor/input type, so today it is 2d image and audio, but new profiles can be added in future for other input types 33 | * Saurabh suggested that data locality does not have to be binary attribute, could be connected to bandwidth requirment 34 | * In general proposal is worth pursuing further, we should bring it up in workshop for discussion. 35 | * Did not get to third item on agenda, "Profile Compliance Workflow" proposal. To be added to agenda for next meeting. 36 | 37 | ### Action Items 38 | * **Milan**: Publish meeting minutes and recording. 39 | * Next meeting: Sep 4, 2019, 8:00am PST. To be announced via gitter. 40 | * Prepare and review slides for Breakout Session. 41 | * **Ofer**: Prototype profile, profile topology map 42 | * **Saurabh**: profile coompliance workflow 43 | * **Ofer**: Schedule meeting for tomorrow Aug 22, to review 44 | 45 | 46 | -------------------------------------------------------------------------------- /edge/meetings/20190904.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Sep 04, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | 7 | * Aug23 workshop follow up - summary 8 | * What do we do next? 9 | * Edge profile as execution mode control 10 | * Profile topology map 11 | 12 | ## Meeting Minutes 13 | * [Meeting recording](https://youtu.be/KsjM9m9p_L0) 14 | 15 | ### Attendees 16 | * Yedong Liu (Huawei) 17 | * Ofer Rosenberg (Qualcomm) 18 | * Saurabh Tangri (Intel) 19 | * Milan Oljaca (Qualcomm) 20 | 21 | ### Notes 22 | * Workshop readouts from Ofer and Saurabh 23 | * [ONNX_WorkshopAug2019-Edge_WG_Session.pdf](https://github.com/onnx/working-groups/blob/master/edge/workshops/ONNX_WorkshopAug2019-Edge_WG_Session.pdf) 24 | * Discussed [Edge profile as execution mode control](https://github.com/onnx/working-groups/commit/7539dce2cf6c45695ae8717ede6f1d2528c528ee#diff-6ea8a11a2d9744be5a0a63ad62405f02) 25 | * Proposal requires extendig the scope and updates to SoW, as it alters meaning of the profile we specified so far, which is about devide comliance/certification. 26 | * Milan suggested we do not change WG SoW at this point. Execution control is a good idea but seems to be a topic beyond just Edge which is focus of this WG. 27 | We could make recommendation to Infra SIG to consider though. Or perhaps a new WG could be formed? 28 | * Milan suggested we proceed with definiton of profiles as we agreed upon so far and see how well it comes out, and then consider in what ways it could be extended towards execution control. 29 | * Saurabh thought that Edge profiles are fairly static and do not scale well for devices such as laptop as it's capabilities are extensible (e.g. by virtue of plugging in a usb stick). 30 | * Milan stated that device characteristics (such as compute capabilities) need to be specified at time of compliance test. Different laptop configuration is different device spec. 31 | 32 | 33 | ### Action Items 34 | * **Milan**: Publish meeting minutes and recording. 35 | * Next meeting: Sep 18, 2019, 8:00am PST. To be announced via gitter. 36 | * **Milan**: Prepare one topology map profile for review: 2D Image - Basic 37 | 38 | 39 | -------------------------------------------------------------------------------- /edge/meetings/20190925.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Sep 25, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * General sync up on action items 7 | 8 | ## Meeting Minutes 9 | * [Meeting recording](https://youtu.be/gXx21onDLGs) 10 | 11 | ### Attendees 12 | * Yedong Liu (Huawei) 13 | * Manash Goswami (Microsoft) 14 | * Milan Oljaca (Qualcomm) 15 | 16 | ### Notes 17 | * It was ahort meeting, primarily to sync up on next steps 18 | * Manash from Microsoft joined WG for the first time. Planning to be regular going forward. 19 | * Action item from Sep 4 meeting, to prepare one topology map profile for review: 2D Image - Basic. 20 | * Milan explained could not get to it due to heavy work obligations 21 | * Discussed milestones 22 | * Manash suggested we set some deliverable goals before next workshop (Nov 18 in Shanghai) 23 | * Looked over existing SoW. We need to update milestones to adjust to current situation. 24 | * Manash suggested we could try to align profile toloplogy definiton with version of ONNX spec release, 1.7 is coming in Q4 25 | 26 | 27 | ### Action Items 28 | * **Milan**: Publish meeting minutes and recording. 29 | * Next meeting: Oct 9, 2019, 8:00am PST. To be announced via gitter. 30 | * **Milan**: Propose updated timeline/milestones 31 | * **Milan**: Will try to prepare "2D Image - Basic" profile draft 32 | 33 | -------------------------------------------------------------------------------- /edge/meetings/20191016.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Oct 16, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Updates to WG timeline/milestones and SoW 7 | 8 | ## Meeting Minutes 9 | * [Meeting recording](https://youtu.be/oshcYdcxh5U) 10 | 11 | ### Attendees 12 | * Saurabh Tangri (Intel) 13 | * Yedong Liu (Huawei) 14 | * Manash Goswami (Microsoft) 15 | * Ofer Rosenberg (Qualcomm) 16 | 17 | ### Notes 18 | * Started by going over the [Edge WG SoW](https://github.com/onnx/working-groups/blob/master/edge/artifacts/onnx-edge-wg-sow.md) 19 | * Identified the following issues in deliveries : 20 | * No.3 needs to be updated to reflect the new apporach, of multiple profiles per device 21 | * No.4 needs to change to "collaborate with Operator SIG", as there's no quantization WG 22 | * No.5 - currently no active ModelZoo SIG. Put on hold 23 | * Add a new delivery : creating a document (white paper) describing the different options for sets of profiles, with initial suggested set per option. See more details below. 24 | * Adjust goals as follows : 25 | * 2019Q3 - write the white paper described above, do a survery to select between options, and decide on an intial set 26 | * Defining the Profiles 27 | * Manash suggested to capture the set of profiles first, before defining them 28 | * A few different ways to define profiles : 29 | * Initial direction : one profile per device (like "smart speaker") - suggestion is to depreate this approach 30 | * Based on "sensor" type (Video, Audio, Speech) and level (Low, Mid, High) 31 | * Based on Network type (Classification, Segmentation, NLP) and level 32 | * Ofer suggested to write a white paper and create a survey, similar what the foundation WG did 33 | * Applying/using the Profiles 34 | * Ofer - Profiles can be provided to the runtime/backends in few ways : 35 | * A new optional field in the ONNX container. 36 | * Not mandatory, as it may be hard for converter to support it initially - the App/Runtime may come and "edit" the container to add a profile before sending it to the backend 37 | * To implemnt it, converters will need to know what each profile means to examine the model and fill the value 38 | * A sideband API to specify the profile 39 | * Manash - How are profiles supported from the HW point of view ? 40 | * Ofer - One way is to document on a website which HW supports which profiles (based on conformance). Another way is to have an API to specify which profiles are supproted 41 | * Ofer - IMO, APIs are outside of the scope of the current WG. In general the ONNX project focused on the container, and the way to use it (e.g APIs) is left for frameworks and runtimes implementers 42 | * Saurabh - Static vs. Dynamic profiles. An App may want to directly control attribute value, for example run a model with accuracy which is higher than specified in the static profile, or latency which is smaller 43 | * Ofer - If this is part of an API, believe its outside of the scope. But, if we suggest that profiles are embedded in the container, than its possible to support it by allowing to specify addtibute values. 44 | 45 | ### Action Items 46 | * **Ofer**: Publish meeting minutes and recording. 47 | * **Ofer**: Update SoW based on discussion 48 | * **Ofer**: Propose initial draft for profile sets white paper, and work out survey 49 | -------------------------------------------------------------------------------- /edge/meetings/20191030.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Oct 30, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Review updated SoW 7 | * https://github.com/onnx/working-groups/pull/21/commits/aaf1fa79f648a5f6554edefa046293982a1754cd 8 | * Profiles set topology mapping proposal 9 | * https://github.com/onnx/working-groups/pull/22/commits/a16ba2d35c52b5928189ee1c6cdb2cd56dce81a0 10 | 11 | ## Meeting Minutes 12 | * [Meeting recording](https://youtu.be/G6etBdep-gM) 13 | 14 | ### Attendees 15 | * Saurabh Tangri (Intel) 16 | * Yedong Liu (Huawei) 17 | * Ofer Rosenberg (Qualcomm) 18 | * Milan Oljaca (Qualcomm) 19 | 20 | ### Notes 21 | 22 | ### Action Items 23 | * **Yedong**: Prepare draft slides for workshop in Shenghai. 24 | 25 | -------------------------------------------------------------------------------- /edge/meetings/20191113.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Nov 13, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Review slides for Edge WG session at ONNX workshop Shanghai 7 | * https://docs.google.com/presentation/d/16Fr39AVKXzApd1DvoJvzh2zHUilYdmzraUPACuO5hEw/edit?usp=sharing 8 | 9 | ## Meeting Minutes 10 | * [Meeting recording](https://youtu.be/aglO3bCbH3o) 11 | 12 | ### Attendees 13 | * Saurabh Tangri (Intel) 14 | * Ofer Rosenberg (Qualcomm) 15 | * Milan Oljaca (Qualcomm) 16 | * Manash Goswami (Microsoft) 17 | 18 | ### Notes 19 | * Much discussion about static profile compliance and dependency on definiton of model execution compliance (how to run). 20 | 21 | ### Action Items 22 | * **Saurabh**: Prepare slides capturing the above dependency,to include in draft slides for workshop in Shenghai. 23 | 24 | -------------------------------------------------------------------------------- /edge/meetings/20191115.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Nov 15, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Review slides for Edge WG session at ONNX workshop Shanghai 7 | * https://docs.google.com/presentation/d/16Fr39AVKXzApd1DvoJvzh2zHUilYdmzraUPACuO5hEw/edit?usp=sharing 8 | * Intro slides from Saurabh 9 | 10 | ## Meeting Minutes 11 | * [Meeting recording](https://youtu.be/FV-WNdEoLM4) 12 | 13 | ### Attendees 14 | * Saurabh Tangri (Intel) 15 | * Ofer Rosenberg (Qualcomm) 16 | * Milan Oljaca (Qualcomm) 17 | * Yedong Liu (Huawei) 18 | 19 | ### Notes 20 | * Reviewd slides, added opening "Progress Report" slide. 21 | 22 | ### Action Items 23 | * **Saurabh**: Prepare slides capturing the above dependency, to include in draft slides for workshop in Shenghai. 24 | 25 | -------------------------------------------------------------------------------- /edge/meetings/20191204.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Dec 04, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Nov 18 workshop readout (Yedong) 7 | * Discuss path forward 8 | 9 | ## Meeting Minutes 10 | * [Meeting recording](https://youtu.be/a_2xBHU1RgQ) 11 | 12 | ### Attendees 13 | * Saurabh Tangri (Intel) 14 | * Ofer Rosenberg (Qualcomm) 15 | * Milan Oljaca (Qualcomm) 16 | * Manash Goswami (Microsoft) 17 | * Yedong Liu (Huawei) 18 | 19 | ### Notes 20 | * Largely discussed path forward, considered if Edge WG needs to continue ... 21 | 22 | ### Action Items 23 | * **Milan**: Prepare draft of the letter to SC seeking guidance about future of Edge WG. 24 | 25 | -------------------------------------------------------------------------------- /edge/meetings/20191218.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Dec 18, 2019 at 8:00am PST 4 | 5 | ## Agenda 6 | * Discuss messaging to/with SC on future WG engagements 7 | 8 | ## Meeting Minutes 9 | * [Meeting recording](https://youtu.be/a_2xBHU1RgQ) 10 | 11 | ### Attendees 12 | * Saurabh Tangri (Intel) 13 | * Milan Oljaca (Qualcomm) 14 | * Yedong Liu (Huawei) 15 | * Rohit Sharma (dnnCompiler) - for last half of the meeting 16 | 17 | ### Notes 18 | * Reviewed letter to SC 19 | 20 | Dear ONNX SC members, 21 | 22 | As a co-chair, I am writing this letter representing the ONNX Edge WG. The below content is reviewed by the current active members of the WG. We are seeking an advice on matters that involve future WG engagements. 23 | 24 | The Edge WG is at the point where we are not certain if WG should continue, or perhaps be paused for a while, and want to get guidance from SC. There are several reasons that we are considering this. 25 | 1. Original SoW has been under review for last 3 months. The WG have started with idea of defining static edge profiles, defining attributes and methodology which would be used by various hardware vendors for compliance purposes. There is ongoing debate about need to define execution mode compliance and related to general purpose computing devices and how/if they can be truly compliant to static edge profiles. This topic has been brought up in last workshop in Shanghai. 26 | 2. We have representatives from only 3 hardware vendors actively participating, so there is a question of legitimacy for the community of any approach we select. There are many hardware vendors out there, and as it appears not much interest on this topic. 27 | 3. The execution mode / operating environment control recommendations (e.g. through an API or ONNX model attributes or something else) have larger scope than Edge. It is more of the Infrastructure SIG topic. 28 | 29 | We are looking for the guidance from SC about how we should proceed. The scope that we started with appears to be constraining us, it evolved into broader domain which has impact beyond our definition of Edge. Perhaps a new WG should be formed to address it. 30 | Given above, we are recommending that Edge WG should at least be paused until there is evidence that there is enough interest in the topic. Alternatively, it could evolve into different WG with charter to cover broader scope of compliance. 31 | 32 | Best regards. 33 | ONNX Edge WG 34 | 35 | 36 | ### Action Items 37 | * **Milan**: Send email to Prasanth Pulavarthi, CC WG members 38 | 39 | -------------------------------------------------------------------------------- /edge/meetings/20200108.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jan 08, 2020 at 8:00am PST 4 | 5 | ## Agenda 6 | * Follow up on letter to SC about Edge WG progress and future 7 | 8 | ## Meeting Minutes 9 | * [Meeting recording](https://youtu.be/zA7urDFQdFA) 10 | 11 | ### Attendees 12 | * Prasanth Pulavarthi (Microsoft) 13 | * Saurabh Tangri (Intel) 14 | * Milan Oljaca (Qualcomm) 15 | * Yedong Liu (Huawei) 16 | * Ofer Rosenberg (Qualcomm) 17 | 18 | ### Notes 19 | * Prasanth shared feedback from ONNX SC 20 | * Edge WG letter was discussed by the steering committee on 12/19. 21 | * SC agreed that it makes sense to end the Edge WG for now. WGs are meant to be short term with a specific scope. Its completely okay if the conclusion is that the scope is not right or not of enough interest. We can always spin up another WG with the same or different scope when the time is right. 22 | * Edge WG contributors present in the meeting voted and made decison to retire Edge WG. 23 | * From active contributors, only Manash was not present 24 | * Prasanth will share decision with SC 25 | 26 | ### Action Items 27 | * **Milan**: Capture meeting minutes 28 | * **Milan**: Announce Edge WG retirement on onnx/edge and onnx/Lobby gitter channels 29 | 30 | 31 | -------------------------------------------------------------------------------- /edge/workshops/ONNX_WorkshopAug2019-Edge_WG_Session.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/workshops/ONNX_WorkshopAug2019-Edge_WG_Session.pdf -------------------------------------------------------------------------------- /edge/workshops/ONNX_WorkshopMar2019-Edge&MLPerf.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/workshops/ONNX_WorkshopMar2019-Edge&MLPerf.pdf -------------------------------------------------------------------------------- /edge/workshops/ONNX_WorkshopMar2019-Edge_WG_Session.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/workshops/ONNX_WorkshopMar2019-Edge_WG_Session.pdf -------------------------------------------------------------------------------- /edge/workshops/ONNX_WorkshopNov2019-Edge_WG_Session.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/edge/workshops/ONNX_WorkshopNov2019-Edge_WG_Session.pdf -------------------------------------------------------------------------------- /foundation/LF AI - ONNX Community Discussion.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/foundation/LF AI - ONNX Community Discussion.pdf -------------------------------------------------------------------------------- /foundation/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | __Status update__: _On October 17, 2019 the ONNX Steering Committee voted to join Linux Foundation AI (LF AI). On October 24, 2019 the LF AI Technical Advisory Board voted to accept our [proposal](https://github.com/lfai/proposing-projects/blob/master/proposals/onnx.adoc) to join LF AI and on October 31, 2019 the LF AI board unanimously approved ONNX joining LF AI as a Graduate Project._ 4 | 5 | _A formal announcement will be made on November 14, 2019. If you are an ONNX member company and would like to provide a quote for the press release from LF AI, please contact [Prasanth Pulavarthi](mailto:Prasanth.Pulavarthi@microsoft.com)._ 6 | 7 | # ONNX Foundation Working Group 8 | 9 | This repository is where ONNX Foundation WG will capture various artifacts and deliverables. The purpose of this WG is to identify and evaluate non-profit foundation options for the ONNX consortium, then execute on best option. 10 | 11 | __Status update__: _On October 17, 2019 the ONNX Steering Committee voted to join Linux Foundation AI (LF AI). On October 24, 2019 the LF AI Technical Advisory Board voted to accept our [proposal](https://github.com/lfai/proposing-projects/blob/master/proposals/onnx.adoc) to join LF AI and on October 31, 2019 the LF AI board unanimously approved ONNX joining LF AI as a Graduate Project._ 12 | 13 | _A formal announcement will be made on November 14, 2019. If you are an ONNX member company and would like to provide a quote for the press release from LF AI, please contact [Prasanth Pulavarthi](mailto:Prasanth.Pulavarthi@microsoft.com)._ 14 | 15 | 16 | ## Working Group Status 17 | **ACTIVE** 18 | 19 | ## Contributors 20 | *Note: Contributors list will be updated based on participation.* 21 | * Jim Spohrer (IBM) (co-lead) 22 | * Ryan Loney (Intel) (co-lead) 23 | * Alex Sergeev (Uber) 24 | * David Aronchick (Microsoft) 25 | * Wenjing Chu (Futurewei) 26 | 27 | ## Meeting logistics 28 | * WG chair(s) will drive and facilitate the meetings 29 | * Publish agenda, produce meeting notes, action items, etc. 30 | * Meetings are weekly 31 | * ***Thursdays 1:00pm PST***, starting Sep 5, 2019 until November 1, 2019. 32 | * Will post a message to [ONNX Foundation Gitter Channel](https://gitter.im/onnx/foundation) with agenda and meeting invite link 33 | * Up to 7 days before the meeting by WG chair, but proposals to agenda item updates are welcomed from all contributors via gitter 34 | * Meeting/Telepresence tool: WebEx 35 | * Meetings will be recorded and published to [ONNX YouTube channel](https://www.youtube.com/channel/UCIg4Cceqra3rtJEC5LPTdtw) 36 | * September 10, 2019 meeting with Eclipse: https://youtu.be/uX74-EyGp7Q 37 | * September 11, 2019 meeting with LF AI: https://youtu.be/9xSPLnCDRMc 38 | * [LF AI slide deck](https://github.com/onnx/working-groups/blob/master/foundation/LF%20AI%20-%20ONNX%20Community%20Discussion.pdf) 39 | * September 12, 2019 meeting with Apache: https://youtu.be/t5HENqpEdRw 40 | * September 20, 2019 meeting with LF JDF: https://youtu.be/IpDFHC0mzqY 41 | 42 | 43 | ## Discussion mechanics 44 | * [ONNX Foundation WG Gitter Channel](https://gitter.im/onnx/foundation) 45 | * [ONNX Foundatiom WG repo](https://github.com/onnx/working-groups/foundation) 46 | * Document reviews 47 | * We will use Google docs for draft reviews until we get to final doc proposal, at which point it will get transcribed into .md and committed to [ONNX Foundation WG repo](https://github.com/onnx/working-groups/foundation). 48 | * [Link to draft proposal](https://docs.google.com/document/d/1nGV8un_BcOeAbIiDP-Ze9Ar2_QUpp5okoCWhxv0cyAo/edit?usp=sharing) 49 | * [Foundation Options Summary](https://github.com/onnx/working-groups/blob/master/foundation/foundation-options.md) 50 | * A summary of the options considered is published on onnx/working-groups/foundation repo. 51 | * [Foundation Options Poll](https://forms.gle/RSfhq12oPG3hgXGH7) 52 | * We are collecting input from the ONNX Community on their preferences for each option. Please vote for your top choices using the Poll. 53 | -------------------------------------------------------------------------------- /generative-ai/ReadMe.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Generative AI Working Group (WG) 4 | 5 | This repository is where the ONNX Generative AI (GenAI) WG will capture various artifacts and deliverables. 6 | 7 | ## Objective 8 | 9 | As discussed in the ONNX steering committee meeting on 4/2/2025 ([meeting minutes](https://github.com/onnx/steering-committee/blob/main/meeting-notes/2025/20250402.md#growing-onnx-with-evolving-needs)), we are establishing this working group to address the evolving needs of ONNX to support Generative AI. The scope of this working group includes, but not limited to: 10 | 11 | ### ONNX Functions and Operators 12 | 13 | - Propose new decomposable ONNX functions 14 | - New Metadata required to describe device specific graphs or Function groupings. 15 | - Reference implementations for the proposed ONNX functions 16 | - Standardize popular GenAI operations in ONNX to reduce the usage of contrib ops 17 | - Define methods/scripts for converting existing models to leverage newly defined ONNX operators and functions 18 | - Ensure that the framework converters are updated to export the ONNX models with newly defined operators 19 | 20 | ### Generative AI Pipelines 21 | 22 | - Define GenAI pipelines for end-to-end execution of GenAI ONNX models 23 | - Standardize high-level APIs that can be integrated into applications for usages like text generation, image generation 24 | - Define interfaces for vendors to plug in their optimized implementations for GenAI pipelines 25 | - Define pipeline constructs to enable seamless development of new pipelines 26 | - Provide pipeline optimizations including decoding strategies, kv cache management, efficient parallelism and batching techniques 27 | - Explore constructs needed for building Agentic and reasoning workflows 28 | 29 | ## Working Group Status 30 | **ACTIVE** 31 | 32 | # Slack channel 33 | https://lfaifoundation.slack.com/archives/C08MERYU84T 34 | 35 | # WG Lead(s) 36 | 37 | * Yamini Nimmagadda (Intel) 38 | * Ramakrishna Sivakumar (AMD) 39 | 40 | # Logistics 41 | 42 | * WG leads will drive the meeting. 43 | * Meeting annoucements will be posted in our Slack channel: https://lfaifoundation.slack.com/archives/C08MERYU84T 44 | * Feedbacks and topic requests are welcome from everyone. 45 | * Documents and artifacts: https://github.com/onnx/working-groups/tree/main/generative-ai 46 | 47 | # Meeting notes 48 | 49 | The meeting notes can be found [here](https://github.com/onnx/working-groups/tree/main/generative-ai/meetings) 50 | -------------------------------------------------------------------------------- /generative-ai/meetings/ReadMe.md: -------------------------------------------------------------------------------- 1 | The meeting minutes from the ONNX Generative AI working group will be posted in this directory 2 | -------------------------------------------------------------------------------- /generative-ai/meetings/meeting_1_date_April_23.md: -------------------------------------------------------------------------------- 1 | # Recording and Transcript: 2 | 3 | https://zoom.us/rec/share/UmJO9yfv2ZzMt2lnG0VKP59SsHnLwR_yA-0ybOw3N4Tv5XYxvG-TSPsENJC8-LX0.ZzyJp4IihZYLZ4FF 4 | 5 | # Meeting Minutes: 6 | 7 | - Continued from last week's overview of the working group's scope; this session focused on diving deeper, identifying actionable items, and considering breakout groups. The material shared in the meeting can be found [here](https://docs.google.com/presentation/d/1PYAHauEVhhdTuKMYiOsOjOVz6u8OzWka/edit?slide=id.g34f6d699a61_0_0#slide=id.g34f6d699a61_0_0): 8 | 9 | ## ONNX operators/functions: 10 | - Preference is to define operations as functions rather than operators when they can be composed from existing primitives 11 | - Nested functions are supported in ONNX (as long as there's no recursion), allowing functions to use other function-based ops 12 | - There's limited usage of ONNX functions in practice today; need to improve adoption via exporters and tools like Optimum or ModelBuilder 13 | - Need to discuss how function usage can be preserved in exported graphs so that backends can recognize and optimize them instead of seeing expanded subgraphs – this doesn’t seem to be the case today 14 | - There was a discussion about FlexAttention and whether we can use it in ONNX to represent different attention variants. Rama mentioned that it is more complex than traditional attention and behaves like a higher-order operator and needs some sort of control flow 15 | - Yamini proposed to develop PoCs that express different attention variants using FlexAttention to evaluate feasibility 16 | 17 | ## Contrib ops: 18 | - There is interest in standardizing commonly used contrib ops as ONNX functions. There's a need for better documentation and tooling around contrib ops. The best way to understand the implementation is to check the code 19 | - Shubham raised a PR to implement the SkipLayerNormalization, but there were concerns expressed by Yuan about whether this was intended for fusion pattern and where to draw the line between adding genuinely useful patterns vs. overloading the spec with too many function ops 20 | - Proposal to review common contrib ops and select candidates to elevate to standardized ONNX functions or operators 21 | 22 | ## Backend-specific representations: 23 | - Can we include optimized subgraphs or precompiled blobs in an ONNX model while keeping it hardware-agnostic? 24 | - One idea: use model-local functions to define backend-specific function variants, allowing each backend to select its optimized version 25 | - This enables ahead-of-time (AOT) compilation without turning the model into a hardware-specific blob 26 | - Shared weights should be referenced, not duplicated, across backend-specific functions to avoid bloating 27 | - Suggestion to introduce a generalized version of ONNX Runtime’s EP context nodes, which can bundle custom logic and refer to shared weights 28 | - For quantized models, QDQ pattern matching must be performed before function inlining to preserve valid quantization patterns 29 | - Cleaner approach: allow function definitions to include QDQ variants as attributes, enabling the inliner to insert them properly 30 | 31 | ## Proposed Breakout groups: 32 | - Group #1: Identify, define, and implement new ONNX functions from popular models, contrib ops, GenAI pipelines 33 | - Group #2: Analyze the current state of model exporters. Identify gaps and implement better function usage in exporters 34 | - Group #3: Define backend specific representations to support aot compilation, weight sharing, structured function inlining etc. 35 | 36 | Please let Yamini know if you are interested in participating in the above breakout groups and how you would like to contribute 37 | -------------------------------------------------------------------------------- /generative-ai/meetings/meeting_2_date_April_30.md: -------------------------------------------------------------------------------- 1 | # Recording and Transcript: 2 | 3 | https://zoom.us/rec/share/ZgK5OQxRDc8YD5obHRMMMVwRy7daTL5t90zK0742cCXEaH4s_fIOOrg4tOCxRi-G.Z2W99T-5kgy1t_ya 4 | 5 | # Meeting Minutes 6 | 7 | This week’s discussion is focused on the 3 breakout groups for operators, exporters, and backend specific representations. The slides from today’s discussion are [here](https://docs.google.com/presentation/d/1PYAHauEVhhdTuKMYiOsOjOVz6u8OzWka/edit?slide=id.g353238bb0c4_0_0#slide=id.g353238bb0c4_0_0). Below are the notes: 8 | 9 | ## Group #1 – Operators: 10 | - Contrib op Analysis: 11 | - There are 112 ops in contrib op list today 12 | - We should identify some criteria to identify ops that can be standardized from contrib ops. Below are some suggestions: 13 | - Github issues or feature requests raised by community to support certain patterns or contrib ops 14 | - Customer usage 15 | - How models are represented in other frameworks and exported 16 | - The group reviewed ops in the [sheet](https://docs.google.com/spreadsheets/d/1JIykwXJEPT8FTLzvb0_5p_odvTYS9zzZjPNN33ziEbQ/edit?gid=0#gid=0) 17 | - Inputs from Kunal and Rama: 18 | - Operator additions (e.g., rotary embedding, attention, RMSNorm) were based on model usage feedback within Microsoft. 19 | - Flex attention is of interest and want to explore it for representing different attention variants 20 | - Beam search and greedy search were initially added as contrib ops due to limitations in ONNX representation. 21 | - Today, ONNX Runtime GenAI is preferred for generation loops rather than encoding them as static ONNX graphs. 22 | - Sid to analyze the search ops if they can benefit from ONNX representation and potentially backend acceleration. 23 | - Paged Attention is also of interest, particularly to handle continuous batching requests. There is a PR for CUDA EP: https://github.com/microsoft/onnxruntime/pull/24595 24 | - Recommenders to add description in the column D of the sheet to the proposed operators 25 | - Max suggested introducing a special I/O concept in ONNX to mark tensors as reusable cache (e.g., for KV caching), avoiding separate input/output definitions and enabling more efficient memory handling that can help with stateful execution. This might not be an operator, could be an IR extension. It is documented in the sheet to track the idea 26 | 27 | ## Group #2 – Exporters: 28 | - Kshitij has been doing experiments to understand the status of pytorch exporter and observed that functions are not preserved with latest opsets. Many ONNX functions exist but aren’t seen in pytorch exported models 29 | - Rama mentioned that his team is working on pytorch exporter. Current exporter uses low-level ops, but work is ongoing to building fusion passes to detect and replace subgraphs with high-level ops like Attention. 30 | - Torchdynamo based onnx exporter will be prioritized going forward 31 | - Alex asked if the fusions are part of PyTorch exporter or ORT. Rama mentioned that the fusion logic lives in onnxscript and is not tied to ORT 32 | - Yamini asked if this group should be looking into this analysis as Rama’s team is already aware of the issues and looking into it. Rama welcomed more contributors to join and help accelerate the effort 33 | 34 | ## Group #3 – Backend representations: 35 | 36 | - The group briefly discussed about backend-specific representation and if we can use the EPContext design in a standard way 37 | - Max also suggested that we look into quantization representation as it is different for different backends 38 | - Yamini suggested the breakout group to come up with a proposal, do necessary experiments/POCs and present in 2 weeks 39 | 40 | ## Logistics: 41 | - Meeting will be extended to 45 mins going forward to allow more time for discussions 42 | - Below are the 3 channels for the breakout groups. Please feel free to join one or all of them and use those channels for discussion and exchange of ideas: 43 | - Breakout group for operators: https://lfaifoundation.slack.com/archives/C08QJA20HV1 44 | - Breakout group for exporters: https://lfaifoundation.slack.com/archives/C08QD56HLUD 45 | - Breakout group for backend representations: https://lfaifoundation.slack.com/archives/C08R47MGK16 46 | -------------------------------------------------------------------------------- /generative-ai/meetings/meeting_3_date_May_7.md: -------------------------------------------------------------------------------- 1 | # Recording and Transcript: 2 | 3 | https://zoom.us/rec/share/wWuUfGL8Jm_MMMt5MvDsHsFresKX9wOd92rOzMDl37JYurkBkEYuI69l60lnHr2R.zLi6dzeGKL1hZ7u3 4 | 5 | # Meeting Minutes: 6 | 7 | ## Operators: 8 | - The group discussed operators from the [sheet](https://docs.google.com/spreadsheets/d/1JIykwXJEPT8FTLzvb0_5p_odvTYS9zzZjPNN33ziEbQ/edit?gid=0#gid=0) and assigned a priority to the operators. The operators that appear when models are exported are assigned P1 and operators from other pipeline elements are assigned P2. 9 | - Yuan noted that while the existing attention operator in ONNX can cover most cases, it lacks support for quantization, mixed precision, and in-place key-value (KV) caching. He shared a [proposal](https://docs.google.com/document/d/1H-tYkH0DlxEMFafk7dKolNMcEf6Lpk1HOeL33u0bMlY) with the team for feedback. 10 | - The group also discussed flex attention operator. Sahar highlighted the need to support user-defined functions (e.g., for score_mod and mask_mod), which the current ONNX spec doesn't directly support. 11 | - Yuan and Rama proposed using graph-valued attributes to embed such custom computations as they can encode flexible logic like score_mod as subgraphs. This makes the op more general and extensible but requires more backend complexity and thoughtful model export design. 12 | - Need to check if both mask_mod and score_mod are needed. Mask_mod offers performance benefits by skipping certain computations. However, it can be implemented within score_mod, so having both may be redundant functionally but useful for optimization. 13 | - Yamini confirmed that ONNX supports nested functions, which can also be potentially used for modular computation inside attention ops. The group to further explore torch.export for flex attention and analyze the ops exported in torch fx graph. 14 | 15 | ## Backend representation: 16 | - The group discussed backend-specific representation following the EPContext design. Rama suggested to have an interpretable graph instead of a black box blob. 17 | - Yamini did a simple POC and showed an ONNX model with a ‘BackendContext’ function that points to compiled blob in the node attribute and stores the original subgraph. This is a more interpretable backend representation using a subgraph function and can include detailed backend-specific metadata 18 | - However, this becomes complicated if we need to store multiple backend representations in the same graph and different backends support different subgraphs. 19 | - The group discussed handling different quantization types and backend-specific scopes without storing redundant weights, suggesting solutions like scoping within a shared graph. 20 | - Yamini to start a draft for the backend context framework and share with others to collaborate on the proposal. 21 | 22 | ## Exporters: 23 | - Kshitij is working on an ONNX Script fix to ensure the attention module exports as a single attention op instead of a decomposed subgraph. 24 | - Shubham confirmed it can be done by adding a rewrite rule for scaled_dot_product_attention and will coordinate with Kshitij. 25 | 26 | ## Others: 27 | - Max suggested a better handling of KV Cache by enabling partial writes to I/O tensors using indexing or interleaved formats, to preserve ONNX's functional semantics. 28 | - Rama suggested adding top-level graph annotations to mark input/output aliasing, allowing backends to optimize without altering internal graph structures. Aliasing should be made explicit, not inferred by backends, to ensure correctness—especially with more complex attention patterns. 29 | - Max to document the ideas and share them with the working group for further deliberation and feedback. 30 | - Yamini suggested forming another breakout workgroup in the future to discuss GenAI pipelines and ideas like above that may not be captured by existing breakout groups. 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /generative-ai/meetings/meeting_kickoff_date_April_17.md: -------------------------------------------------------------------------------- 1 | # Recording and Transcript: 2 | 3 | https://zoom.us/rec/share/OqLomSSqJNn-tDvc3Y_ZD5D2Pr6nTWF4HNEfJZ9dDH-EmiQ_xgUnBhXSrC4TJeRS.RQ29nDpFMTpQuif5 4 | 5 | # Meeting Minutes: 6 | 7 | - Link to the kick-off meeting slides: https://docs.google.com/presentation/d/1PYAHauEVhhdTuKMYiOsOjOVz6u8OzWka 8 | - Link to the sheet for adding ops/functions: https://docs.google.com/spreadsheets/d/1JIykwXJEPT8FTLzvb0_5p_odvTYS9zzZjPNN33ziEbQ 9 | - Rama suggested adding extensions to the scope of the working group, including backend-specific ONNX representation. Yamini requested if Rama could provide examples for the extensions 10 | - Mayank suggested a Europe-friendly time for the meetings going forward. Yamini to set up recurring weekly meetings at 8 AM PST – WG to give feedback if anyone has a hard conflict 11 | - Rama suggested sending agenda ahead of time to include folks who can provide better insights on the topics 12 | - Yamini shared the sheet with ops/functions (link above) and requested all to provide inputs 13 | - Krishna asked for guidance on ONNX operators vs. functions. Rama advised proposing as functions, if the logic can be decomposed into existing ops. Anything that cannot be represented with existing ops can be proposed as an operator 14 | - Georgy to share op specification of ScaledDotProductAttention and clarify whether it should be proposed as an operator or function 15 | # Action Items: 16 | 17 | - @G. Ramalingam 18 | to provide examples of backend-specific ONNX representation extensions 19 | - @Yamini nimmagadda 20 | to set up recurring weekly meetings and circulate meeting agendas in advance of each meeting. @All to let Yamini know if anyone is interested to co-chair the working group 21 | - @All to review and contribute to the ops/functions input sheet: https://docs.google.com/spreadsheets/d/1JIykwXJEPT8FTLzvb0_5p_odvTYS9zzZjPNN33ziEbQ. Please provide the specification, justification, and examples 22 | - @Georgy Krivoruchko 23 | to share the op specification for ScaledDotProductAttention and clarify whether it should be proposed as an operator or a function 24 | -------------------------------------------------------------------------------- /multi-device/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Multi-device Working Group (WG) 4 | 5 | This repository is where the ONNX Multi-device WG will capture various artifacts and deliverables. 6 | 7 | ## Objective 8 | 9 | The AI industry has grown model size at a faster rate than the growth in hardware capabilities. To enable running SoTA models with ONNX, the ONNX spec needs support for expressing execution of the network on multiple devices. We propose a new Multi-device Working Group, whose exit criteria would be a ratified extension to the ONNX spec for expressing multi-device capability in an ONNX model. 10 | To seed discussion in the proposed WG, we have an initial proposal that introduces a few extensions to ONNX that allow expression of multi-device capability with minor changes relative to a single device network. By expressing some optional information to an ONNX network, the model can be scaled to multiple devices. 11 | 12 | ## Working Group Status 13 | **ACTIVE** 14 | 15 | # Slack channel 16 | Please sign up at https://slack.lfai.foundation/ and join the [onnx-multi-device](https://lfaifoundation.slack.com/archives/C05JY32GCCS) channel 17 | 18 | # WG Lead(s) 19 | 20 | * Kevin Chen (NVIDIA) 21 | * Ganesan Ramalingam (Microsoft) 22 | 23 | # Logistics 24 | 25 | * WG leads will drive the meeting. 26 | * Meeting annoucements will be posted in our Slack channel: https://lfaifoundation.slack.com/archives/C05JY32GCCS 27 | * Feedbacks and topic requests are welcome from everyone. 28 | * Documents and artifacts: https://github.com/onnx/working-groups/tree/main/multi-device 29 | 30 | # WG Meeting Info 31 | 32 | * Planned meetings can be found at the [group homepage](https://lists.lfaidata.foundation/g/onnx-wg-multidevice) 33 | * Recordings will be uploaded to the [group files page](https://lists.lfaidata.foundation/g/onnx-wg-multidevice/files/Meeting%20Recordings) 34 | 35 | # Meeting notes 36 | 37 | The meeting notes can be found [here](https://github.com/onnx/working-groups/tree/main/multi-device/meetings) 38 | -------------------------------------------------------------------------------- /multi-device/meetings/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains meeting notes for the ONNX Multi-device Working Group 4 | -------------------------------------------------------------------------------- /multi-device/meetings/notes-09-13-2023.md: -------------------------------------------------------------------------------- 1 | # 09-13-2023 2 | 3 | ## Attendees 4 | 5 | - Kevin (NVIDIA) 6 | - Rama (Microsoft) 7 | - Wei-Sheng (Microsoft) 8 | - Yuan Yao (NVIDIA) 9 | - Devang (Microsoft) 10 | - Alex (IBM) 11 | 12 | ## Recording 13 | - [Recording Link](https://lists.lfaidata.foundation/g/onnx-wg-multidevice/files/Meeting%20Recordings) 14 | 15 | ## Agenda 16 | 17 | - Recurring meetings monthly on each second Wednesday at 5PM PST. Full calendar details found at https://lists.lfaidata.foundation/g/onnx-wg-multidevice 18 | 19 | - Discussion on initial proposal - [Multi-device proposal](https://docs.google.com/document/d/1tikVgXz4huFbkiXDGaRrI0oIV_rgcH-hxsM06TS7J4o/edit?usp=sharing) 20 | 21 | ## Meeting Notes 22 | 23 | - @Kevin discussed in detail the proposal that was started by Micah in the 2023 ONNX community meeting 24 | - Feedback from the community 25 | - Proposal added too much complexity, a few fields (such as topology) can be omitted 26 | - Preference for defining communication operations explicitly rather than through generic IJoin / ISplit operations 27 | 28 | ## Action Items 29 | 30 | - @Everyone to review the proposal, add comments 31 | - @Everyone to think of and discuss potential alternatives for the next meeting. -------------------------------------------------------------------------------- /multi-device/meetings/notes-10-11-2023.md: -------------------------------------------------------------------------------- 1 | # 10-11-2023 2 | 3 | ## Attendees 4 | 5 | - Kevin (NVIDIA) 6 | - Rama (Microsoft) 7 | - Yuan (NVIDIA) 8 | - Tung (IBM) 9 | - Alex (IBM) 10 | - Yasushi (IBM) 11 | 12 | ## Recording 13 | - [Recording Link](https://lists.lfaidata.foundation/g/onnx-wg-multidevice/files/Meeting%20Recordings) 14 | 15 | ## Agenda 16 | 17 | - Discussion on alternative proposal from Rama - [Multi-device proposal](https://docs.google.com/presentation/d/1Sklg10gujE9bbjQloapdFv8-Dht6WYN1XAQcVZ_UXDg/edit#slide=id.g28c031b729a_6_75) 18 | 19 | ## Meeting Notes 20 | 21 | - Discussions on one model vs multi-model representation of multi-device ONNX 22 | - Disccusion on whether the outcome of this WG - the current goal is to amend the spec so that an optimizer can produce a well-defined model that is meant to be run across multiple devices with some user hints 23 | - Emphasis on providing concrete examples of usage. We need to define the common operations that are used 24 | 25 | ## Action Items 26 | 27 | - @Everyone to review the proposals, think of answers to these high level questions 28 | -------------------------------------------------------------------------------- /multi-device/meetings/notes-11-08-2023.md: -------------------------------------------------------------------------------- 1 | # 11-08-2023 2 | 3 | ## Attendees 4 | 5 | - Kevin (NVIDIA) 6 | - Rama (Microsoft) 7 | - Wei-Sheng (Microsoft) 8 | - Yasushi (IBM) 9 | 10 | ## Recording 11 | - [Recording Link](https://lists.lfaidata.foundation/g/onnx-wg-multidevice/files/Meeting%20Recordings) 12 | 13 | ## Agenda 14 | 15 | - Shortening meeting cadence (2 weeks instead of one month) 16 | - Defining requirements 17 | - Discussion on single model vs multi-model representation 18 | - Any other discussions 19 | 20 | 21 | ## Meeting Notes 22 | 23 | ### Requirements 24 | What should we be able to represent? 25 | * Model parallelism 26 | * Tensor parallelism 27 | * Data parallelism (not too relevant in inference usecase) 28 | * Multiple devices across multiple nodes 29 | * Ability to define subgroups with devices 30 | 31 | What operators do we need to accomplish the above? 32 | * Model parallelism 33 | - TensorSend 34 | - TensorRecv 35 | * Tensor parallelism 36 | - TensorScatter 37 | - TensorAllGather 38 | - TensorAllReduce (specialized on reduce operation) 39 | - TensorAllScatterReduce (composed as a function) 40 | * How do initializers / weights of other operators get tiled? 41 | * Sharding vs Reshuffling spec of tiling 42 | * How expressive of a sharding pattern do we want to support? 43 | - LLAMA2 implementation only requires sharding across one axis 44 | https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/contrib_ops/cuda/collective/sharding_spec.h 45 | - General sharding spec can make current operations unrepresentable with current ONNX primitives 46 | - General sharding spec provides undue pressure on backends to support potentially uncommon sharding 47 | * Distributed versions of primitives from ONNXRT: https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/test/python/onnxruntime_test_distributed.py 48 | 49 | Who produces these models? 50 | * Nothing stopping upstream framework to insert collective ops at certain points 51 | * Most likely third-party optimization tools will insert this based on target hardware domain 52 | 53 | ### Single model vs Multi-model representation 54 | 55 | * Single model - every device executes subset of nodes 56 | * Single model - every device executes the same model depending on device ID 57 | * Multi-model - every device executes its own model 58 | 59 | Targeting # 2 and # 3: 60 | 61 | Base ONNX representation: 62 | 63 | `Input -> Add -> Output` 64 | 65 | Single model representation: 66 | 67 | `Input(2x2) -> TensorScatter(axis 0 across two devices) -> Add (2x2) -> AllGather -> Output` 68 | 69 | Multi model representation: 70 | 71 | Device 0: 72 | `input(2x2) -> TensorScatter(axis 0 across two devices) -> Add (1x2) -> AllGather -> Output` 73 | 74 | Device 1: 75 | 76 | `TensorScatter(some way to represent a recv from device 0) -> Add(1x2) -> AllGather(some way to represent sending back to device0)` 77 | 78 | ### Questions 79 | * Do we assume some pre-determined machine configuration when generating the ONNX? 80 | - In ORT LLAMA implementation, we cannot change number of devices for memory concerns. 81 | - The optimized implementation needs to know the number of devices 82 | - Use case - generate a model with N number of devices at build time or runtime? 83 | * Single model vs multi-model 84 | - If axis 0 is dynamic - some shape analysis is required 85 | - If the subgraph between TensorScatter & AllGather is large, how do we represent that? 86 | - Depends on the well-formed definition of collective operations 87 | 88 | 89 | ## Action Items 90 | 91 | - Review ORT example, come up with definitions for collective operations, more concrete examples. 92 | -------------------------------------------------------------------------------- /multi-device/meetings/notes-12-13-2023.md: -------------------------------------------------------------------------------- 1 | # 12-13-2023 2 | 3 | ## Attendees 4 | 5 | - Kevin (NVIDIA) 6 | - Rama (Microsoft) 7 | - Wei-Sheng (Microsoft) 8 | - Yuan (NVIDIA) 9 | 10 | ## Recording 11 | - [Recording Link](https://lists.lfaidata.foundation/g/onnx-wg-multidevice/files/Meeting%20Recordings) 12 | 13 | ## Meeting Notes 14 | 15 | * https://docs.google.com/document/d/1nhNVkg5uz60iJ7xrW1op-xEhAGOzTW9W-Y4OWlbmtKs/edit?usp=sharing for shared proposal. 16 | * Pipeline parallelsim will be deprioritized in initial implementation. We will focus on tensor parallelism 17 | * Need to think and work on the design for before and after sharding representations, and get the list of operations that can be run distributively. 18 | -------------------------------------------------------------------------------- /preprocessing/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Preprocessing Working Group (WG) 4 | 5 | This repository is where ONNX Preprocessing WG will capture various artifacts and deliverables. The purpose of this WG is to standarize data pre/post processing and feature extraction operations for Deep Neural Networks. 6 | 7 | ## Working Group Status 8 | **ACTIVE** 9 | 10 | # Slack channel 11 | Please sign up at https://slack.lfai.foundation/ and join [onnx-preprocessing](https://lfaifoundation.slack.com/archives/C02AANGFBJB) channel. 12 | 13 | # WG Lead(s) 14 | 15 | * Joaquin Anton (NVIDIA) (Oct 13, 2021 - Current) 16 | 17 | # Logistics 18 | 19 | * WG leads will drive the meeting. 20 | * Meeting annoucement will be posted in our Slack channel: https://lfaifoundation.slack.com/archives/C02AANGFBJB 21 | * Feedbacks and topic request are welcome by all. 22 | * Documents and artifacts: https://github.com/onnx/working-groups/tree/main/preprocessing 23 | 24 | # WG Meeting Info 25 | 26 | * Meeting occurs every second Wednesday of the month at 8 AM PST. 27 | * Zoom Meeting link: https://zoom.us/j/92887463161?pwd=M2JvbWhaVERTV0NNN0NzZmw2Q3Ztdz09 28 | * Meeting ID: 928 8746 3161 29 | 30 | # Meeting notes 31 | 32 | The meeting notes can be found [here](https://github.com/onnx/working-groups/tree/main/preprocessing/meetings) 33 | -------------------------------------------------------------------------------- /preprocessing/meetings/20210811.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Aug 11, 2021 at 3:00pm UTC 4 | 5 | ## Agenda 6 | * Introduction 7 | * Member introductions 8 | * Meeting date/time and frequency 9 | * Preferred method of communication 10 | * Define problem statement 11 | * Motivation 12 | * Gathering suggestions 13 | * Arrive at a consensus 14 | * Define action items for the next meeting 15 | 16 | ## Attendees 17 | * Joaquin Anton (NVIDIA) 18 | * Janusz Lisiecki (NVIDIA) 19 | * Michal Szolucha (NVIDIA) 20 | * Mayank Kaushik (NVIDIA) 21 | * Ashwini Khade (Microsoft) 22 | * Rodolfo G Esteves (Intel) 23 | * Ganesan Ramalingan (Microsoft) 24 | * Jacky Chen (Microsoft) 25 | * Matteo Interlandi (Microsoft) 26 | * Prasanth Pulavarthi (Microsoft) 27 | * Ohara Moriyoshi (IBM) 28 | 29 | ## Recording & Slides 30 | 31 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/ONNX%20pre_post%20processing%20and%20featurization%20WG%20kick-off%20meting-20210811_180626-Meeting%20Recording.mp4) 32 | * [Slides](slides/20210811_slides.pdf) 33 | 34 | ## Meeting Notes 35 | 36 | * We agreed to meet monthly, Wednesday (or Tuesday-Thursday) at 5 PM CEST, to accommodate different time-zones, as we have people joining from America, Europe, and Asia. 37 | * We agreed to use Slack as a main communication channel. 38 | * There is a lack of standardization in data preprocessing. Different libraries are used, leading to portability issues. 39 | * The initial goal of the group is to make data preprocessing part of ONNX and come up with a standardized set of operators that will support training and inference for a set of selected networks. The plan is to first standardize preprocessing, then incorporate it into the model. 40 | * We see that there are existing ONNX operators that could be used to define a data preprocessing pipeline. The challenge appears with the different nature of the preprocessing pipeline. A preprocessing pipeline is typically defined on a per-sample basis, and it’s part of the data loader to assemble the preprocessed samples to a uniform batch that is then fed to the model. We don’t yet have a good idea on how the data preprocessing part can be integrated with the ONNX graph, due to its different nature. 41 | * ONNX has no notion of batch in tensors, but the batch can be one of the dimensions. ONNX has a sequence of tensors that could be used to represent a non-uniform batch in the preprocessing pipeline. 42 | * Data preprocessing differs from training to inference. How to incorporate that into the graph? 43 | * Exporting to ONNX graph: Should we only support exporting TF/PyTorch/etc data pre-preprocessing to ONNX or should we also cover usage of other FWs/libraries (e.g. Pillow)? 44 | * How will the runtime implement preprocessing? Implementations are meant to work on big tensors. We might need to define data preprocessing as functions that the runtime could optimize. 45 | 46 | ## Action items: 47 | * **Everybody** A member of each organization to come up with a sorted list of most important networks that we would like to support initially 48 | * **Everybody** Think about how the data processing pipeline could be represented and connected to the model 49 | -------------------------------------------------------------------------------- /preprocessing/meetings/20211110.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Nov 10, 2021 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Merging models - Compose Utils 7 | * Control flow operator for batch processing 8 | * Date/String operators for Kaggle preprocessing coverage 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Takuya Nakaike (IBM) 13 | * Rodolfo Gabriel Esteves (Intel) 14 | * Janusz Lisiecki (NVIDIA) 15 | 16 | ## Recording & Slides 17 | 18 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20211110.mp4) 19 | * [Meeting Slides](slides/20211110_slides.pdf) 20 | 21 | ## Meeting notes: 22 | 23 | Coming back to the discussion about merging ONNX graphs, Joaquin proposed a set of "Compose" utils that can be used to create combined models by merging graphs. 24 | * The tools are not necessarily limited to preprocessing, and could also be used for other purposes such as postprocessing or simply chaining models 25 | * The group agrees with the proposal and it will be sent to approval to the archinfra group. 26 | 27 | We briefly discussed the idea of having a batch processing operator that can execute a subgraph on every sample in a sequence, concatenating the results into a single tensor at the end. 28 | * Contributors are welcome. 29 | * No volunteers so far. Joaquin will investigate the idea further. 30 | 31 | Takuya presented a proposal for a Date parsing operator and String processing operators, that will allow to convert data preprocessing patterns on pandas dataframe to ONNX 32 | * StringConcatenator and StringSplitter are already prototyped as extensions in onnxruntime. 33 | * Date operator is to be considered. 34 | * We discussed that the operators SIG would be a more appropriate forum for the discussion and review of such proposals 35 | * This might concern the preprocessing group later on, due to the nature of the operator (e.g. non-differentiable) 36 | -------------------------------------------------------------------------------- /preprocessing/meetings/20211208.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Dec 8, 2021 at 8:00am PDT 4 | 5 | ## Agenda 6 | * ONNX Compose utils - Status update 7 | * Batch processing with a Loop 8 | 9 | ## Atendees 10 | * Joaquin Anton (NVIDIA) 11 | * Rodolfo Gabriel Esteves (Intel) 12 | * Ganesan Ramalingan (Microsoft) 13 | 14 | ## Recording & Slides 15 | 16 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20211208.mp4) 17 | * [Meeting Slides](slides/20211208_slides.pdf) 18 | 19 | ## Meeting notes: 20 | 21 | * ONNX compose utils has been merged. It allow users to combine two models by connecting inputs/outputs, optionally renaming entries in each graph. 22 | * Joaquin showed some early results for batch preprocessing with a loop. The proposal is to create a "preprocessing" function that applies a subgraph to each sample in the graph, concatenating the results to a single tensor at the end. 23 | * Rama suggests that we divide it into two steps. The first one "SequenceMap" or similar, applies a graph to each sample in a sequence, producing a sequence as an output. The second step, "ConcatFromSequence" already exists. The group agreed with this proposal. 24 | * We discussed how to keep the preprocessing part of the graph identifiable within a combined model. 25 | * We agreed to group the preprocessing part of the graph as a local function in the model, and use model level properties to identify a preprocessing function when applicable. 26 | * We discussed a possibility of using a preprocessing function that would allow to "select" from different implementations at runtime (e.g. no preprocessing, inference pipeline, training pipeline). We agree that this is doable with the current proposal, but we do not need to standardize such usage at this moment. 27 | 28 | ## Action items: 29 | * Joaquin will prepare a draft for the "SequenceMap" function and create an example that uses it, together with a local function tagged as the preprocessing function with model level properties. -------------------------------------------------------------------------------- /preprocessing/meetings/20220112.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jan 12, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * SequenceMap function - status update 7 | * Using model local functions 8 | 9 | ## Atendees 10 | * Joaquin Anton (NVIDIA) 11 | * Natalie Kershaw (Microsoft) 12 | * Andreas Fehlner (TRUMPF Laser GmbH) 13 | * Janusz Lisiecki (NVIDIA) 14 | * Michal Zientkiewicz (NVIDIA) 15 | 16 | ## Recording & Slides 17 | 18 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20220112.mp4) 19 | * [Meeting Slides](slides/20220112_slides.pdf) 20 | 21 | ## Meeting notes: 22 | 23 | * SequenceMap function PR is under review. Seems to work as expected. 24 | * Initial experiments show that using model metadata properties to tag a function as the preprocessing part of the graph will be enough for any backends to detect special parts of the graph (preprocessing, postprocessing). 25 | * Several issues were found regarding using model local functions (ONNX and ONNX runtime): 26 | - https://github.com/microsoft/onnxruntime/issues/10249 27 | - https://github.com/microsoft/onnxruntime/issues/10250 28 | - https://github.com/onnx/onnx/issues/3938 29 | * We will continue with these two lines of work and once those are completed we will be able to create a first prototype (RN50) and publish it 30 | 31 | ## Action items: 32 | -------------------------------------------------------------------------------- /preprocessing/meetings/20220309.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Mar 09, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * SequenceMap function - status update 7 | * Using model local functions 8 | * Sequence support in ONNX parser 9 | * Resize and keep aspect ratio semantics 10 | * (Wenbing Li) Intro about pre-processing in Microsoft 11 | 12 | ## Atendees 13 | * Joaquin Anton (NVIDIA) 14 | * Janusz Lisiecki (NVIDIA) 15 | * Natalie Kershaw (Microsoft) 16 | * Ganesan Ramalingan (Microsoft) 17 | * Prasanth Pulavarthi (Microsoft) 18 | * Wenbing Li (Microsoft) 19 | * Rodolfo G Esteves (Intel) 20 | 21 | ## Recording & Slides 22 | 23 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20220309.mp4) 24 | * [Meeting Slides](slides/20220309_slides.pdf) 25 | 26 | ## Meeting notes: 27 | 28 | * SequenceMap function PR - status update 29 | - PR has one approval from Rama. It'd be good to have a second approval from Operators SIG. 30 | - It'd be good to have at least one usage example. Joaquin will prepare a model using SequenceMap to do batch processing and publish it to ONNX examples, as a reference. 31 | * Model local functions. Several issues found in onnxruntime when working with model local functions. Rama will follow those up 32 | - https://github.com/microsoft/onnxruntime/issues/10250 33 | - https://github.com/microsoft/onnxruntime/issues/10249 34 | - https://github.com/microsoft/onnxruntime/issues/10698 35 | - We discussed the fact that functions don't have type/shape information in their signature, and they work more like a macro. We don't see a need to change that unless it poses a problem. 36 | * Sequence support in ONNX parser 37 | - It'd be good to have support for sequences. We will continue this discussion later. 38 | * Resize and keep aspect ratio semantics 39 | - To be discussed during the next meeting (lack of time) 40 | * Wenbing did a presentation about the experience with pre-processing with ONNX in Microsoft 41 | - Converting arbitrary python code to an ONNX model representation automatically is going to be hard 42 | - Very different domains NLP / Vision / String / Audio / Math. We need to figure out a good strategy for categorizing the different domains in ONNX 43 | - Discussion to be continued 44 | 45 | ## Action items: 46 | -------------------------------------------------------------------------------- /preprocessing/meetings/20220413.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Apr 13, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status - Open issues/PRs 7 | * Resize - Antialiasing 8 | * Domain specific functions for higher level abstractions 9 | * Open floor 10 | 11 | ## Atendees 12 | * Joaquin Anton (NVIDIA) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Liqun Fu (Microsoft) 15 | * Janusz Lisiecki (NVIDIA) 16 | * Rodolfo G Esteves (Intel) 17 | * Michal Zientkiewicz (NVIDIA) 18 | 19 | ## Recording & Slides 20 | 21 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20220413.mp4) 22 | * [Meeting Slides](slides/20220413_slides.pdf) 23 | 24 | ## Meeting notes: 25 | 26 | * Status - Open issues/PRs 27 | - [SequenceMap function PR](https://github.com/onnx/onnx/pull/3892) has been merged. There is an [outstanding issue in ONNX Runtime](https://github.com/microsoft/onnxruntime/issues/10876). 28 | - There is an [effort to rework the way functions are handled in ONNX runtime](https://github.com/microsoft/onnxruntime/pull/11167) 29 | - Two PRs started with tutorials about [using SequenceMap](https://github.com/onnx/tutorials/pull/265) and about [end-to-end integration with ResNet-50](https://github.com/microsoft/onnxruntime/pull/11167). 30 | - Joaquin will bring up the question of missing sequence initializers with the infra group. 31 | - Joaquin will try to update the ResNet-50 model to reflect a variable batch dimension (right now it is hardcoded to 1). 32 | * Resize - Antialiasing 33 | - Joaquin proposed to extend `Resize` with an `antialias` flag, to apply an antialiasing filter when downscaling, as other libraries (PIL) use. The default (and current) is not to apply antialiasing, closer to other libraries (OpenCV). 34 | - Rama will initiate the review of [this PR](https://github.com/onnx/onnx/pull/4126). 35 | * Domain specific functions for higher level abstraction 36 | - Joaquin presented the idea of having domain specific *functions* that are implemented as much as possible in terms of existing ONNX operators 37 | - The domain specific functions serve two purposes: Simplicity for common operations (e.g. CenteredCrop) and possibility for optimizations, as the backends might choose to offer a specialized implementation for those. 38 | - We discussed the possibility to deliver such domain specific functions as a separate library. This has its value for custom operations, however, we think that for general purpose operations like image processing, it is benefitial to make them part of the standard. 39 | - We agreed that when proposing domain specific functions, we will try to generalize as much as possible, to avoid necessary overlap in functionality (two functions applying two very similar operations, but expressed a bit differently). 40 | * Open floor 41 | - SequenceMap has opset version 16, but was merged after the release. It should be adjusted to version 17. -------------------------------------------------------------------------------- /preprocessing/meetings/20220511.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed May 11, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status - Open issues/PRs 7 | * Resize-17 - Antialias, axes and keep_aspect_ratio_policy 8 | * ai.onnx.image.CenterCropPad 9 | * Open floor 10 | 11 | ## Atendees 12 | * Joaquin Anton (NVIDIA) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Liqun Fu (Microsoft) 15 | * Rodolfo G Esteves (Intel) 16 | 17 | ## Recording & Slides 18 | 19 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20220511.mp4) 20 | * [Meeting Slides](slides/20220511_slides.pdf) 21 | 22 | ## Meeting notes: 23 | 24 | * Status - Open issues/PRs 25 | - Next ONNX runtime release should support SequenceMap 26 | - There is an [outstanding issue in ONNX Runtime](https://github.com/microsoft/onnxruntime/issues/10876). This makes SequenceMap unusable. There is a proposed fix in the issue report. 27 | - We agreed that sequence initializers are not necessary (no particular usecase for it), and we can simply remove the warning about not being able to constant-fold SequenceEmpty, to not to confuse users. 28 | - [SequenceMap tutorial](https://github.com/onnx/tutorials/pull/265) will wait for the ONNX runtime release. 29 | - [End-to-end preprocessing integration with ResNet-50](https://github.com/microsoft/onnxruntime/pull/11167) is work-in-progress as we will probably update it with the new op extensions as we merge them. 30 | * Resize-17 - Antialias, axes and keep_aspect_ratio_policy 31 | - The Resize extension proposal was extended to have `axes` and `keep_aspect_ratio_policy` attributes. 32 | - `axes` allows users to select a subset of dimensions to be resized, while keeping others intact. For example, when resizing an image with `[H, W, C]` layout, the user can specify `axes=(0, 1)`, and provide only the desired height and width (no need to specify scale or size for the channel dimension). The same can be done with an image with `[C, H, W]` layout, by specifying `axes=(1, 2)`. 33 | - `keep_aspect_ratio_policy` allows resizing to some maximum or minimum dimensions, while keeping the original aspect ratio of the input. The default behavior is the current one, which is to stretch the input, disregarding the original aspect ratio. 34 | - The group agreed to initiate the review of [this PR](https://github.com/onnx/onnx/pull/4126). 35 | * ai.onnx.image.CenterCropPad 36 | - First operator to introduce `ai.onnx.image` domain 37 | - [PR ready for review](https://github.com/onnx/onnx/pull/4190) 38 | - We discussed the implications of a new domain (does it mean it is part of the standard or not?) 39 | - After some consideration, the group decided that it is better to stay in the default domain for now, until we have a bigger picture on how we want to organize existing ops into domains. 40 | - Joaquin will adjust the PR to use the default domain. 41 | -------------------------------------------------------------------------------- /preprocessing/meetings/20220608.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed June 8, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status - Open issues/PRs 7 | * ONNX Community Day 6/24/22 8 | * Naming choice for models that include preprocessing 9 | * Open floor 10 | 11 | ## Atendees 12 | * Joaquin Anton (NVIDIA) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Janusz Lisiecki (NVIDIA) 15 | 16 | ## Recording & Slides 17 | 18 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20220608.mp4) 19 | * [Meeting Slides](slides/20220608_slides.pdf) 20 | 21 | ## Meeting notes: 22 | 23 | * Status - Open issues/PRs 24 | - There are some standing issues with SequenceMap in ONNX runtime: [Issue 1](https://github.com/microsoft/onnxruntime/issues/10876), [Issue 2](https://github.com/microsoft/onnxruntime/issues/10698) 25 | - Rama prepared a PR that reworks function expansion implementation in ONNX runtime and it should address those problems: [Restructure function inliner](https://github.com/microsoft/onnxruntime/pull/11731). Joaquin will double check if that solves the issues above. 26 | - The review of Resize-17 operator has been going on for a while with not much progress (2 months). It can be attributed to focus on the new ONNX release. There is an effort to find more volunteers to review PRs in the ONNX operators SIG. Joaquin will send a reminder to the Operators team slack channel. 27 | - There is an issue with CenterCropPad. The current implementation is trying to access the rank of the input in order to populates the default axes. This is problematic because we can only learn about statically shaped inputs. Joaquin will try to write an alternative path that won't use axes, when those are not provided. If there are obstacles with this approach, we agree to make `axes` a mandatory attribute. 28 | - Perhaps we don't need to extend Shape with `axes`, as we can use `Gather` operator easily as a second step. Joaquin will try this out. 29 | * ONNX Community Day 6/24/22 30 | - 2-4 minute update from SIG/WG leaders 31 | - Joaquin can't attend, will pre-record a short presentation 32 | * Naming choice for models that include preprocessing 33 | - Topic raised in #onnx-preprocessing Slack channel 34 | - Do we want to create a term to refer to models that include data preprocessing and if so, what should the term be. 35 | - We see no need in having a formal name for it, since it is not clear what constitutes preprocessing, and an ONNX model will not be fundamentally different for including some "preprocessing" steps. 36 | - We can always revisit this idea if necessary later on. 37 | -------------------------------------------------------------------------------- /preprocessing/meetings/20220713.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed July 13, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status - Open issues/PRs 7 | * Open floor 8 | 9 | ## Atendees 10 | * Joaquin Anton (NVIDIA) 11 | * Ganesan Ramalingan (Microsoft) 12 | * Natalie Kershaw (Microsoft) 13 | * Janusz Lisiecki (NVIDIA) 14 | * Muthu Baskaran (Qualcomm) 15 | 16 | ## Recording & Slides 17 | 18 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20220713.mp4) 19 | * [Meeting Slides](slides/20220713_slides.pdf) 20 | 21 | ## Meeting notes: 22 | 23 | * Status - Open issues/PRs 24 | - Resize-18 and CenterCropPad-18 PR were recently merged. It will take ~3 months until they are part of an ONNX/ONNXRuntime release. 25 | - Still [an open issue](https://github.com/microsoft/onnxruntime/issues/10698) related to usage of local functions and graph attributes. Rama will look into it. 26 | - ResNet models have now arbitrary batch size. There was an issue found with some versions that also include value_infos. More details in [the comments of the PR](https://github.com/onnx/models/pull/537). [A new PR](https://github.com/onnx/models/pull/542) will also fix value_infos. 27 | - Documentation doesn't say which is the minimum protobuf version required. There was an issue with protobuf 3.0, which was solved [by this PR](https://github.com/onnx/onnx/pull/4354). Joaquin will check with Jacky about the required minimum version. 28 | -------------------------------------------------------------------------------- /preprocessing/meetings/20221019.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Oct 19, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Progress status - Open issues/PRs 7 | * Next steps 8 | * Open floor 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Natalie Kershaw (Microsoft) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Janusz Lisiecki (NVIDIA) 15 | * Xavier Dupre (Microsoft) 16 | 17 | ## Recording & Slides 18 | 19 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20221019.mp4) 20 | * [Meeting Slides](slides/20221019_slides.pdf) 21 | 22 | ## Meeting notes: 23 | 24 | ### Status - Open issues/PRs 25 | Few PRs/issues closed since the last time 26 | - [Add Batch processing with SequenceMap tutorial](https://github.com/onnx/tutorials/pull/265) 27 | - [Use RepeatedPtrField::Get ... to be compatible with protobuf 3.0 API](https://github.com/onnx/onnx/pull/4354) 28 | - [Local functions with subgraphs: GraphProto attribute inferencing error](https://github.com/microsoft/onnxruntime/issues/10698) 29 | 30 | ### Future steps 31 | - Next steps are publishing the ResNet end-to-end models (including preprocessing) 32 | - We discussed the possibility of either replacing existing models with their end-to-end counterparts or adding the new model separately, keeping the existing unmodified. The group agreed on publishing the end-to-end model separately without modifying the existing one. 33 | - We discussed the possibility of publishing a separate preprocessing-only model and letting the user combine it into a composite model with compose utils. The group agreed that it's simpler for the users to just pull the end-to-end model directly. 34 | 35 | ### Open floor 36 | - We briefly discussed the possibility of looking into converters to include preprocessing. The different nature of the data preprocessing pipeline (including a lot of custom code) would make it difficult to have a generic converted. However, we could look into supporting some use cases. However, this particular topic falls a little bit outside of the scope of this group, and more to the scope of particular converter implementers. We will continue the discussin on the next meeting. 37 | - Some examples of converters are: 38 | * https://github.com/onnx/tensorflow-onnx 39 | * https://pytorch.org/docs/stable/onnx.html 40 | * https://github.com/microsoft/onnxruntime-extensions 41 | 42 | ## Action items 43 | - Natalie to discuss with teams working on converters. 44 | -------------------------------------------------------------------------------- /preprocessing/meetings/20221109.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Nov 9, 2022 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Exporting preprocessing to ONNX 7 | * ONNX <-> DALI 8 | * Open floor 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Natalie Kershaw (Microsoft) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Xavier Dupre (Microsoft) 15 | * Janusz Lisiecki (NVIDIA) 16 | * Gabriel Esteves (Intel) 17 | 18 | ## Recording & Slides 19 | 20 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20221109.mp4) 21 | * [Meeting Slides](slides/20221109_slides.pdf) 22 | 23 | ## Meeting notes: 24 | 25 | ### Exporting preprocessing to ONNX 26 | 27 | Reiterated on the problem of converting preprocessing from different FWs to ONNX 28 | 29 | Two main challenges: 30 | - Custom code 31 | - high variety of external libraries processing the raw signal, with no common language 32 | 33 | Natalie introduced the work done in [onnxruntime-extensions] to pre-generate pre and post processing to most common models: 34 | https://github.com/microsoft/onnxruntime-extensions/pull/312/files#diff-8fd48c1c9e5277ce39cdaac456d4cd3a3854b0b0aebf318769ed2afc3d010e0a 35 | 36 | At the moment the only solution is writing the preprocessing steps directly in ONNX format. 37 | 38 | ### ONNX <-> DALI 39 | 40 | Joaquin introduced [NVIDIA DALI](https://github.com/NVIDIA/DALI), a library for data loading and preprocessing. 41 | 42 | DALI uses a similar graph representation that would allow converting back and forth to ONNX. 43 | 44 | DALI can be integrated with most common Deep Learning frameworks (PyTorch, Tensorflow, MXNet, Paddle) and DeepLearningExamples showcase DALI implementations for the data pipelines. This could be also used as a source for generating preprocessing ONNX models. 45 | 46 | We discussed the possibility to start onnx-dali and dali-onnx repositories in ONNX. We agreed the best is to come up with a prototype first and then bring it up with the Steering Committee. 47 | 48 | ### Open floor 49 | 50 | ## Action items 51 | -------------------------------------------------------------------------------- /preprocessing/meetings/20230111.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jan 11, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * Resize - flip-invariant 8 | * Open floor 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Natalie Kershaw (Microsoft) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Janusz Lisiecki (NVIDIA) 15 | 16 | ## Recording & Slides 17 | 18 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230111.mp4) 19 | * [Meeting Slides](slides/20230111_slides.pdf) 20 | 21 | ## Meeting notes: 22 | 23 | ### Status 24 | 25 | - ResNet-50 tutorial https://github.com/onnx/tutorials/pull/266 26 | - Waiting for ONNX Runtime support for ONNX opset 18 27 | - ONNX Runtime code freeze will take place this week 28 | - Should cover the needed operators 29 | - Joaquin to give it a test once it gets merged to main 30 | 31 | ### Resize - flip-invariant 32 | 33 | - Current ONNX definition presents a potential problem when using a scale producing an output size with a fractional part that needs to be truncated 34 | - When this occurs, there's the question on where to place the ROI 35 | - Current ONNX definition places it to the beggining of the input: data=[100, 200], scale=1.3 -> output=[100., 165.38] 36 | - This results in an operation that is not flip-invariant, that is, flipping a dimension before or after the resize operator produces different results. 37 | - Joaquin proposed to add a new coordinate transform mode "half_pixel_symmetric" that retains the relative position of the center of the region in the output. 38 | - "half_pixel_symmetric" will produce flip-invariant results 39 | - The group agrees with such proposal, Joaquin will create a PR 40 | 41 | ### Open floor 42 | 43 | - ONNX runtime will add python scripts to prepend the preprocessing steps to a few of the ONNX zoo models. 44 | - It should be part of the new ONNX runtime release. 45 | - There are few example networks but the user can choose to write their own. 46 | 47 | -------------------------------------------------------------------------------- /preprocessing/meetings/20230308.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed March 8, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * Next steps 8 | * Open floor 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Natalie Kershaw (Microsoft) 13 | * Xavier Dupré (Microsoft) 14 | * Ganesan Ramalingan (Microsoft) 15 | * George Nash (Intel) 16 | * Janusz Lisiecki (NVIDIA) 17 | * Liqun Fu (Microsoft) 18 | 19 | ## Recording & Slides 20 | 21 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230308.mp4) 22 | * [Meeting Slides](slides/20230308_slides.pdf) 23 | 24 | ## Meeting notes: 25 | 26 | ### Status 27 | 28 | 29 | - [MERGED] Add Data preprocessing with ONNX: ResNet-50 example - https://github.com/onnx/tutorials/pull/266 30 | - [MERGED] Add Resize-19: half_pixel_symmetric - https://github.com/onnx/onnx/pull/4862 31 | - [Pending] Using ONNX parser in SequenceMap tutorial - https://github.com/onnx/tutorials/pull/277 32 | 33 | ### Next steps 34 | 35 | - Publishing ONNX models with preprocessing 36 | - We discussed the possibility of publishing all ResNet models augmented with the preprocessing steps. This is convenient for the user but adds a lot of overhead to maintain the models. 37 | - After some discussion we settled on posting the preprocessing model as a separate step and add instructions for the user to combine with one of the networks. 38 | - Additionally, we will look into adding a convenient API in onnx.hub that combines a network with a preprocessing model. 39 | - Some of work has been also done in onnxruntime-extensions 40 | - https://onnxruntime.ai/docs/tutorials/mobile/superres.html 41 | - https://github.com/microsoft/onnxruntime-extensions/blob/main/onnxruntime_extensions/tools/add_pre_post_processing_to_model.py 42 | - Natalie will bring the topic of adding image decoding/encoding to the ONNX spec on the next meeting. 43 | - We discussed adding a stand-alone documentation page describing the procedure of adding preprocessing to a model. 44 | - Once the documentation is completed and the ResNet model instructions published in the zoo, we will call the mission of this group accomplished and conclude it. 45 | 46 | 47 | ### Open floor -------------------------------------------------------------------------------- /preprocessing/meetings/20230419.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed April 19, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * Image decoding operator - 8 | * Porting (more) models onnxruntime-extensions 9 | * Open floor 10 | 11 | ## Atendees 12 | * Joaquin Anton (NVIDIA) 13 | * Natalie Kershaw (Microsoft) 14 | * Xavier Dupré (Microsoft) 15 | * Ganesan Ramalingan (Microsoft) 16 | * Janusz Lisiecki (NVIDIA) 17 | 18 | ## Recording & Slides 19 | 20 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230419.mp4) 21 | * [Meeting Slides](slides/20230419_slides.pdf) 22 | 23 | ## Meeting notes: 24 | 25 | ### Status 26 | 27 | - [MERGED] Add a way to load a composite model (preprocessing + network) via onnx.hub - https://github.com/onnx/onnx/pull/4976 28 | - [MERGED] Add ResNet preprocessing model -https://github.com/onnx/models/pull/594 29 | - [MERGED] Fix support for negative axes in Pad and CenterCropPad - https://github.com/onnx/onnx/pull/5007 30 | - [Pending] Using ONNX parser in SequenceMap tutorial - https://github.com/onnx/tutorials/pull/277 31 | 32 | - We discussed the fact that the documentation of onnx.hub doesn't reflect the new action functions. Xavier address that in this PR: https://github.com/onnx/onnx/pull/5156 33 | 34 | ### Image decoding operator 35 | 36 | - We discussed the utility of having an image decoder operator and the possible attributes that the operator should have 37 | - There are some options that would allow to match typical implementations (libjpeg) but would make it harder for implementers to comply with the spec. The rule of thumb is to add things that are generic and can be expressed mathematically. 38 | - We discussed adding an "image" domain. We should have image encoders as well for some applications (e.g. super resolution) 39 | - Image encoding is more complicated, as each format has its own encoding parameters. Because of that, it'd be easier to organize if we add dedicated format encoders: JpegEncoder, PngEncoder, etc. 40 | 41 | ### Porting (more) models onnxruntime-extensions 42 | 43 | - We discussed that it'd be nice to port some of the work done in onnxruntime-extensions to the ONNX model zoo 44 | - Would be interesting to see a new kind of application (e.g. NLP), to see if there are any gaps we didn't anticipate when working on the vision models. 45 | - Natalie agreed to analyze the BERT use case and come back with a plan of the work that needs to be done to publish this to the hub (needed operators, etc). 46 | 47 | ### Open floor 48 | 49 | 50 | ## Action items 51 | - Joaquin will come with a proposal for an initial version of the image decoder to the next meeting 52 | - Xavier to look at the missing documentation in onnx.hub 53 | - Natalie to analyze the BERT preprocessing model and share a plan 54 | -------------------------------------------------------------------------------- /preprocessing/meetings/20230510.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed May 10, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * Image decoder/encoder operator proposal 8 | * Open floor 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Alexandre Eichenberger (IBM, ONNX) 13 | * Liqun Fu (Microsoft) 14 | * George Nash (Intel) 15 | * Janusz Lisiecki (NVIDIA) 16 | 17 | ## Recording & Slides 18 | 19 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230510.mp4) 20 | * [Meeting Slides](slides/20230510_slides.pdf) 21 | 22 | ## Meeting notes: 23 | 24 | ### Status 25 | 26 | - [Merged] Extend python API documentation - https://github.com/onnx/onnx/pull/5156 27 | - [Pending] Using ONNX parser in SequenceMap tutorial - https://github.com/onnx/tutorials/pull/277 28 | 29 | ### Image decoder/encoder operator proposal 30 | 31 | - As discussed previously, Joaquin proposed two operators: ImageDecode and ImageRead, used to load/decode images (as in cv2.imread or PIL.open) 32 | - Supported formats are: BMP, JPEG, JPEG2000, TIFF, PNG, WebP, Portable image format (PBM, PGM, PPM, PXM, PNM), all of those easily supported through generic tools such as OpenCV or Pillow 33 | - Backend implementers might decide to have specialized decoders as implementations, but the should be easy to implement via generic tools. 34 | - Channel-last layout will be the default, as image processing pipelines usually operate with such layout. 35 | - Chroma upsampling "linear" mode proposal matches what's done in libjpeg, which is not part of the JPEG standard. Most image toolkits use libjpeg under the hood, therefore using this kind of chroma upsampling 36 | - The operator shall be open for extension, but attempt to cover the most typical use-case first. 37 | - We discussed the possibility to support JPEG lossless formats, typically used in medical imaging. We've seen some work being done in libjpeg-turbo that would enable supporting lossless JPEGs easily, but for now we will keep it out of scope. 38 | - Image encoders were not yet discussed. 39 | - ImageRead works with file paths and ImageDecode works with encoded streams. 40 | 41 | ## Open floor 42 | 43 | ### Date processing 44 | - Alexandre Eichenberger talked about a proposal discussed in the roadmap meeting regarding date processing - https://github.com/onnx/steering-committee/blob/main/roadmap/2023-docs/18-date-parsing.pdf 45 | - The preprocessing group has no objection to such proposal, but should be eventually reviewed by the Operators SIG. 46 | - Joaquin will reach out to the original output to invite him to the upcoming meetings for further discussions. 47 | 48 | ### Medical: 3D GridSample, Grid generators, Sliding window inferer 49 | - Liqun Fu talked about porting medical models (e.g. UNet-3D) from MONAI AI project to ONNX 50 | - There are currently a few blockers: 51 | * ND GridSample to support volumetric inputs - https://github.com/onnx/onnx/pull/5010 52 | * Grid generators to match Torch's affine_grid_generator - https://github.com/onnx/onnx/issues/4983 53 | * SlidingWindowInferer 54 | - The group agrees those proposals make sense and see no issues with it. 55 | - The Operators SIG should agree with the details. 56 | 57 | ### Porting Microsoft preprocessing extensions to ONNX 58 | - Alexandre Eichenberger asked whether we want to move some of the Microsoft extensions done for preprocessing to the ONNX standard 59 | - Such work has been already started (as discussed in the previous meeting) 60 | * ImageDecode is one of them 61 | * BERT model is currently under analysis (Natalie Kershaw) 62 | 63 | 64 | -------------------------------------------------------------------------------- /preprocessing/meetings/20230614.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed June 14, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * Image decoder operator proposal 8 | * Open floor 9 | 10 | ## Atendees 11 | * Joaquin Anton (NVIDIA) 12 | * Liqun Fu (Microsoft) 13 | * Ganesan Ramalingan (Microsoft) 14 | * Xavier Dupré (Microsoft) 15 | * George Nash (Intel) 16 | * Janusz Lisiecki (NVIDIA) 17 | 18 | ## Recording & Slides 19 | 20 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230614.mp4) 21 | * [Meeting Slides](slides/20230614_slides.pdf) 22 | 23 | ## Meeting notes: 24 | 25 | ### Status 26 | 27 | - [IN PROGRESS] Affine grid generator operator - https://github.com/onnx/onnx/pull/5225 28 | - [IN PROGRESS] Image decoder operator - https://github.com/onnx/onnx/pull/5294 29 | - [MERGED] Using ONNX parser in SequenceMap tutorial - https://github.com/onnx/tutorials/pull/277 30 | 31 | ### Image decoder operator proposal 32 | 33 | - We talked about the ImageDecoder PR 34 | - We agreed that generated test images are enough for the purpose of our tests 35 | - We'll need to figure out how extensive we want the test coverage to be regarding codec specific options. 36 | - We should make opencv2 dependency optional and just fail the test if not available. Should list the dependencies only in the requirements-dev list. 37 | - We agreed to add a note that support for lossless JPEG is optional, since it would be hard to find a suitable reference implementation. 38 | - Channel-last (interleaved) layout is chosen. Planar layouts can be achieved via transpose. We can always add the option to support planar layouts later if needed. 39 | 40 | ## Open floor 41 | 42 | ### Operators to support different applications from MONAI 43 | 44 | - Medical data, segmentation/classification) 45 | - Affine grid generator. Operator currently in review. 46 | - Sliding window inferrer is work in progress 47 | - We discussed the possibility to introduce higher level abstractions for affine matrix generators. Joaquin pointed how it is done in NVIDIA DALI as an example: [Example](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/examples/math/geometric_transforms.html), [Documentation](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/operations/nvidia.dali.fn.transforms.html) 48 | - Several examples from MONAI could be enabled with the Affine grid generator + Sliding window inferrer. 49 | -------------------------------------------------------------------------------- /preprocessing/meetings/20230712.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed July 12, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * Image decoder operator 8 | * String operators 9 | * New Operator: ParseDateTime 10 | * Open floor 11 | 12 | ## Atendees 13 | * Joaquin Anton (NVIDIA) 14 | * Ganesan Ramalingan (Microsoft) 15 | * Aditya Goel (Quantco) 16 | * Christian Bourjau (Quantco) 17 | * George Nash (Intel) 18 | * Janusz Lisiecki (NVIDIA) 19 | 20 | ## Recording & Slides 21 | 22 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230712.mp4) 23 | * [Meeting Slides](slides/20230712_slides.pdf) 24 | 25 | ## Meeting notes: 26 | 27 | ### Status 28 | 29 | - [IN PROGRESS] Affine grid generator operator - https://github.com/onnx/onnx/pull/5225 30 | - [IN PROGRESS] Image decoder operator - https://github.com/onnx/onnx/pull/5294 31 | - [IN PROGRESS] StringSplit operator - https://github.com/onnx/onnx/pull/5371 32 | - [IN PROGRESS] StringConcat Operator - https://github.com/onnx/onnx/pull/5350 33 | - [IN PROGRESS] Add RegexFullMatch operator - https://github.com/onnx/onnx/pull/5401 34 | 35 | ### Image decoder operator proposal 36 | 37 | - As discussed during review, the operator has been moved to a separate domain ai.onnx.io 38 | - IO as a name is a little bit vague, but enables future additions for pre and post processing. 39 | - It is not clear for us what implications we want to have when using domains. Are the domains less important compared to core opset? Are those optional? 40 | - The usage of domains shall be discussed further on the Operator SIG tomorrow (July 13, 2023). 41 | 42 | ### String operators 43 | 44 | - Aditya Goel presented three open PRs with string process utils: 45 | 46 | * StringSplit operator - https://github.com/onnx/onnx/pull/5371 47 | * StringConcat Operator - https://github.com/onnx/onnx/pull/5350 48 | * Add RegexFullMatch operator - https://github.com/onnx/onnx/pull/5401 49 | 50 | - We discussed a standing issue with SpringSplit, about how to represent a variadic number of strings. The two options were to use an extra dimension for the number of strings, which will be padded to the maximum of the batch. Alternatively, we could have a densely packed representation that will use the memory more efficiently, but would be hard to consume. The group agrees to go with the extra dimension + padding approach. 51 | 52 | ### ParseDateTime 53 | 54 | - Christian Bourjau presented a new operator proposal, which was presented during the roadmap meeting, and also discussed in the past. 55 | - https://github.com/onnx/onnx/issues/5409 56 | - The operator parses a string representation of a date/time to a number of seconds (or other unit). It should behave like UNIX's strptime. 57 | - We discussed the choice of using double for the parsed string, instead of the more natural integer representation. One advantage is that double can represent NaN. 58 | - Christian will check what other Python libraries use for representation and also discuss it further during the ONNX Operators SIG meeting. 59 | 60 | ## Open floor 61 | -------------------------------------------------------------------------------- /preprocessing/meetings/20231011.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Oct 11, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * Status 7 | * ImageDecoder in ResNet - ONNX / ORT release plans 8 | * Next steps 9 | * Open floor 10 | 11 | 12 | ## Atendees 13 | * Joaquin Anton (NVIDIA) 14 | * Ganesan Ramalingan (Microsoft) 15 | * Liqun Fu (Microsoft) 16 | * George Nash (Intel) 17 | * Janusz Lisiecki (NVIDIA) 18 | 19 | ## Recording & Slides 20 | 21 | * Meeting Recording not available 22 | * [Meeting Slides](slides/20231011_slides.pdf) 23 | 24 | ## Meeting notes: 25 | 26 | ### Status 27 | 28 | * [MERGED] Affine grid generator operator - https://github.com/onnx/onnx/pull/5225 29 | * [MERGED] Image decoder operator - https://github.com/onnx/onnx/pull/5294 30 | * [MERGED] StringSplit operator - https://github.com/onnx/onnx/pull/5371 31 | * [MERGED] StringConcat Operator - https://github.com/onnx/onnx/pull/5350 32 | * [MERGED] Add RegexFullMatch operator - https://github.com/onnx/onnx/pull/5401 33 | 34 | ### ImageDecoder in ResNet - ONNX / ORT release plans 35 | 36 | * ONNX 1.15 release - Tentative October 10 37 | * ONNX Runtime supporting opset 20: 38 | 39 | - 2 months after the ONNX release 40 | - Code freeze ~November 41 | - Release ~December 42 | 43 | * Draft PR: 44 | 45 | - https://github.com/onnx/models/pull/627 46 | - Will wait until ONNX runtime has support for image decoder 47 | 48 | * Implementation 49 | 50 | - Planned for next release. Will probably use OpenCV or and/other libraries (TurboJPEG) 51 | 52 | ### Next steps 53 | 54 | * Enable preprocessing models for other domains 55 | 56 | - NLP (BERT?) - tokenizer, etc 57 | - Medical (UNet-3D?) - Liqun Fu will present about its requirements on next meeting 58 | - Audio 59 | 60 | * Moving work from onnxruntime-extensions into the standard 61 | 62 | - Need to be addressed on operator by operator basis 63 | 64 | ### Open floor 65 | 66 | * The goal of this group is to fill the gap on ONNX models, so that the preprocessing part is not done separately. 67 | * Proposing operators can be handled by the Operators SIG 68 | -------------------------------------------------------------------------------- /preprocessing/meetings/20231108.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Nov 8, 2023 at 8:00am PDT 4 | 5 | ## Agenda 6 | * UNet-3D progress - @Liqun Fu 7 | * Open floor 8 | 9 | ## Atendees 10 | * Liqun Fu (Microsoft) 11 | * Joaquin Anton (NVIDIA) 12 | * Ganesan Ramalingan (Microsoft) 13 | * Xavier Dupré (Microsoft) 14 | 15 | ## Recording & Slides 16 | 17 | * [Meeting Recording](https://lists.lfaidata.foundation/g/onnx-wg-preprocessing/files/onnx_preprocessing_20230712.mp4) 18 | * No slides 19 | 20 | ## Meeting notes: 21 | 22 | ### UNet-3D progress 23 | 24 | * Liqun Fu has been working on conversion of UNet networks from Monai to ONNX 25 | * convert_to_onnx -> exports MONAI model to ONNX 26 | * Getting support from the team for CI system would help 27 | * Medical imaging workflow consists of 3 stages: preprocessing, inferer, and postprocessing. 28 | * Currently we only have support for the inferer part, excluding the Sliding window inferer. 29 | * A basic model is applied to patches of the image 30 | 31 | * Preprocessing: 32 | 33 | - Adjust orientation and spacing 34 | - Postprocessing usually reverts what was done in preprocessing 35 | 36 | * Two operators: 37 | 38 | - AffineGrid -> Generate grid for adjusting orientation & spacing 39 | - GridSample supporting now 3D 40 | 41 | * Needs more time 42 | * Repo keeps changing -> hard to keep track 43 | * What mathematical blocks do we need? 44 | 45 | ### Open floor 46 | 47 | * The ORT teams has a concern regarding adding a dependency with OpenCV 48 | * Most likely will be handled via an ORT extension 49 | * Discussion to be continued next time 50 | 51 | -------------------------------------------------------------------------------- /preprocessing/meetings/20240110.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wed Jan 10, 2025 at 8:00am PDT 4 | 5 | ## Atendees 6 | * Joaquin Anton (NVIDIA) 7 | * Ganesan Ramalingan (Microsoft) 8 | * Janusz Lisiecki (NVIDIA) 9 | 10 | ## Meeting notes: 11 | 12 | ### Migration to LFX 13 | 14 | - We migrated the meeting from groups.io to LFX. 15 | - Rama reported that he didn't get a calendar invite 16 | - Joaquin will check that the list of attendees is up-to-date 17 | 18 | ### ImageDecoder support on ONNX runtime 19 | 20 | - We discussed during the previous meeting that the ONNX runtime team has concerns with introducing a dependency with a library such as OpenCV, due to binary size. 21 | - We agreed to continue the discussion on this meeting but Liqun Fu could not attend, who knows more about the progress 22 | - Joaquin will start a Github issue on ONNX runtime repository to follow this up. 23 | 24 | ### Other domains 25 | 26 | - We have discussed earlier about expanding preprocessing to other domains. 27 | - Currently two lines of work: NLP (Natalie) and Volumetric images (Liqun). 28 | - Both reported that they don't have enough time to focus on those at the moment. 29 | 30 | ### Linear Algebra domain 31 | 32 | - https://github.com/onnx/sigs/issues/169 33 | - https://github.com/onnx/onnx/pull/5821 34 | - There is an initiative to add a linear algebra domain 35 | - From the point of view of this group, it is OK to add new domains if this is the direction that the Operators SIG want to follow. 36 | - Joaquin suggested to not label the domain as "pre-processing" but there's no strong opinion either way. 37 | - To be discussed at the Operators SIG. -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20210811_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20210811_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20211013_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20211013_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20211110_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20211110_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20211208_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20211208_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20220112_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20220112_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20220309_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20220309_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20220413_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20220413_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20220511_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20220511_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20220608_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20220608_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20220713_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20220713_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20221019_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20221019_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20221109_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20221109_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20230111_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20230111_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20230308_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20230308_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20230419_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20230419_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20230510_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20230510_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20230614_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20230614_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20230712_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20230712_slides.pdf -------------------------------------------------------------------------------- /preprocessing/meetings/slides/20231011_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/meetings/slides/20231011_slides.pdf -------------------------------------------------------------------------------- /preprocessing/notebooks/batch-preprocessing/loop-test.onnx.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/batch-preprocessing/loop-test.onnx.1.png -------------------------------------------------------------------------------- /preprocessing/notebooks/batch-preprocessing/loop-test.onnx.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/batch-preprocessing/loop-test.onnx.2.png -------------------------------------------------------------------------------- /preprocessing/notebooks/batch-preprocessing/preprocessing.onnx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/batch-preprocessing/preprocessing.onnx.png -------------------------------------------------------------------------------- /preprocessing/notebooks/images/cute-4074304_224.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/images/cute-4074304_224.jpg -------------------------------------------------------------------------------- /preprocessing/notebooks/images/grasshopper-4357903_1280.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/images/grasshopper-4357903_1280.jpg -------------------------------------------------------------------------------- /preprocessing/notebooks/images/image_info.txt: -------------------------------------------------------------------------------- 1 | ====================== 2 | Image source - pixabay.com 3 | 4 | License : 5 | CC0 Creative Commons 6 | Free for commercial use 7 | No attribution required 8 | 9 | https://pixabay.com/photos/snail-shell-mollusk-probe-mucus-4291306/ - snail-4291306_1280.jpg 10 | https://pixabay.com/photos/cute-big-friendly-happy-4074304/ - cute-4074304_224.jpg 11 | https://pixabay.com/photos/grasshopper-cricket-insect-nature-4357903/ - grasshopper-4357903_1280.jpg 12 | 13 | ====================== 14 | -------------------------------------------------------------------------------- /preprocessing/notebooks/images/snail-4345504_1280.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/images/snail-4345504_1280.jpg -------------------------------------------------------------------------------- /preprocessing/notebooks/preprocessing-rn50/onnx-preprocessing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/preprocessing-rn50/onnx-preprocessing.png -------------------------------------------------------------------------------- /preprocessing/notebooks/preprocessing-rn50/rn50-preprocessing-resize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/preprocessing/notebooks/preprocessing-rn50/rn50-preprocessing-resize.png -------------------------------------------------------------------------------- /release/docs/onnx_release_verification.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Release Verification 4 | 5 | ## SCENARIO 1 (Ideal case) 6 | 7 | Assumption: 8 | 1. ONNX specification is independent and all the verification process is inside ONNX core repo. It doesn't depend on any backend/frontend converter or runtime. 9 | 2. Each operator has a very clear description on how it works. If there are any optional attributes, then a default value will be defined for each optional attribute. If there are optional inputs, then user can provide an empty string as name to skip that input. 10 | 3. The test cases of each operator cover all combination of required/optional inputs and required/optional attributes. The test cases cover corner cases too. The expected values of the test cases are all computed in the testcases. 11 | 12 | Release Verification Process: 13 | 1. Run ONNX CI. If the CI pass, then the release verification process completed. 14 | 15 | ## SCENARIO 2 16 | 17 | Assumption: 18 | 1. ONNX specification is independent. 19 | 2. Each operator has good description on how it works. But some operators didn't define the default values for optional attribute/inputs 20 | 3. Test cases only cover the main or simple usages, didn't cover corner cases. Some operators don't have any test cases at all. 21 | 4. The verification process depends on a backend runtime to verify the ONNX specification is implementable. And ONNX relies on this backend runtime to test the operators that don't have any test cases in ONNX core. This runtime supported all of the latest ONNX operators. 22 | 5. ONNX doesn't depend on any frontend/backend converter. However, in release 1.7 ONNX depend on a few frontend converters like TensorFlow-ONNX converter. 23 | 24 | Release Verification Process: 25 | 1. Run ONNX CI. 26 | 2. Run all the ONNX backend tests on the selected backend runtime in assumption 4. 27 | 3. Create and run test that is not covered in ONNX backend tests on the selected backend runtime. 28 | 4. Run test on converter? 29 | 30 | Problems need to address in Scenario 2: 31 | 1. What is the selected backend runtime? ONNX Runtime? Or some other runtime? 32 | ```ONNX Runtime is the selected backend runtime.``` 33 | 2. Who in charge of making sure the selected backend runtime supports all the latest ONNX operators before the next ONNX release? 34 | ```ONNX Runtime community is responsible to implement all ONNX operators in a reasonable timeframe.``` 35 | 3. Should the author of the PR that introduce the new/updated operator implement the support for that operator on the selected backend runtime? Or the author just needs to open an issue on the selected runtime and let the runtime to prioritize when to implement it? 36 | ```No. ONNX Runtime community is responsible to implement all ONNX operators in a reasonable timeframe.``` 37 | 4. Who is responsible to create the test cases that is not covered in ONNX backend tests and the selected backend runtime? 38 | 5. How to verify those ONNX operators that are not supported in the selected runtime? Wait until the selected runtime support them and if there are any problem open issue in ONNX at that time? 39 | ```ONNX Runtime community is responsible to implement all ONNX operators in a reasonable timeframe. Any operators that are not supported by ONNX Runtime are not fully verify yet. ONNX Runtime community will open issue for ONNX operator specification problem identify during their implementation time. ONNX Runtime community will try to align with the ONNX release schedule.``` 40 | 6. What converters are ONNX depend on? Frontend or backend converter or both? TensorFlow-ONNX converter? Or some other frontend converter? How about backend converter? How does the converter verify ONNX? What test need to run? 41 | 42 | 43 | What is the release verification process in Release 1.7? 44 | 1. What are the actual verification tests in release 1.7? 45 | 2. After running ONNX CI, did it run any other test? 46 | 3. Did it run test on any backend/frontend converter or runtime? If yes, what kind of test did it run? How to run those tests? 47 | -------------------------------------------------------------------------------- /release/meetings/001-20200603.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday June 03, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Introduction 7 | * Background 8 | * Discussions 9 | * Working group objectives 10 | * Working group exit criteria 11 | 12 | ## Attendees 13 | * Chin Huang (IBM) 14 | * Ashwini Khade (Microsoft) 15 | * Winnie Tsang (IBM) 16 | 17 | ## Notes 18 | * Reviewed and discussed WG objectives 19 | * Identify and document release process 20 | * Clarify and enhance release verification 21 | * Identify areas to be fixed and enhanced for release builds in ONNX, wheel builder, and onnx-feedstock 22 | * Recommend release operational guidelines in communication, frequency, scope, dependencies, and schedule 23 | * Reviewed and discussed high level WG exit criteria. Given the limited resources, mostly 24 | * Keep release process doc up-to-date 25 | * Recommend release verification enhancements to SC and Infra SIG 26 | * Open issues to track work items in release repos 27 | * Document release operational guidelines 28 | * Draft release 1.8 plan and help finish release 1.8 29 | * Communication channels 30 | * Gitter: https://gitter.im/onnx/Releases 31 | * WG repo: https://github.com/onnx/working-groups 32 | * LF AI: https://lists.lfai.foundation/g/onnx-wg-release 33 | * We would like to target to complete the working group missions after ONNX 1.8 is released 34 | * Next meeting is in two weeks 35 | 36 | ## Action items 37 | * Chin - update Steering Committee with the working group objectives and exit criteria 38 | * All - think about general approaches and detailed actions for WG to operate effectively 39 | -------------------------------------------------------------------------------- /release/meetings/002-20200624.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday June 24, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Objectives review 7 | * Discussions 8 | * Release process 9 | * Release verification 10 | * Automation projects 11 | * Operational guideline 12 | * Release 1.8 13 | 14 | ## Attendees 15 | * Chin Huang (IBM) 16 | * Ashwini Khade (Microsoft) 17 | * Winnie Tsang (IBM) 18 | * Tobias Oberrauch 19 | 20 | ## Notes 21 | * Quickly reviewed working group objectives 22 | * Reviewed the updated release process doc, key points in addition to specific 1.7 steps (Chin will open a PR) 23 | * Change release interval from 2 months to 4 months 24 | * Set the target date for 1.8, Sept 15, 2020 25 | * The 1.8 features/scope will be decided by SC, SIGs, and WGs and tracked in a separate doc/issue 26 | * The release verification should to be clear, transparent, and automated if possible. Winnie will look into a proposal. 27 | * A couple of release automation projects need some updates (Tobias will take a look) 28 | * wheel builder should build Windows wheels in Azure Pipelines and upload to Pypi and TestPypi 29 | * onnx feedstock should produce ONNX Windows packages for conda users 30 | * Discussed the release operational guidelines and tentatively agreed (Ashwini will take a look) 31 | * Time based releases, 3-4 times a year 32 | * Release manager, rotating between participating companies, should be identified 4-6 weeks prior to target release date 33 | * Scope should be decided by SC, SIGs, and WGs 34 | * Dependencies should be clearly documented for each release if can't be generally covered by the verification doc. 35 | * Open an issue to communicate release date, scope, dependencies (could be a checklist of ONNX related projects so the owners are actively participating in the release process) and track progress 36 | 37 | ## Action items 38 | * All - look into specific areas and prepare to provide proposals or updates 39 | -------------------------------------------------------------------------------- /release/meetings/003-20200715.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday July 15, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release process 8 | * Release verification 9 | * Automation projects 10 | * Operational guidelines 11 | * Release 1.8 12 | 13 | ## Attendees 14 | * Chin Huang (IBM) 15 | * Ashwini Khade (Microsoft) 16 | * Winnie Tsang (IBM) 17 | * Prasanth Pulavarthi (Microsoft) 18 | * Wei-Sheng Chin (Microsoft) 19 | 20 | ## Notes 21 | * Steering Committee updates 22 | * Mid Sept for 1.8, do we have enough new features? Should we do a time-based release or a feature-based? The current propoal is a time-based release around Sept-Oct. Will gather inputs from Steering Committee, working groups, and SIGs to finalize the plan. 23 | * Release manager to be identified 6 weeks before target date. 24 | * Align community meeting closer to release 25 | * Reviewed the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 26 | * Adddional work to be done for approval? We would like to include the release verification details for 1.7 as a reference. Chin will follow up with the people who worked on release 1.7 verification. 27 | * The release verification should to be clear, transparent, and automated if possible. 28 | * Current issue: dependency on some runtime and frontend converters 29 | * Proposed solutions: review https://github.com/onnx/working-groups/blob/master/release/docs/onnx_release_verification.md 30 | * Scenario 2 is more like what we have been practicing, define operator specs in onnx first and verify the spec in onnx runtime implementation. Winnie will update the doc to reflect our conclusion, ONNX runtime will be the primary backend to verify release candidates. 31 | * A couple of release automation projects need some updates 32 | * wheel builder and onnx feedstock 33 | * We will open an issue to track progress in ONNX repo (Chin to do) 34 | * Did not have time to discuss the release operational guidelines 35 | * where to put the guidelines? ONNX or WG repo 36 | * Release 1.8 37 | * Open an issue to communicate release date, scope, dependencies (Ashwini to do) 38 | 39 | ## Action items 40 | * All - look into specific areas and prepare to provide proposals or updates 41 | -------------------------------------------------------------------------------- /release/meetings/004-20200729.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday July 29, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Steering committee update 8 | * Release verification 9 | * Automation projects 10 | * Operational guidelines 11 | * Release 1.8 12 | 13 | ## Attendees 14 | * Chin Huang (IBM) 15 | * Ashwini Khade (Microsoft) 16 | * Winnie Tsang (IBM) 17 | * Changming Sun (Microsoft) 18 | * Faith Xu (Microsoft) 19 | * Dmitri Smirnov (Microsoft) 20 | 21 | ## Notes 22 | * Steering Committee updates 23 | * Release dependency needs to be defined (ONNX Runtime, etc). Should it depend on converters? We will recommend no dependency on converters. 24 | * Time based release with exit criteria documented (an issue in ONNX github) Do we have nough features for the time based release? Do we need big list of features? We will get advice from SC this afternoon. 25 | * Release package is quite large at the moment. Consider having multiple packages. (issue https://github.com/onnx/onnx/issues/2917) We recommend having scripts to generate data, not including test data in the release package, instead of creating two packages. 26 | * Where should ONNX Runtime live? 27 | * Reviewed the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 28 | * Requested for 1.7 verification details (issue https://github.com/onnx/onnx/issues/2614) Ashwini to follow up with Zeeshan. 29 | * A key problem is release 1.7 is that shape inference can be verified only if op is implemented. 30 | * New ops need to have python reference implementation based on a backend. The CI will cover node level tests. Still need release verification. 31 | * The release verification should to be clear, transparent, and automated if possible. 32 | * Scenario 2 is the practical choice, ONNX runtime will be the primary backend to verify release candidates. 33 | * A couple of release automation projects need some updates 34 | * wheel builder and onnx feedstock 35 | * Opened an issue to track progress in ONNX repo (issue https://github.com/onnx/onnx/issues/2904). We should consolidate build process in one place, Github Actions or wheel_builder. More investigation is needed and Changming doesn't have time to make the change for 1.8. Likely the 1.8 process will be the same as 1.7. 36 | * Release operational guidelines 37 | * where to put the guidelines? ONNX or WG repo 38 | * Release 1.8 39 | * Open an issue to communicate release date, scope, dependencies. Ashwini will open an issue after clarifying the date/features with SC. 40 | 41 | ## Action items 42 | * All - look into specific areas and prepare to provide proposals or updates 43 | -------------------------------------------------------------------------------- /release/meetings/005-20200819.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday August 19, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Steering committee update 8 | * Release verification 9 | * Automation projects 10 | * Operational guidelines 11 | * Release 1.8 12 | 13 | ## Attendees 14 | * Chin Huang (IBM) 15 | * Ashwini Khade (Microsoft) 16 | * Winnie Tsang (IBM) 17 | 18 | ## Notes 19 | * Steering Committee updates 20 | * Do we have enough features for the time based release? 21 | * Workshop is scheduled for 10/14, aligned with release, indicating release 1.8 should be mid-late Oct. 22 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 23 | * Requested for 1.7 verification details (issue https://github.com/onnx/onnx/issues/2614) 24 | * The release verification process updates: reached out to developer, will be followed up. 25 | * A couple of release automation projects need some updates 26 | * wheel builder consolidation with github actions: suggest to first update the existing env setup in wheels_builder for 1.8 and work on consolidation later 27 | * onnx feedstock enable Windows conda package: a resource is identified, however would like to know whether Tobias has made progress. 28 | * Opened an issue to track progress in ONNX repo (issue https://github.com/onnx/onnx/issues/2904) 29 | * Release operational guidelines 30 | * release process doc + release tracking issue 31 | * Release 1.8 32 | * Open an issue to communicate release date, scope, dependencies (issue https://github.com/onnx/onnx/issues/2942) 33 | * Identify release manager? In next release WG meeting 34 | * Release features/highlights? Will follow the roadmap discusssions and update the tracking issue 35 | * New and updated operators 36 | * Hard to see the new and updated operators between releases. opeartor.md and operator_sets.h provide the list however no details on the changes. It would be very helpful for ONNX users and developers to see the spec changes and additions in one place. 37 | 38 | ## Action items 39 | * Identity 1.8 release features and associate them with SC roadmap doc 40 | * Look into wheel builder env updates for release 1.8 41 | * Fix Windows conda package and install, identify the resource to work on this high priority item for 1.8 42 | * Identify and document release verification process 43 | -------------------------------------------------------------------------------- /release/meetings/006-20200902.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Sept 2, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Ashwini Khade (Microsoft) 14 | * Winnie Tsang (IBM) 15 | * Rajeev Nalawadi (Intel) 16 | * Jason Plurad (IBM) 17 | 18 | ## Notes 19 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 20 | * The release verification process updates: reached out to developer, expect to get updates next meeting. Should identify what has been done and how to run it so anyone can do it for future releases. 21 | * A couple of release automation projects need some updates (issue https://github.com/onnx/onnx/issues/2904) 22 | * wheel builder update for new env setup for 1.8: nothing is broken, need to update environment and dependency versions, such as python and protobuf 23 | * onnx feedstock enable Windows conda package: currently working on it, priority 24 | * Release 1.8 (issue https://github.com/onnx/onnx/issues/2942) 25 | * Identified release manager, will join next meeting 26 | * Plan release date, doesn't have to be after workshop which is on Oct 14. The exact date will be determined based on features and roadmap discussions in a couple of weeks. 27 | * Release features/highlights, to be summarized and documented by release manager 28 | * New and updated operators 29 | * How/where to see the new and updated operators between releases? Open an issue to track the desired feature. Would like make clear the changes in detail including defaults, data types, attributes vs inputs, new or optional attributes, logic changes. Description might need to be added to source code for doc generation. 30 | 31 | ## Action items 32 | * Identify release verification details 33 | * Look into environments and dependencies for 1.8 34 | -------------------------------------------------------------------------------- /release/meetings/007-20200916.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Sept 16, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Ashwini Khade (Microsoft) 14 | * Winnie Tsang (IBM) 15 | * Jackie Chun-Wei Chen (Microsoft) 16 | * Jason Plurad (IBM) 17 | 18 | ## Notes 19 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 20 | * The release verification process: Ashwini connected with Zeeshan, will provide update next meeting 21 | * A couple of release automation projects need some updates (issue https://github.com/onnx/onnx/issues/2904) 22 | * onnx feedstock enable Windows conda package: PR merged 2 days ago and Windows package for release 1.7 is available at Conda Forge, https://anaconda.org/conda-forge/onnx 23 | * wheel builder update for new env setup for 1.8: the first step is to replace travis with Azure pipelines in wheel builder. If time permits, will do everything in Github Actions. 24 | * Release 1.8 (issue https://github.com/onnx/onnx/issues/2942) 25 | * Identified release manager, Jackie Chen 26 | * Plan release date, a week after the workshop, so it should in the week of 10/19 27 | * Release features/highlights, will gather inputs from SIG and WG leaders such as Wei-Sheng Chin 28 | * Python 2.7 is no longer supported, as determined and documented during 1.7 release, should be removed from the release 1.8 build pipelines 29 | * Any roadmap discussions would affect release 1.8 scope? Should clarify with Steering Committee 30 | * Will start weekly meetings from now until release 1.8 is out 31 | 32 | ## Action items 33 | -------------------------------------------------------------------------------- /release/meetings/008-20200923.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Sept 23, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Ashwini Khade (Microsoft) 14 | * Winnie Tsang (IBM) 15 | * Jackie Chun-Wei Chen (Microsoft) 16 | * Jason Plurad (IBM) 17 | 18 | ## Notes 19 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 20 | * The release verification process is not clear. The functional verification is by running some tests in ONNT runtime. We should document and close the PR by 1.8 release time 21 | * A couple of release automation projects need some updates (issue https://github.com/onnx/onnx/issues/2904) 22 | * onnx feedstock enable Windows conda package: partner companies to comment on whether working or not using Windows Conda environment 23 | * wheel builder update for new env setup for 1.8: will be worked to use Azure pipelines 24 | * Release 1.8 (issue https://github.com/onnx/onnx/issues/2942) 25 | * Release features/highlights: create a 1.8 milestone in github with the release date so developers can tag PRs for the release 26 | * Python 2.7 is no longer supported, some work has been done to remove from CI, will need to update the release 1.8 build process 27 | * Any roadmap discussions would affect release 1.8 scope? No specific items from roadmap discussions. Will leave it to indivisual item owners to decide if fit into the release timeframe 28 | 29 | ## Action items 30 | * Create 1.8 milestone and announce in slack 31 | -------------------------------------------------------------------------------- /release/meetings/009-20200930.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Sept 30, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Ashwini Khade (Microsoft) 14 | * Winnie Tsang (IBM) 15 | * Jackie Chun-Wei Chen (Microsoft) 16 | * Jason Plurad (IBM) 17 | 18 | ## Notes 19 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 20 | * The release verification process is not clear. The functional verification is by running some tests in ONNT runtime. We should document and close the PR by 1.8 release time. Details will come from Jackie. 21 | * A couple of release automation projects need some updates (issue https://github.com/onnx/onnx/issues/2904) 22 | * onnx feedstock enable Windows conda package: Jason from IBM has confirmed that the Windows Conda package works with the onnx-tensorflow converter * wheel builder update for new env setup for 1.8: Jackie will update it to use Azure pipelines 23 | * Release 1.8 (issue https://github.com/onnx/onnx/issues/2942) 24 | * Release features/highlights: created a 1.8 milestone, deadline 10/9, in github with the release date and a few PRs are tagged for the release 25 | * Should we tag completed PRs? Easiest way to filter and identify the PRs for a particular release, which was done for 1.6 and 1.7. Up to the release manager to decide the final approach for 1.8. 26 | * How to gather inputs and provide summary for release notes? For 1.7, Chin went through 1. the the list of PRs for new and updated ops 2. ArchInfra SIG lead Ke for general highlights 3. Training working group leads for key additions for training support. For 1.8, we could ask SIG and WG leads to provide summary. 27 | 28 | ## Action items 29 | -------------------------------------------------------------------------------- /release/meetings/010-20201007.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Oct 7, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Ashwini Khade (Microsoft) 14 | * Winnie Tsang (IBM) 15 | * Jackie Chun-Wei Chen (Microsoft) 16 | * Jason Plurad (IBM) 17 | 18 | ## Notes 19 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 20 | * The release verification process is not clear. Jackie provided the updates: 21 | * The functional verification will be done by running node and shape inference tests in ONNX runtime, using ONNX repo release branch. 22 | * The package verification will be done by running ONNX unit tests and model tests with model zoo models, using release candidates from TestPypi. 23 | * A couple of release automation projects need some updates (issue https://github.com/onnx/onnx/issues/2904) 24 | * onnx feedstock enables Windows conda package: completed and verified 25 | * wheel build updated for release 1.8: to be worked on to build release packages and push them to TestPypi and Pypi 26 | * Release 1.8 (issue https://github.com/onnx/onnx/issues/2942) 27 | * Release features/highlights: created a 1.8 milestone, deadline 10/9 28 | * Should we provide release 1.8 update in 10/14 workshop? There was a short update for 1.7 as part of the TSC session in last workshop. Jackie to reach out to TSC or workshop host for a time slot. 29 | * We should plan and announce the date to create release branch so that developers know the deadline to complete and merger their PRs. 30 | 31 | ## Action items 32 | -------------------------------------------------------------------------------- /release/meetings/011-20201104.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Oct 7, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Winnie Tsang (IBM) 14 | * Jacky Chun-Wei Chen (Microsoft) 15 | * Anna June 16 | 17 | ## Notes 18 | * Review the updated release process doc (PR https://github.com/onnx/onnx/pull/2866) 19 | * The release verification process will be documented. Jackie confirmed: 20 | * The functional verification is done by running node and shape inference tests in ONNX runtime, using a commit ID in ONNX repo release branch. 21 | * The package verification is done by running ONNX unit tests and model tests with model zoo models, using release candidates from TestPypi. 22 | * A couple of release automation projects need some updates (issue https://github.com/onnx/onnx/issues/2904) 23 | * wheel builder is updated for release 1.8 24 | * Release 1.8 (issue https://github.com/onnx/onnx/issues/2942) 25 | * CI is much faster after transfer to travis.com 26 | * Python 3.5 has problem uploading packages. The solution is to upload to AWS. Jacky builds wheel files on the side and uploads manually. 27 | * There is on-going PRs in ONNX Runtime related to bumping up version number. 28 | * Currently verifying 1.8 candidate with converters. Fighting a bug in Tensorflow-ONNX now, expected to be resolved by the end of week. 29 | * ONNX-Tensorflow team has tried 1.8 candidate with success, excepting unable to parse the non-numeric version number. Shouldn't be a problem with formal release. 30 | * The resource build is just a gz file. The instruction to pip install from TestPypi is updated to make it clearer. 31 | * We will consider to have a postmorten meeting to share findings and discuss what can be done better for release 1.9. Jacky will annource it in slack after releaes 1.8 is out. 32 | * Any additional work items for this working group? We are almost done with 1.8 and the working group immediate mission is complete therefore might be closed. As far as how to do next release without release working group? Unsually it is handled by the release manager from a rotating company. Should double check with TSC. 33 | 34 | ## Action items 35 | -------------------------------------------------------------------------------- /release/meetings/012-20201111.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Wednesday Nov 11, 2020 at 10:00am-10:30am PST 4 | 5 | ## Agenda 6 | * Discussions 7 | * Release verification 8 | * Automation projects 9 | * Release 1.8 10 | 11 | ## Attendees 12 | * Chin Huang (IBM) 13 | * Jacky Chun-Wei Chen (Microsoft) 14 | * Anna June (VMWare) 15 | 16 | ## Notes 17 | * Review the updated release process doc 18 | * The release verification process is documented in a new PR (https://github.com/onnx/onnx/pull/3102) 19 | * Recommend to make the verification script between ONNX and ONNX Runtime available in ONNX repo so next release team can easily access it 20 | * Recommend to open an issue to track the verification of converters so it is transparent 21 | * A couple of release automation projects need some updates 22 | * wheel builder is updated for release 1.8 23 | * Release 1.8 24 | * Available in Pypi and conda on 11/04/2020 25 | * Should we conclude and close the release working group since 1.8 is out and the release instructions documented? 26 | * Consider adding to-dos to the ONNX release doc 27 | * Connect with LF AI to make the release annoucement there 28 | * Recommend ONNX-Tensorflow to handle non numeric version numbers such as 1.8.0rc since the packages in TestPypi will have 'rc' in their version number 29 | * Consider to have a postmorten meeting to share findings and discuss what can be done better for release 1.9 30 | 31 | ## Action items 32 | -------------------------------------------------------------------------------- /safety-related-profile/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Safety-Related Profile Working Group (WG) 4 | 5 | This repository is where ONNX Safety-Related Profile WG will capture various artifacts and deliverables. 6 | 7 | The purpose of this WG is to elaborate an ONNX profile dedicated to safety-related systems. 8 | 9 | An ONNX model is the representation of an ML/AI model to be implemented. In other words, it is the specification for the implementation activity. 10 | The model may be either interpreted by a tool or translated into some lower level equivalent representation (e.g., some source code) by a tool or a human. In both cases, to be able to interpret the model according to the intent of its designer, its syntax and semantics must be clear and non-ambiguous. 11 | This requirement is applicable to any system, but it is critical for systems for which a failure may have a critical business or safety impact. 12 | This is for instance the case in the aeronautical domain for which evidences shall be provided to show that the model semantics is actually preserved throughout the implementation phase (see ARP 6983, "Process Standard for Development and Certification/Approval of Aeronautical Safety-related Products implementing AI"). 13 | We consider that the current ONNX standard does not fully satisfy these requirements and that there is a need (i) to clarify the industrial needs in that matter, (ii) to identify and address the weaknesses of the current standard in a systematic manner in order to produce an "safety-related ONNX profile" that would to fulfil these needs. 14 | We also consider that these needs are specific and that the proposed changes and clarifications of the syntax, semantics, and documentation shall not prevent the use of ONNX in domains where constraints are relaxed. For instance, introducing restrictions on the parameters values for operators of the safety-related profile must not affect the usage of the same operators out of the profile. 15 | 16 | ## Working Group Status 17 | **ACTIVE** 18 | 19 | # Slack channel 20 | Please sign up at https://slack.lfai.foundation/ and join [onnx-safety-related-profile-wg](https://app.slack.com/client/TPUCV58TG/CPS6Y3N21) channel. 21 | 22 | # WG Lead(s) 23 | 24 | * Eric JENN (IRT Saint-Exupery) and Jean SOUYRIS (Airbus) (July 22, 2024 - Current) 25 | 26 | # Logistics 27 | 28 | * WG leads will drive the meeting. 29 | * Meeting annoucement will be posted in our Slack channel: https://lfaifoundation.slack.com/archives/C02AANGFBJB 30 | * Feedbacks and topic request are welcome by all. 31 | * Documents and artifacts: https://github.com/onnx/working-groups/tree/main/safety-related-profile 32 | 33 | # WG Meeting Info 34 | 35 | * Meeting (to be defined). 36 | * TEAMS Meeting link: (to be defined) 37 | * Meeting ID: (to be defined) 38 | 39 | # Meeting notes 40 | 41 | The meeting notes can be found [here](https://github.com/onnx/working-groups/tree/main/safety-related-profile/meetings) 42 | -------------------------------------------------------------------------------- /safety-related-profile/deliverables/README.md: -------------------------------------------------------------------------------- 1 | All deliverables: 2 | (none as of Sept. 2024) 3 | -------------------------------------------------------------------------------- /safety-related-profile/documents/Attic/ONNX_Op_classification.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/Attic/ONNX_Op_classification.xlsx -------------------------------------------------------------------------------- /safety-related-profile/documents/Attic/ONNX_format_comments_v0.1.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/Attic/ONNX_format_comments_v0.1.docx -------------------------------------------------------------------------------- /safety-related-profile/documents/Attic/spec_conventions: -------------------------------------------------------------------------------- 1 | # Operator specification conventions 2 | 3 | **This is a DRAFT** to be completed. 4 | 5 | ## Specification principles 6 | 7 | The specification follows the following *principles*. 8 | 9 | 1. We are only concerned with the syntactic and semantics elements 10 | concerning the **inference**. All ONNX constructs dealing with 11 | training are out of the scope of the Safety-Related Standard. 12 | 13 | 2. We distinguish an *informal* and a *formal* specification. The 14 | *informal* part is aimed at facilitating the understanding of the 15 | ONNX construct (e.g., operator). It may be incomplete, as far as it 16 | is clearly indicated. Conversely, the *formal* specification shall 17 | be absolutely complete and non ambiguous. 18 | 19 | 3. Using mathematical formalism shall be avoided if not required. Since 20 | we are essentially[^1] targeting data and computer scientists, using 21 | a very cryptic – yet perfectly well defined and mathematically 22 | grounded – notation may reveal being error prone and, consequently, 23 | counter effective. 24 | 25 | 4. The specification can rely on a formal language, as far as this 26 | language does not violate rule (2) above. 27 | -------------------------------------------------------------------------------- /safety-related-profile/documents/README.md: -------------------------------------------------------------------------------- 1 | This folder contains the working documents produced by the workgroup: 2 | - Our [Statement of Work](sow.md) 3 | - An example of [operator specification (CONV)](./conv_specification_example/README.md) (this is a work in progress) 4 | - Analysis of ONNX 5 | - Analysis results (synthesis) can be found in [issues.md](./issues.md) 6 | (using part of [elements of analysis of ONNX](ONNX_format_comments_v0.1.docx) 7 | -------------------------------------------------------------------------------- /safety-related-profile/documents/analysis_of_standards/SONNX_requirements_draft1.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/analysis_of_standards/SONNX_requirements_draft1.docx -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/README.md: -------------------------------------------------------------------------------- 1 | Images used in the CONV specification example 2 | -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/autopad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/autopad.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv-dep-3-channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv-dep-3-channels.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv-dep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv-dep.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv-std-3 channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv-std-3 channels.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv-std-3-channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv-std-3-channels.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv-std.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv-std.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv_pad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv_pad.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/conv_stride.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/conv_stride.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/dilation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/dilation.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/grouped_convolution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/grouped_convolution.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/imgs/why3_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/imgs/why3_workflow.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/reviews/README.md: -------------------------------------------------------------------------------- 1 | This directory contains the reviews of the **CONV2D operator** description. 2 | 3 | | Id | Who | 4 | |----------|---------------------------| 5 | |Review #1 | [Sebastian](sebastian.md) | 6 | |Review #2 | [Jean-Loup](jean-loup.md) | 7 | -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/reviews/imgs/README.md: -------------------------------------------------------------------------------- 1 | Images used in reviews. 2 | -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/reviews/imgs/jl-img1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/reviews/imgs/jl-img1.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/reviews/imgs/jl-img2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/conv_specification_example/reviews/imgs/jl-img2.png -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/reviews/jean-loup.md: -------------------------------------------------------------------------------- 1 | # Jean-Loup 2 | - Convention section: I think that "i.e. the number of lines (resp. columns)" should be added to the notation "X.H" 3 | - Restrictions subsection: I do not understand "with the tag". To which tag it refers? 4 | - Informal specification subsubsection: I think that the use of "." for multiplication in the formula should be avoided because "." is also used for tensor characteristic in the same formula, cf. "W.H". \times, i.e., "x" should be used instead. For instance "m . strides[0]" should be "m x strides[0]". 5 | - Inputs and outputs subsubsection: I think the the constraint X.C = Y.C is questionable. Indeed, to my understanding convolution is often used to transform a large image with 3 channels (R,G,B) to smaller image with a large number of channels, cf. image below ![attached image](imgs/jl-img1.png). 6 | - Attributes subsubsection: strides is not an integer, but an integer[2] or a list of integers. 7 | - Attributes subsubsection: I am surprised by the inconsistency between the exemple "stride[0]=1, stride[1]=2" and the figure. 8 | - Attributes subsubsection: I think that "is moved" is not appropriate. I suggest "Stride attribute determines the horizontal and vertical distance between kernel applying. For instance stride[0] = 2 and stride[1] = 3 indicates that closest kernel applying are separated by 2 columns for the horizontal dimension or by 3 lines for the vertical dimension." 9 | - Attributes subsubsection: I think there are formatting issues in the auto_pad paragraph. See below ![second attached image](imgs/jl-img2.png). 10 | 11 | -------------------------------------------------------------------------------- /safety-related-profile/documents/conv_specification_example/reviews/sebastian.md: -------------------------------------------------------------------------------- 1 | Sebastian's comment can be found in his [modified document](https://github.com/SebastianBoblest/working-groups/tree/lookAtConvSpec/safety-related-profile/documents/conv_specification_example) or in this [pull request ](https://github.com/ericjenn/working-groups/pull/19). 2 | -------------------------------------------------------------------------------- /safety-related-profile/documents/issues.md: -------------------------------------------------------------------------------- 1 | File moved [here](../deliverables/issues/issues.md). -------------------------------------------------------------------------------- /safety-related-profile/documents/mngt/README.md: -------------------------------------------------------------------------------- 1 | All files concerning the workgroup management: 2 | - Initial workplan in Excel format 3 | -------------------------------------------------------------------------------- /safety-related-profile/documents/mngt/WG_objectives_and_organisation.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/mngt/WG_objectives_and_organisation.docx -------------------------------------------------------------------------------- /safety-related-profile/documents/mngt/Workplan.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/mngt/Workplan.xlsx -------------------------------------------------------------------------------- /safety-related-profile/documents/needs.md: -------------------------------------------------------------------------------- 1 | File moved [here](../deliverables/needs/needs.md). -------------------------------------------------------------------------------- /safety-related-profile/documents/procedures/ModifGithubONNX.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/procedures/ModifGithubONNX.pdf -------------------------------------------------------------------------------- /safety-related-profile/documents/procedures/collaborative_docs.md: -------------------------------------------------------------------------------- 1 | 1. Working on documents where only one person is involved: 2 | - When a person is working alone on a document, they are invited to submit a "pull request" on GitHub. Once this request is made, Eric Jenn will be responsible for validating it. 3 | 3. Collaborative work on automatically mergeable documents with formats such as (e.g., .txt, markdown, read.me...) containing simple content or with a simple defined structure: 4 | - In the case where multiple collaborators work on a mergeable document, each individual is able to submit their own "pull request". Due to the simple nature of the document, the merging process is considered trivial. Therefore, Eric Jenn will be responsible for merging and validating the "pull requests". This case only applies if the document is considered simple to merge. 5 | 4. Work on Non-Mergeable Documents (e.g., docx, ppt, excel) or complex documents requiring coordination: 6 | - These documents will be identified, and for this type of non-mergeable document or if they require complex formatting and synchronization, a document manager will be designated. The manager will be responsible for defining the structure of the document and sending this document outline to each contributor via email (with the stakeholders of each section identified). The contributors will submit their results to the document manager via email. The document manager will aggregate the information by merging the different parts; They will also be responsible for publishing the document on GitHub at a frequency they deem appropriate, even if the document is not complete. In order to avoid any complex merges carried out by Eric, they will be the only one to submit the "pull request". 7 | -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_graph/graph.md: -------------------------------------------------------------------------------- 1 | (Specification of the graph semantics and constraints. *To be completed*) -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/README.md: -------------------------------------------------------------------------------- 1 | Images used in the CONV specification example 2 | -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/autopad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/autopad.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv-dep-3-channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv-dep-3-channels.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv-dep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv-dep.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv-std-3 channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv-std-3 channels.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv-std-3-channels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv-std-3-channels.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv-std.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv-std.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv_pad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv_pad.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/conv_stride.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/conv_stride.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/dilation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/dilation.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/grouped_convolution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/grouped_convolution.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/conv/imgs/why3_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/conv/imgs/why3_workflow.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/where/imgs/ONNX_WHERE.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/documents/profile_opset/where/imgs/ONNX_WHERE.png -------------------------------------------------------------------------------- /safety-related-profile/documents/profile_opset/where/imgs/README.md: -------------------------------------------------------------------------------- 1 | Images for the WHERE operator 2 | -------------------------------------------------------------------------------- /safety-related-profile/documents/reqs.md: -------------------------------------------------------------------------------- 1 | File moved [here](../deliverables/reqs/reqs.md). -------------------------------------------------------------------------------- /safety-related-profile/documents/usecases.md: -------------------------------------------------------------------------------- 1 | File moved [here](../deliverables/scope/scope.md). -------------------------------------------------------------------------------- /safety-related-profile/meetings/Other_meetings/2024-11 - ISCLP meeting - SONNX.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/Other_meetings/2024-11 - ISCLP meeting - SONNX.pptx -------------------------------------------------------------------------------- /safety-related-profile/meetings/README.md: -------------------------------------------------------------------------------- 1 | Location of the SRP meeting minutes. 2 | 3 | - Proposals for presentations during the WG meetings shall be done in [presentation proposals](https://github.com/ericjenn/working-groups/blob/3f64a7b8b5dc4d9214f3d69e4e3b7ec1523b56cc/safety-related-profile/meetings/presentation_proposals.md) 4 | 5 | 6 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/analysis_sub_wg/README.md: -------------------------------------------------------------------------------- 1 | This is the working area for the ONNX analysis sub-wg. 2 | Results of this activities can be found in file [issues.md](../../documents/issues.md). 3 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/analysis_sub_wg/minutes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/analysis_sub_wg/minutes.md -------------------------------------------------------------------------------- /safety-related-profile/meetings/analysis_sub_wg/slides-18-12.md: -------------------------------------------------------------------------------- 1 | --- 2 | marp: true 3 | footer: '_2024-12-18 SONNX WG meeting_ - Eric' 4 | --- 5 | # SONNX 6 | ## ONNX analysis sub-wg #1 7 | 8 | --- 9 | # **Objectives** 10 | - 11 | --- 12 | # **Method** 13 | --- 14 | # **Participants** 15 | --- 16 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/errors/2024-11-06 - Numerical issues.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/errors/2024-11-06 - Numerical issues.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/errors/README.md: -------------------------------------------------------------------------------- 1 | This directory contains minutes of discussions dedicated to the question of numerical accuracy, specification of FP operators, etc. : 2 | - A [few elements](01_what_is_the_issue.md) about the issue 3 | - The [slides](slides-06-11.md) presented on 11/06 WG meeting (Eric) 4 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/errors/images/ada_accuracy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/errors/images/ada_accuracy.png -------------------------------------------------------------------------------- /safety-related-profile/meetings/errors/slides-06-11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/errors/slides-06-11.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/formal_methods/slides-29-11.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/formal_methods/slides-29-11.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/general/2024-06-28 - MEET-UP/280624_ONNX_MeetUp.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/general/2024-06-28 - MEET-UP/280624_ONNX_MeetUp.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/general/2024-06-28 - MEET-UP/README.md: -------------------------------------------------------------------------------- 1 | Slides presented during the ONNX June 2024 meet-up 2 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/general/2024-09-25 - KOM/2024-09-25 - SONNX KOM.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/general/2024-09-25 - KOM/2024-09-25 - SONNX KOM.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/general/README.md: -------------------------------------------------------------------------------- 1 | This directory contains high-level presentations about the WG. 2 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/general/slides/2024-12-04-modifiedconv2Dinformalspec.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/general/slides/2024-12-04-modifiedconv2Dinformalspec.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/general/slides/README.md: -------------------------------------------------------------------------------- 1 | This directory contains various presentations done during our periodic meetings: 2 | - 2024/12/04 [Reviews and Updated Informal Specification for Conv2D](2024-12-04-modifiedconv2Dinformalspec.pdf) 3 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/operator_spec_sub_wg/README.md: -------------------------------------------------------------------------------- 1 | This directory contains the material used/produced by the operator specification sub working group. -------------------------------------------------------------------------------- /safety-related-profile/meetings/operator_spec_sub_wg/SONNX_Operator_List.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/operator_spec_sub_wg/SONNX_Operator_List.xlsx -------------------------------------------------------------------------------- /safety-related-profile/meetings/presentation_proposals.md: -------------------------------------------------------------------------------- 1 | # Proposals 2 | | Subject | Who | When (proposed) | When (actual) | 3 | |-------------------------------------|---------------------------|------------------|---------------| 4 | | Code generation for neural networks | Sebastian Boblest (Bosch) | 2024/11 | 2024/12/18 | 5 | | ONNX MLIR | Alexandre Eichenberger (IBM) | 2024/10 | 2025/02/12 | 6 | | Evaluation and improvement of SW verifiers on FP Neural Networks | Edoardo Manino (Manchester U) | 2024/11 | 202(/01/15) | 7 | 8 | 9 | 10 | # Meetings 11 | - 2024/11/20 12 | - 2024/12/04 13 | - 2024/12/18 : Sebastian 14 | - 2025/01/15 : Edoardo 15 | - 2025/01/29 16 | - 2025/02/12 : Alexandre 17 | - 2025/02/26 18 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/reqs_sub_wg/README.md: -------------------------------------------------------------------------------- 1 | This is the work area for the "requirement definition sub workgroup": 2 | - list of operators needed to cover the [Use Cases](../../documents/usecases.md), in this [Excel sheet](./ONNX_operators_for_Use_Cases.xlsx). 3 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/reqs_sub_wg/minutes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/reqs_sub_wg/minutes.md -------------------------------------------------------------------------------- /safety-related-profile/meetings/reqs_sub_wg/slides-18-12.md: -------------------------------------------------------------------------------- 1 | --- 2 | marp: true 3 | footer: '_2024-12-18 SONNX WG meeting_ - Eric' 4 | --- 5 | # SONNX 6 | ## REQs definition sub-wg #1 7 | 8 | --- 9 | # **Objectives** 10 | --- 11 | # **Method** 12 | --- 13 | # **Participants** 14 | --- 15 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/slides/2024-12-18-Embedded_AI_Coder_Bosch_Boblest.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/slides/2024-12-18-Embedded_AI_Coder_Bosch_Boblest.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/slides/2025-01-15-Edoardo_Manino_SONNX_slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/meetings/slides/2025-01-15-Edoardo_Manino_SONNX_slides.pdf -------------------------------------------------------------------------------- /safety-related-profile/meetings/slides/README.md: -------------------------------------------------------------------------------- 1 | Slides presented during our periodic meetings. 2 | - [Slides presented by Sebastian on Bosch's code generatuoin tool](./2024-12-18-Embedded_AI_Coder_Bosch_Boblest.pdf) 3 | - [Slides presented by Edoardo on bit-precise neural network verification](./2025-01-15-Edoardo_Manino_SONNX_slides.pdf) 4 | -------------------------------------------------------------------------------- /safety-related-profile/meetings/team.md: -------------------------------------------------------------------------------- 1 | | Last name | First name | COMPANY | Activity | Contributions (R/W) | 2 | |-----------|------------|---------|---------------|------------------------| 3 | | BARRETEAU | Michel | Thales | 4 | | BELEGUER ALMECIJA | Salvador | Airbus DS | 5 | | BELFY | Henri | THALES AVS | Relative ONNX embedded part (Operators Definitions...) | R/W (limited W)| 6 | | BELKACEM | Thiziri | Airbus Protect | (W) D1.b.x / D2.a / D4.a ; (R) D1.a / D1.c / D3.x / D5.x | R/W | 7 | | BERJON | Pierre | Airbus Operations SAS | 8 | | BEYER | Michael | BOSCH | 9 | | BOBLEST | Sebastian | BOSCH | | R/(W) final contribution clearance in progress | 10 | | BRIAND | Julien | DGA-TA | 11 | | CHICHIN | Sergei | Airbus Operations SAS | 12 | | COMBES | Yohann | Sopra-stéria | 13 | | CORRENSON | Loïc | CEA | 14 | | DE GRANCEY | Florence | THALES AVS | 15 | | DE MORAIS PINHAO | Paula | Embraer | 16 | | DMITRIEV | Konstantin | TUM | 17 | | DUFOUR | Jean-Louis | Safran Electronics and Defense | 18 | | EVANS | Adrian | CEA | 19 | | FARGES | Jean-Loup | ONERA | 20 | | FEHLNER | Andreas | TRUMPF | 21 | | FERREIRA | Oseias | Embraer | 22 | | FERREIRA DE OLIVEIRA | Helen | Embraer | 23 | | GABREAU | Christophe | Airbus Operations SAS | 24 | | GARION | Christophe | ISAE | Formal specification of ONNX operators | R/W (limited W) | 25 | | GIRARD | Julien | CEA | 26 | | HARNOUFI | Bouchra | Airbus DS | 27 | | JENN | Eric | IRT St Ex | All | WG co-lead 28 | | JIMENEZ | Arnaud | CS-SOPRASTERIA | 29 | | KLUGE | Regina | INFINEON | 30 | | KRISHNA | Ram | | 31 | | LAVAL | Damien | Naval Group | 32 | | LEMESLE | Augustin | CEA | 33 | | LIU | Cong | Collins Aerospace| 34 | | MANINO | Edoardo | Manchester U | | | 35 | | MARABOTTO | Christophe | IRT Saint-Exupéry | 36 | | MARANDAS | David-Gilles | Airbus Operations SAS | 37 | | MARCADE | Jean-Paul | Mathworks | 38 | | ELO | Marko | CROSSCONTROL | 39 | | MAXIM | Cristian | IRT SystemX | 40 | | MENDONCA | Guiliano | Embraer | 41 | | MESSNER | Jonas | BOSCH | 42 | | OLIVIO | Paula | Embraer | 43 | | PAGETTI | Claire | ONERA | 44 | | PONTES | Fabricio | Embraer | 45 | | POPESCU | Francesca | Airbus Protect | Stay informed 46 | | POTOP-BUTUCARU | Dumitru | INRIA | Strong interest in execution model and control primitives of ONNX 47 | | REVERDY | Céline | THALES R&T | 48 | | RIBAS de AMARAL | Janaina | Airbus DS | 49 | | ROUFFET | Jean-Baptiste | Airbus Protect | (W) D1.b.x / D2.a / D4.a ; (R) D1.a / D1.c / D3.x / D5.x | R/W | 50 | | SADOCK | Christiano | Embraer | 51 | | SCHMIDT | Bernard | BOSCH | 52 | | SCHORN | Christoph | BOSCH | 53 | | SOUYRIS | Jean | Airbus Operations SAS | All | WG Co-lead 54 | | THIERION | Philippe | AMPERE | 55 | | TOSTES | Pedro | Embraer | 56 | | TURKI | Mariem | IRT St Ex | 57 | | VALOT | Nicolas | Airbus Helicopters | 58 | | VIDALIE | Julien | Airbus Protect | (W) D1.a / D1.b.x / D1.c ; (R) D3.a / D5.a | R/W | 59 | | VILLANI | Anaisa | Embraer | 60 | -------------------------------------------------------------------------------- /safety-related-profile/tools/LeNet5.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/safety-related-profile/tools/LeNet5.onnx -------------------------------------------------------------------------------- /safety-related-profile/tools/README.md: -------------------------------------------------------------------------------- 1 | # TOOLS and UTILITIES 2 | 3 | This directory contains some tools and utilities developed by the WG to support its activities. 4 | 5 | ## Prerequisites 6 | 7 | install dependencies 8 | 9 | `pip install -r requirements.txt` 10 | 11 | ## Run example for LeNet5 12 | 13 | `inv depend` 14 | 15 | Expected result for LeNet5: 16 | 17 | ``` 18 | onnx.__version__='1.16.1', opset=21, IR_VERSION=10 19 | 20 | Operator dependencies: 21 | AveragePool v19 22 | Conv v11 23 | Gemm v13 24 | Reshape v21 25 | Softmax v13 26 | Tanh v13 27 | ``` 28 | 29 | ## [onnx_depend.py](onnx_depend.py) 30 | 31 | The script prints the onnx Nodes' Opererator dependencies for opset 'ai.onnx'. 32 | 33 | `python -m onnx_depend --model_path .onnx` -------------------------------------------------------------------------------- /safety-related-profile/tools/onnx_depend.py: -------------------------------------------------------------------------------- 1 | """ 2 | ******************************************************************************* 3 | * ACETONE: Predictable programming framework for ML applications in safety-critical systems 4 | * Copyright (c) 2024. ONERA 5 | * Copyright (c) 2024. AIRBUS 6 | * This file is part of ACETONE 7 | * 8 | * ACETONE is free software ; 9 | * you can redistribute it and/or modify it under the terms of the GNU Lesser General Public 10 | * License as published by the Free Software Foundation ; 11 | * either version 3 of the License, or (at your option) any later version. 12 | * 13 | * ACETONE is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY ; 14 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 15 | * See the GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License along with this program ; 18 | * if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 | ****************************************************************************** 20 | """ 21 | import onnx 22 | import argparse 23 | from onnx import ( 24 | defs, 25 | IR_VERSION, 26 | __version__ 27 | ) 28 | from onnx.defs import onnx_opset_version 29 | 30 | if __name__ == "__main__": 31 | print(f"onnx.__version__={__version__!r}, opset={onnx_opset_version()}, IR_VERSION={IR_VERSION}") 32 | parser = argparse.ArgumentParser(description='print/modify onnx model strings') 33 | parser.add_argument('--model_path', required=True,help='onnx model path') 34 | args = parser.parse_args() 35 | 36 | onnx_model = onnx.load(args.model_path) 37 | onnx.checker.check_model(onnx_model) 38 | print('\nOperator dependencies:') 39 | local_func = [f.name for f in onnx_model.functions] 40 | op = {} 41 | for typ in onnx_model.graph.node: 42 | if typ.op_type not in local_func: 43 | #collect the onnx operator used in the model nodes except local function names 44 | op[typ.op_type] = defs.get_schema(typ.op_type,'').since_version 45 | for f in onnx_model.functions: 46 | for typ in f.node: 47 | if typ.op_type not in local_func: 48 | #collect the onnx operator used in the local functions except other local function names 49 | op[typ.op_type] = defs.get_schema(typ.op_type,'').since_version 50 | 51 | sops = [f'{x}\t\tv{op[x]}' for x in sorted(list(op.keys()))] 52 | print ('\n'.join(sops)) 53 | -------------------------------------------------------------------------------- /safety-related-profile/tools/requirements.txt: -------------------------------------------------------------------------------- 1 | onnx==1.16.1 2 | invoke==2.2.0 3 | -------------------------------------------------------------------------------- /safety-related-profile/tools/tasks.py: -------------------------------------------------------------------------------- 1 | from invoke import task 2 | 3 | @task 4 | def depend(c, docs=False): 5 | c.run("python -m onnx_depend --model_path LeNet5.onnx") 6 | -------------------------------------------------------------------------------- /training/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ONNX Training Working Group 4 | 5 | This is artifacts repository where ONNX Training working group will capture various artifacts and deliverables. 6 | Structure of the space will evolve over time. 7 | 8 | ## Working Group Status 9 | **ACTIVE** 10 | 11 | ## Slack channel 12 | **Please sign up at https://slack.lfai.foundation/ and join [onnx-training](https://lfaifoundation.slack.com/archives/C018K560U14) channel.** 13 | 14 | ## Contributors 15 | *Note: Contributors list will be updated as per participation and contributions.* 16 | * Svetlana Levitan (IBM) (chair) 17 | * Wei-Sheng Chin (Microsoft) 18 | * Chin Huang (IBM) 19 | * 20 | 21 | ## Meeting logistics 22 | * WG chair will drive and facilitate the meetings 23 | * Publish agenda, produce meeting notes, action items, etc. 24 | * Meetings are bi-weekly 25 | * ***Tuesdays 10:30am PST*** 26 | * Will post a message to [ONNX Training Gitter Channel](https://gitter.im/onnx/training) with meeting invite link 27 | * Up to 7 days before the meeting by WG chair, but proposals to agenda item updates are welcomed from all contributors via gitter 28 | * Meeting/Telepresence tool: Webex 29 | * Meetings will be recorded and the recordings posted to YouTube, with links here: 30 | 31 | * Recording of July 23, 2019 meeting: https://youtu.be/uProIFfvQ7M 32 | * Recording of September 3, 2019 meeting: https://youtu.be/8VCbSbJHODA 33 | 34 | ## Discussion mechanics 35 | * [ONNX Training WG Gitter Channel](https://gitter.im/onnx/training) 36 | * [ONNX Training WG repo](https://github.com/onnx/working-groups/training) 37 | * Document reviews 38 | * Will use Google doc for draft reviews until we get to final doc proposal, at which point it will get transcribed into .md and committed to [ONNX Training WG repo](https://github.com/onnx/working-groups/training). 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /training/workshops/August23_2019/TrainingReadout.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/training/workshops/August23_2019/TrainingReadout.pdf -------------------------------------------------------------------------------- /training/workshops/August23_2019/TrainingStory.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/training/workshops/August23_2019/TrainingStory.pdf -------------------------------------------------------------------------------- /training/workshops/August23_2019/TrainingUpdate.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/training/workshops/August23_2019/TrainingUpdate.pdf -------------------------------------------------------------------------------- /training/workshops/August23_2019/TrainingWorkshopStart.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/training/workshops/August23_2019/TrainingWorkshopStart.pdf -------------------------------------------------------------------------------- /training/workshops/August23_2019/onnx_2_tf_test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onnx/working-groups/bcd61cf1aee0f20fdf9629d28e5202599410be61/training/workshops/August23_2019/onnx_2_tf_test.pdf --------------------------------------------------------------------------------