├── LICENSE ├── README.md ├── include ├── onnxruntime_c_api.h └── onnxruntime_c_api_expanded.h ├── lib └── libonnxruntime.so.1.6.0 ├── onnxruntime.nimble ├── sample ├── C_Api_Sample.nim └── squeezenet.onnx └── src └── onnxruntime.nim /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 YesDrX 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # onnxruntime-nim 2 | [onnxruntime C Api](https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_c_api.h) wrapped for nim 3 | 4 | * Wrapped C Api is generated using [c2nim](https://github.com/nim-lang/c2nim) 5 | * [Onnxruntime Home Page](https://www.onnxruntime.ai/) 6 | * [Onnxruntime Github](https://github.com/microsoft/onnxruntime) 7 | * Header file [onnxruntime_c_api.h](https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_c_api.h) 8 | 9 | ## Install 10 | 11 | ### 1. Install Onnxruntime 12 | * Go to [Onnxruntime Releases](https://github.com/microsoft/onnxruntime/releases/tag/v1.6.0) 13 | * Choose source code/compiled binaries for your system, such as the one for [Linux X86_64 CPU 1.6.0](https://github.com/microsoft/onnxruntime/releases/download/v1.6.0/onnxruntime-linux-x64-1.6.0.tgz) 14 | * Unzip the file, copy ./include and ./lib to the right places 15 | 16 | For example, on Linux, you may copy ./lib to /usr/lib/x86_64-linux-gn/, and ./include/ to /usr/include/ 17 | Or just put ./include ./lib into your project; as long as you and your code know where they are. 18 | 19 | ### 2. Install Onnxruntime C Api for nim 20 | ```nim 21 | git clone https://github.com/YesDrX/onnxruntime-nim 22 | cd onnxruntime-nim 23 | nimble install 24 | ``` 25 | ``` 26 | nimble install onnxruntime 27 | ``` 28 | 29 | ### 3. Sample Code 30 | * [C_Api_Sample.nim](https://github.com/YesDrX/onnxruntime-nim/blob/main/sample/C_Api_Sample.nim) is a direct translation from [C_Api_Sample.cpp](https://github.com/microsoft/onnxruntime/blob/master/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp) 31 | * Assume you have both Onnxruntime library and Onnxruntime-nim installed 32 | ```nim 33 | cd ./sample 34 | nim c --run C_Api_Sample.nim 35 | ``` 36 | * Output 37 | ``` 38 | Using Onnxruntime C Api : 1.6.0 39 | WARNING: Since openmp is enabled in this build, this API cannot be used to configure intra op num threads. Please use the openmp environment variables to control the number of threads. 40 | Using Onnxruntime C API 41 | Number of inputs = 1 42 | Input 0 : name= data_0 43 | Input 0 : type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT 44 | Input 0 : num_dims = 4 45 | Input 0 : dim 0 = 1 46 | Input 0 : dim 1 = 3 47 | Input 0 : dim 2 = 224 48 | Input 0 : dim 3 = 224 49 | [Class 0] : 0.000045 50 | [Class 1] : 0.003846 51 | [Class 2] : 0.000125 52 | [Class 3] : 0.001180 53 | [Class 4] : 0.001317 54 | Done! 55 | ``` 56 | 57 | ``` 58 | -------------------------------------------------------------------------------- /include/onnxruntime_c_api.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | #pragma once 5 | #include 6 | #include 7 | #include 8 | 9 | // This value is used in structures passed to ORT so that a newer version of ORT will still work with them 10 | #define ORT_API_VERSION 6 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | // SAL2 Definitions 17 | #ifndef _WIN32 18 | #define _In_ 19 | #define _In_z_ 20 | #define _In_opt_ 21 | #define _In_opt_z_ 22 | #define _Out_ 23 | #define _Outptr_ 24 | #define _Out_opt_ 25 | #define _Inout_ 26 | #define _Inout_opt_ 27 | #define _Frees_ptr_opt_ 28 | #define _Ret_maybenull_ 29 | #define _Ret_notnull_ 30 | #define _Check_return_ 31 | #define _Outptr_result_maybenull_ 32 | #define _In_reads_(X) 33 | #define _Inout_updates_all_(X) 34 | #define _Out_writes_bytes_all_(X) 35 | #define _Out_writes_all_(X) 36 | #define _Success_(X) 37 | #define _Outptr_result_buffer_maybenull_(X) 38 | #define ORT_ALL_ARGS_NONNULL __attribute__((nonnull)) 39 | #else 40 | #include 41 | #define ORT_ALL_ARGS_NONNULL 42 | #endif 43 | 44 | #ifdef _WIN32 45 | // Define ORT_DLL_IMPORT if your program is dynamically linked to Ort. 46 | // dllexport is not used, we use a .def file. 47 | #ifdef ORT_DLL_IMPORT 48 | #define ORT_EXPORT __declspec(dllimport) 49 | #else 50 | #define ORT_EXPORT 51 | #endif 52 | #define ORT_API_CALL _stdcall 53 | #define ORT_MUST_USE_RESULT 54 | #define ORTCHAR_T wchar_t 55 | #else 56 | #define ORT_EXPORT 57 | #define ORT_API_CALL 58 | #define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) 59 | #define ORTCHAR_T char 60 | #endif 61 | 62 | #ifndef ORT_TSTR 63 | #ifdef _WIN32 64 | #define ORT_TSTR(X) L##X 65 | #else 66 | #define ORT_TSTR(X) X 67 | #endif 68 | #endif 69 | 70 | // Any pointer marked with _In_ or _Out_, cannot be NULL. 71 | 72 | // Windows users should use unicode paths when possible to bypass the MAX_PATH limitation 73 | // Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. 74 | // for ReleaseXXX(...) functions, they can accept NULL pointer. 75 | 76 | #ifdef __cplusplus 77 | // For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept. 78 | // Such complex condition is needed because compilers set __cplusplus value differently. 79 | #ifndef __has_feature 80 | #define __has_feature(x) 0 81 | #endif 82 | #if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept))) 83 | #define NO_EXCEPTION noexcept 84 | #else 85 | #define NO_EXCEPTION throw() 86 | #endif 87 | #else 88 | #define NO_EXCEPTION 89 | #endif 90 | 91 | // Copied from TensorProto::DataType 92 | // Currently, Ort doesn't support complex64, complex128 93 | typedef enum ONNXTensorElementDataType { 94 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, 95 | ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float 96 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t 97 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t 98 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t 99 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t 100 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t 101 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t 102 | ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string 103 | ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, 104 | ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, 105 | ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double 106 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t 107 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t 108 | ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components 109 | ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components 110 | ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision 111 | } ONNXTensorElementDataType; 112 | 113 | // Synced with onnx TypeProto oneof 114 | typedef enum ONNXType { 115 | ONNX_TYPE_UNKNOWN, 116 | ONNX_TYPE_TENSOR, 117 | ONNX_TYPE_SEQUENCE, 118 | ONNX_TYPE_MAP, 119 | ONNX_TYPE_OPAQUE, 120 | ONNX_TYPE_SPARSETENSOR, 121 | } ONNXType; 122 | 123 | typedef enum OrtLoggingLevel { 124 | ORT_LOGGING_LEVEL_VERBOSE, 125 | ORT_LOGGING_LEVEL_INFO, 126 | ORT_LOGGING_LEVEL_WARNING, 127 | ORT_LOGGING_LEVEL_ERROR, 128 | ORT_LOGGING_LEVEL_FATAL, 129 | } OrtLoggingLevel; 130 | 131 | typedef enum OrtErrorCode { 132 | ORT_OK, 133 | ORT_FAIL, 134 | ORT_INVALID_ARGUMENT, 135 | ORT_NO_SUCHFILE, 136 | ORT_NO_MODEL, 137 | ORT_ENGINE_ERROR, 138 | ORT_RUNTIME_EXCEPTION, 139 | ORT_INVALID_PROTOBUF, 140 | ORT_MODEL_LOADED, 141 | ORT_NOT_IMPLEMENTED, 142 | ORT_INVALID_GRAPH, 143 | ORT_EP_FAIL, 144 | } OrtErrorCode; 145 | 146 | #define ORT_RUNTIME_CLASS(X) \ 147 | struct Ort##X; \ 148 | typedef struct Ort##X Ort##X; 149 | 150 | // The actual types defined have an Ort prefix 151 | ORT_RUNTIME_CLASS(Env); 152 | ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success 153 | ORT_RUNTIME_CLASS(MemoryInfo); 154 | ORT_RUNTIME_CLASS(IoBinding); 155 | ORT_RUNTIME_CLASS(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool) 156 | ORT_RUNTIME_CLASS(Value); 157 | ORT_RUNTIME_CLASS(RunOptions); 158 | ORT_RUNTIME_CLASS(TypeInfo); 159 | ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); 160 | ORT_RUNTIME_CLASS(SessionOptions); 161 | ORT_RUNTIME_CLASS(CustomOpDomain); 162 | ORT_RUNTIME_CLASS(MapTypeInfo); 163 | ORT_RUNTIME_CLASS(SequenceTypeInfo); 164 | ORT_RUNTIME_CLASS(ModelMetadata); 165 | ORT_RUNTIME_CLASS(ThreadPoolParams); 166 | ORT_RUNTIME_CLASS(ThreadingOptions); 167 | ORT_RUNTIME_CLASS(ArenaCfg); 168 | 169 | #ifdef _WIN32 170 | typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; 171 | #else 172 | typedef OrtStatus* OrtStatusPtr; 173 | #endif 174 | 175 | // __VA_ARGS__ on Windows and Linux are different 176 | #define ORT_API(RETURN_TYPE, NAME, ...) ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION 177 | 178 | #define ORT_API_STATUS(NAME, ...) \ 179 | ORT_EXPORT _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT 180 | 181 | // XXX: Unfortunately, SAL annotations are known to not work with function pointers 182 | #define ORT_API2_STATUS(NAME, ...) \ 183 | _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT 184 | 185 | // Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT 186 | #define ORT_API_STATUS_IMPL(NAME, ...) \ 187 | _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION 188 | 189 | #define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) 190 | 191 | // When passing in an allocator to any ORT function, be sure that the allocator object 192 | // is not destroyed until the last allocated object using it is freed. 193 | typedef struct OrtAllocator { 194 | uint32_t version; // Initialize to ORT_API_VERSION 195 | void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); 196 | void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); 197 | const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); 198 | } OrtAllocator; 199 | 200 | typedef void(ORT_API_CALL* OrtLoggingFunction)( 201 | void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, 202 | const char* message); 203 | 204 | // Set Graph optimization level. 205 | // Refer https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md 206 | // for in-depth undersrtanding of Graph Optimizations in ORT 207 | typedef enum GraphOptimizationLevel { 208 | ORT_DISABLE_ALL = 0, 209 | ORT_ENABLE_BASIC = 1, 210 | ORT_ENABLE_EXTENDED = 2, 211 | ORT_ENABLE_ALL = 99 212 | } GraphOptimizationLevel; 213 | 214 | typedef enum ExecutionMode { 215 | ORT_SEQUENTIAL = 0, 216 | ORT_PARALLEL = 1, 217 | } ExecutionMode; 218 | 219 | // Set the language projection, default is C, which means it will classify the language not in the list to C also. 220 | typedef enum OrtLanguageProjection { 221 | ORT_PROJECTION_C = 0, // default 222 | ORT_PROJECTION_CPLUSPLUS = 1, 223 | ORT_PROJECTION_CSHARP = 2, 224 | ORT_PROJECTION_PYTHON = 3, 225 | ORT_PROJECTION_JAVA = 4, 226 | ORT_PROJECTION_WINML = 5, 227 | ORT_PROJECTION_NODEJS = 6, 228 | } OrtLanguageProjection; 229 | 230 | struct OrtKernelInfo; 231 | typedef struct OrtKernelInfo OrtKernelInfo; 232 | struct OrtKernelContext; 233 | typedef struct OrtKernelContext OrtKernelContext; 234 | struct OrtCustomOp; 235 | typedef struct OrtCustomOp OrtCustomOp; 236 | 237 | typedef enum OrtAllocatorType { 238 | Invalid = -1, 239 | OrtDeviceAllocator = 0, 240 | OrtArenaAllocator = 1 241 | } OrtAllocatorType; 242 | 243 | /** 244 | * memory types for allocator, exec provider specific types should be extended in each provider 245 | * Whenever this struct is updated, please also update the MakeKey function in onnxruntime/core/framework/execution_provider.cc 246 | */ 247 | typedef enum OrtMemType { 248 | OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider 249 | OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED 250 | OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED 251 | OrtMemTypeDefault = 0, // the default allocator for execution provider 252 | } OrtMemType; 253 | 254 | typedef enum OrtCudnnConvAlgoSearch { 255 | EXHAUSTIVE, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx 256 | HEURISTIC, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7 257 | DEFAULT, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM 258 | } OrtCudnnConvAlgoSearch; 259 | 260 | /// 261 | /// Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA 262 | /// 263 | typedef struct OrtCUDAProviderOptions { 264 | int device_id; // cuda device with id=0 as default device. 265 | OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn conv algo search option 266 | size_t cuda_mem_limit; // default cuda memory limitation to maximum finite value of size_t. 267 | int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo. 268 | int do_copy_in_default_stream; 269 | } OrtCUDAProviderOptions; 270 | 271 | /// 272 | /// Options for the OpenVINO provider that are passed to SessionOptionsAppendExecutionProvider_OpenVINO 273 | /// 274 | typedef struct OrtOpenVINOProviderOptions { 275 | #ifdef __cplusplus 276 | OrtOpenVINOProviderOptions() : device_type{}, enable_vpu_fast_compile{}, device_id{}, num_of_threads{} {} 277 | #endif 278 | const char* device_type; // CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32 279 | unsigned char enable_vpu_fast_compile; // 0 = false, nonzero = true 280 | const char* device_id; 281 | size_t num_of_threads; // 0 uses default number of threads 282 | } OrtOpenVINOProviderOptions; 283 | 284 | struct OrtApi; 285 | typedef struct OrtApi OrtApi; 286 | 287 | struct OrtApiBase { 288 | const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; // Pass in ORT_API_VERSION 289 | // nullptr will be returned if the version is unsupported, for example when using a runtime older than this header file 290 | 291 | const char*(ORT_API_CALL* GetVersionString)() NO_EXCEPTION; 292 | }; 293 | typedef struct OrtApiBase OrtApiBase; 294 | 295 | ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION; 296 | 297 | struct OrtApi { 298 | /** 299 | * \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus 300 | */ 301 | OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; 302 | 303 | OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; 304 | 305 | /** 306 | * \param status must not be NULL 307 | * \return The error message inside the `status`. Do not free the returned value. 308 | */ 309 | const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; 310 | 311 | /** 312 | * \param out Should be freed by `OrtReleaseEnv` after use 313 | */ 314 | ORT_API2_STATUS(CreateEnv, OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out); 315 | 316 | /** 317 | * \param out Should be freed by `OrtReleaseEnv` after use 318 | */ 319 | ORT_API2_STATUS(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, 320 | OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out); 321 | 322 | // Platform telemetry events are on by default since they are lightweight. You can manually turn them off. 323 | ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env); 324 | ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env); 325 | 326 | // TODO: document the path separator convention? '/' vs '\' 327 | // TODO: should specify the access characteristics of model_path. Is this read only during the 328 | // execution of CreateSession, or does the OrtSession retain a handle to the file/directory 329 | // and continue to access throughout the OrtSession lifetime? 330 | // What sort of access is needed to model_path : read or read/write? 331 | ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, 332 | _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); 333 | 334 | ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length, 335 | _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); 336 | 337 | ORT_API2_STATUS(Run, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options, 338 | _In_reads_(input_len) const char* const* input_names, 339 | _In_reads_(input_len) const OrtValue* const* input, size_t input_len, 340 | _In_reads_(output_names_len) const char* const* output_names1, size_t output_names_len, 341 | _Inout_updates_all_(output_names_len) OrtValue** output); 342 | 343 | /** 344 | * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use 345 | */ 346 | ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options); 347 | 348 | // Set filepath to save optimized model after graph level transformations. 349 | ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, 350 | _In_ const ORTCHAR_T* optimized_model_filepath); 351 | 352 | // create a copy of an existing OrtSessionOptions 353 | ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options, 354 | _Outptr_ OrtSessionOptions** out_options); 355 | 356 | // Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model 357 | // has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance. 358 | // See [docs/ONNX_Runtime_Perf_Tuning.md] for more details. 359 | ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode); 360 | 361 | // Enable profiling for this session. 362 | ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); 363 | ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options); 364 | 365 | // Enable the memory pattern optimization. 366 | // The idea is if the input shapes are the same, we could trace the internal memory allocation 367 | // and generate a memory pattern for future request. So next time we could just do one allocation 368 | // with a big chunk for all the internal memory allocation. 369 | // Note: memory pattern optimization is only available when SequentialExecution enabled. 370 | ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options); 371 | ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options); 372 | 373 | // Enable the memory arena on CPU 374 | // Arena may pre-allocate memory for future usage. 375 | // set this option to false if you don't want it. 376 | ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options); 377 | ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options); 378 | 379 | // < logger id to use for session output 380 | ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid); 381 | 382 | // < applies to session load, initialization, etc 383 | ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); 384 | ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); 385 | 386 | ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, 387 | GraphOptimizationLevel graph_optimization_level); 388 | 389 | // Sets the number of threads used to parallelize the execution within nodes 390 | // A value of 0 means ORT will pick a default 391 | // Note: If you've built ORT with OpenMP, this API has no effect on the number of threads used. In this case 392 | // use the OpenMP env variables to configure the number of intra op num threads. 393 | ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads); 394 | 395 | // Sets the number of threads used to parallelize the execution of the graph (across nodes) 396 | // If sequential execution is enabled this value is ignored 397 | // A value of 0 means ORT will pick a default 398 | ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads); 399 | 400 | /* 401 | Create a custom op domain. After all sessions using it are released, call OrtReleaseCustomOpDomain 402 | */ 403 | ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out); 404 | 405 | /* 406 | * Add custom ops to the OrtCustomOpDomain 407 | * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released 408 | */ 409 | ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op); 410 | 411 | /* 412 | * Add a custom op domain to the OrtSessionOptions 413 | * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released 414 | */ 415 | ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain); 416 | 417 | /* 418 | * Loads a DLL named 'library_path' and looks for this entry point: 419 | * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); 420 | * It then passes in the provided session options to this function along with the api base. 421 | * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in 422 | * session options are destroyed, or if an error occurs and it is non null. 423 | */ 424 | ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, 425 | void** library_handle); 426 | 427 | /** 428 | * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these 429 | * functions to enable them in the session: 430 | * OrtSessionOptionsAppendExecutionProvider_CPU 431 | * OrtSessionOptionsAppendExecutionProvider_CUDA 432 | * OrtSessionOptionsAppendExecutionProvider_ 433 | * The order they are called indicates the preference order as well. In other words call this method 434 | * on your most preferred execution provider first followed by the less preferred ones. 435 | * If none are called Ort will use its internal CPU execution provider. 436 | */ 437 | 438 | ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); 439 | ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); 440 | ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out); 441 | 442 | /** 443 | * \param out should be freed by OrtReleaseTypeInfo after use 444 | */ 445 | ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info); 446 | 447 | /** 448 | * \param out should be freed by OrtReleaseTypeInfo after use 449 | */ 450 | ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, 451 | _Outptr_ OrtTypeInfo** type_info); 452 | 453 | /** 454 | * \param out should be freed by OrtReleaseTypeInfo after use 455 | */ 456 | ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index, 457 | _Outptr_ OrtTypeInfo** type_info); 458 | 459 | /** 460 | * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it. 461 | */ 462 | ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, 463 | _Outptr_ char** value); 464 | ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, 465 | _Outptr_ char** value); 466 | ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index, 467 | _Inout_ OrtAllocator* allocator, _Outptr_ char** value); 468 | 469 | /** 470 | * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseRunOptions after use 471 | */ 472 | ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out); 473 | 474 | ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value); 475 | ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value); 476 | ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions*, _In_ const char* run_tag); 477 | 478 | ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out); 479 | ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out); 480 | ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out); 481 | 482 | // Set a flag so that ALL incomplete OrtRun calls that are using this instance of OrtRunOptions 483 | // will exit as soon as possible. 484 | ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options); 485 | // Unset the terminate flag to enable this OrtRunOptions instance being used in new OrtRun calls. 486 | ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options); 487 | 488 | /** 489 | * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value 490 | * \param out Should be freed by calling OrtReleaseValue 491 | * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx 492 | */ 493 | ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len, 494 | ONNXTensorElementDataType type, _Outptr_ OrtValue** out); 495 | 496 | /** 497 | * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. 498 | * p_data is owned by caller. OrtReleaseValue won't release p_data. 499 | * \param out Should be freed by calling OrtReleaseValue 500 | */ 501 | ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, 502 | size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type, 503 | _Outptr_ OrtValue** out); 504 | 505 | /** 506 | * \Sets *out to 1 iff an OrtValue is a tensor, 0 otherwise 507 | */ 508 | ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out); 509 | 510 | // This function doesn't work with string tensor 511 | // this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd. 512 | ORT_API2_STATUS(GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** out); 513 | 514 | /** 515 | * \param value A tensor created from OrtCreateTensor... function. 516 | * \param s each A string array. Each string in this array must be null terminated. 517 | * \param s_len length of s 518 | */ 519 | ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len); 520 | 521 | /** 522 | * \param value A tensor created from OrtCreateTensor... function. 523 | * \param len total data length, not including the trailing '\0' chars. 524 | */ 525 | ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); 526 | 527 | /** 528 | * \param s string contents. Each string is NOT null-terminated. 529 | * \param value A tensor created from OrtCreateTensor... function. 530 | * \param s_len total data length, get it from OrtGetStringTensorDataLength 531 | */ 532 | ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s, 533 | size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len); 534 | 535 | /** 536 | * Don't free the 'out' value 537 | */ 538 | ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo*, 539 | _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out); 540 | 541 | /** 542 | * Return OnnxType from OrtTypeInfo 543 | */ 544 | ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ enum ONNXType* out); 545 | 546 | /** 547 | * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo 548 | */ 549 | ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out); 550 | 551 | ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type); 552 | 553 | /** 554 | * \param info Created from CreateTensorTypeAndShapeInfo() function 555 | * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. 556 | * \param dim_count length of dim_values 557 | */ 558 | ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); 559 | 560 | ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*, 561 | _Out_ enum ONNXTensorElementDataType* out); 562 | ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); 563 | ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, 564 | size_t dim_values_length); 565 | ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, 566 | _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length); 567 | 568 | /** 569 | * Return the number of elements specified by the tensor shape. 570 | * Return a negative value if unknown (i.e., any dimension is negative.) 571 | * e.g. 572 | * [] -> 1 573 | * [1,3,4] -> 12 574 | * [2,0,4] -> 0 575 | * [-1,3,4] -> -1 576 | */ 577 | ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); 578 | 579 | /** 580 | * \param out Should be freed by OrtReleaseTensorTypeAndShapeInfo after use 581 | */ 582 | ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); 583 | 584 | /** 585 | * Get the type information of an OrtValue 586 | * \param value 587 | * \param out The returned value should be freed by OrtReleaseTypeInfo after use 588 | */ 589 | ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out); 590 | 591 | ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); 592 | 593 | ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, 594 | enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out); 595 | 596 | /** 597 | * Convenience function for special case of CreateMemoryInfo, for the CPU allocator. Uses name = "Cpu" and id = 0. 598 | */ 599 | ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1, 600 | _Outptr_ OrtMemoryInfo** out); 601 | 602 | /** 603 | * Test if two memory info are equal 604 | * \Sets 'out' to 0 if equal, -1 if not equal 605 | */ 606 | ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out); 607 | 608 | /** 609 | * Do not free the returned value 610 | */ 611 | ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); 612 | ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); 613 | ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out); 614 | ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out); 615 | 616 | ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outptr_ void** out); 617 | ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ptr, void* p); 618 | ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ptr, _Outptr_ const struct OrtMemoryInfo** out); 619 | 620 | // The returned pointer doesn't have to be freed. 621 | // Always returns the same instance on every invocation. 622 | // Please note that this is a non-arena based allocator. 623 | ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); 624 | 625 | // Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable 626 | // optimizations that can take advantage of fixed values (such as memory planning, etc) 627 | ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation, 628 | _In_ int64_t dim_value); 629 | 630 | /** 631 | * APIs to support non-tensor types - map and sequence. 632 | * Currently only the following types are supported 633 | * Note: the following types should be kept in sync with data_types.h 634 | * Map types 635 | * ========= 636 | * std::map 637 | * std::map 638 | * std::map 639 | * std::map 640 | * std::map 641 | * std::map 642 | * std::map 643 | * std::map 644 | * 645 | * Sequence types 646 | * ============== 647 | * std::vector 648 | * std::vector 649 | * std::vector 650 | * std::vector 651 | * std::vector> 652 | * std::vector 653 | */ 654 | 655 | /** 656 | * If input OrtValue represents a map, you need to retrieve the keys and values 657 | * separately. Use index=0 to retrieve keys and index=1 to retrieve values. 658 | * If input OrtValue represents a sequence, use index to retrieve the index'th element 659 | * of the sequence. 660 | */ 661 | ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator, 662 | _Outptr_ OrtValue** out); 663 | 664 | /** 665 | * Returns 2 for type map and N for sequence where N is the number of elements 666 | * in the sequence. 667 | */ 668 | ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out); 669 | 670 | /** 671 | * To construct a map, use num_values = 2 and 'in' should be an arrary of 2 OrtValues 672 | * representing keys and values. 673 | * To construct a sequence, use num_values = N where N is the number of the elements in the 674 | * sequence. 'in' should be an arrary of N OrtValues. 675 | * \value_type should be either map or sequence. 676 | */ 677 | ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values, 678 | enum ONNXType value_type, _Outptr_ OrtValue** out); 679 | 680 | /** 681 | * Construct OrtValue that contains a value of non-standard type created for 682 | * experiments or while awaiting standardization. OrtValue in this case would contain 683 | * an internal representation of the Opaque type. Opaque types are distinguished between 684 | * each other by two strings 1) domain and 2) type name. The combination of the two 685 | * must be unique, so the type representation is properly identified internally. The combination 686 | * must be properly registered from within ORT at both compile/run time or by another API. 687 | * 688 | * To construct the OrtValue pass domain and type names, also a pointer to a data container 689 | * the type of which must be know to both ORT and the client program. That data container may or may 690 | * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for 691 | * verification purposes. 692 | * 693 | * \domain_name - domain name for the Opaque type, null terminated. 694 | * \type_name - type name for the Opaque type, null terminated. 695 | * \data_contianer - data to populate OrtValue 696 | * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected 697 | * data_container size internally. 698 | */ 699 | ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name, 700 | _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out); 701 | 702 | /** 703 | * Fetch data from an OrtValue that contains a value of non-standard type created for 704 | * experiments or while awaiting standardization. 705 | * \domain_name - domain name for the Opaque type, null terminated. 706 | * \type_name - type name for the Opaque type, null terminated. 707 | * \data_contianer - data to populate OrtValue 708 | * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected 709 | * data_container size internally. 710 | */ 711 | 712 | ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, 713 | _Out_ void* data_container, size_t data_container_size); 714 | 715 | ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, 716 | _Out_ float* out); 717 | ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, 718 | _Out_ int64_t* out); 719 | ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, 720 | _Inout_ size_t* size); 721 | 722 | ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); 723 | ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); 724 | ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, 725 | _Out_ const OrtValue** out); 726 | ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, 727 | _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out); 728 | 729 | ORT_CLASS_RELEASE(Env); 730 | ORT_CLASS_RELEASE(Status); // nullptr for Status* indicates success 731 | ORT_CLASS_RELEASE(MemoryInfo); 732 | ORT_CLASS_RELEASE(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool) 733 | ORT_CLASS_RELEASE(Value); 734 | ORT_CLASS_RELEASE(RunOptions); 735 | ORT_CLASS_RELEASE(TypeInfo); 736 | ORT_CLASS_RELEASE(TensorTypeAndShapeInfo); 737 | ORT_CLASS_RELEASE(SessionOptions); 738 | ORT_CLASS_RELEASE(CustomOpDomain); 739 | 740 | // End of Version 1 - DO NOT MODIFY ABOVE (see above text for more information) 741 | 742 | // Version 2 - In development, feel free to add/remove/rearrange here 743 | 744 | /** 745 | * GetDenotationFromTypeInfo 746 | * This api augments OrtTypeInfo to return denotations on the type. 747 | * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor. 748 | */ 749 | ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ const char** const denotation, 750 | _Out_ size_t* len); 751 | 752 | // OrtTypeInfo Casting methods 753 | 754 | /** 755 | * CastTypeInfoToMapTypeInfo 756 | * This api augments OrtTypeInfo to return an OrtMapTypeInfo when the type is a map. 757 | * The OrtMapTypeInfo has additional information about the map's key type and value type. 758 | * This is used by WinML to support model reflection APIs. 759 | * This is used by WinML to support model reflection APIs. 760 | * 761 | * Don't free the 'out' value 762 | */ 763 | ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info, 764 | _Outptr_result_maybenull_ const OrtMapTypeInfo** out); 765 | 766 | /** 767 | * CastTypeInfoToSequenceTypeInfo 768 | * This api augments OrtTypeInfo to return an OrtSequenceTypeInfo when the type is a sequence. 769 | * The OrtSequenceTypeInfo has additional information about the sequence's element type. 770 | * This is used by WinML to support model reflection APIs. 771 | * 772 | * Don't free the 'out' value 773 | */ 774 | ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info, 775 | _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out); 776 | 777 | // OrtMapTypeInfo Accessors 778 | 779 | /** 780 | * GetMapKeyType 781 | * This api augments get the key type of a map. Key types are restricted to being scalar types and use ONNXTensorElementDataType. 782 | * This is used by WinML to support model reflection APIs. 783 | */ 784 | ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out); 785 | 786 | /** 787 | * GetMapValueType 788 | * This api augments get the value type of a map. 789 | */ 790 | ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info); 791 | 792 | // OrtSequenceTypeInfo Accessors 793 | 794 | /** 795 | * GetSequenceElementType 796 | * This api augments get the element type of a sequence. 797 | * This is used by WinML to support model reflection APIs. 798 | */ 799 | ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info, 800 | _Outptr_ OrtTypeInfo** type_info); 801 | 802 | ORT_CLASS_RELEASE(MapTypeInfo); 803 | ORT_CLASS_RELEASE(SequenceTypeInfo); 804 | 805 | /** 806 | * \param out is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it. 807 | * Profiling is turned ON automatically if enabled for the particular session by invoking EnableProfiling() 808 | * on the SessionOptions instance used to create the session. 809 | */ 810 | ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* sess, _Inout_ OrtAllocator* allocator, _Outptr_ char** out); 811 | 812 | /** 813 | * \param out is a pointer to the newly created object. The pointer should be freed by calling ReleaseModelMetadata after use. 814 | */ 815 | ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* sess, _Outptr_ OrtModelMetadata** out); 816 | 817 | /** 818 | * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it. 819 | */ 820 | ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata, 821 | _Inout_ OrtAllocator* allocator, _Outptr_ char** value); 822 | ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata, 823 | _Inout_ OrtAllocator* allocator, _Outptr_ char** value); 824 | ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, 825 | _Outptr_ char** value); 826 | ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata, 827 | _Inout_ OrtAllocator* allocator, _Outptr_ char** value); 828 | /** 829 | * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it. 830 | * 'value' will be a nullptr if the given key is not found in the custom metadata map. 831 | */ 832 | ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata, 833 | _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value); 834 | 835 | ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value); 836 | 837 | ORT_CLASS_RELEASE(ModelMetadata); 838 | 839 | /* 840 | * Creates an environment with global threadpools that will be shared across sessions. 841 | * Use this in conjunction with DisablePerSessionThreads API or else the session will use 842 | * its own thread pools. 843 | */ 844 | ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel logging_level, _In_ const char* logid, 845 | _In_ const OrtThreadingOptions* t_options, _Outptr_ OrtEnv** out); 846 | 847 | /* 848 | * Calling this API will make the session use the global threadpools shared across sessions. 849 | * This API should be used in conjunction with CreateEnvWithGlobalThreadPools API. 850 | */ 851 | ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options); 852 | 853 | ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out); 854 | 855 | ORT_CLASS_RELEASE(ThreadingOptions); 856 | 857 | /** 858 | * \param num_keys contains the number of keys in the custom metadata map 859 | * \param keys is an array of null terminated strings (array count = num_keys) allocated using 'allocator'. 860 | * The caller is responsible for freeing each string and the pointer array. 861 | * 'keys' will be a nullptr if custom metadata map is empty. 862 | */ 863 | ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata, 864 | _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys); 865 | 866 | // Override symbolic dimensions (by specific name strings) with actual values 867 | // if known at session initialization time to enable optimizations that can 868 | // take advantage of fixed values (such as memory planning, etc) 869 | ORT_API2_STATUS(AddFreeDimensionOverrideByName, 870 | _Inout_ OrtSessionOptions* options, _In_ const char* dim_name, 871 | _In_ int64_t dim_value); 872 | 873 | /** 874 | * \param out_ptr will hold a pointer to the array of char * 875 | * representing available providers. 876 | * \param provider_length is a pointer to an int variable where 877 | * the number of available providers will be added. 878 | * The caller is responsible for freeing each char * and the pointer 879 | * array by calling ReleaseAvailableProviders(). 880 | */ 881 | ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr, 882 | _In_ int* provider_length); 883 | 884 | /** 885 | * \param ptr is the pointer to an array of available providers you 886 | * get after calling GetAvailableProviders(). 887 | * \param providers_length is the number of available providers. 888 | */ 889 | ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr, 890 | _In_ int providers_length); 891 | 892 | /** 893 | * \param value - A tensor created from OrtCreateTensor... function. 894 | * \param index - index of string tensor element, length of element at index will be returned. 895 | * \param out - number of UTF-8 bytes that the string contains 896 | */ 897 | ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out); 898 | 899 | /** 900 | * \param s string element contents in UTF-8 encoding. The string is NOT null-terminated. 901 | * \param value A tensor created from OrtCreateTensor... function. 902 | * \param s_len element length, get it from OrtGetStringTensorElementLength. 903 | * \param index offset of element of tensor to return. 904 | */ 905 | ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s); 906 | 907 | /** 908 | * \param value - A tensor created from OrtCreateTensor... function. 909 | * \param s - A null terminated UTF-8 encoded string. 910 | * \param index - index of string tensor element to fill 911 | */ 912 | ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index); 913 | 914 | /** 915 | * Set a single session configuration entry as a pair of strings 916 | * If a configuration with same key exists, this will overwrite the configuration with the given config_value 917 | * \param config_key A null terminated string representation of the config key 918 | * \param config_value A null terminated string representation of the config value 919 | * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h 920 | */ 921 | ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, 922 | _In_z_ const char* config_key, _In_z_ const char* config_value); 923 | 924 | /** 925 | * \param sess valid OrtSession instance 926 | * \param mem_info - valid OrtMemoryInfo instance 927 | * \param - out a ptr to a new instance of OrtAllocator according to the spec within mem_info 928 | * if successful 929 | * \return OrtStatus or nullptr if successful 930 | */ 931 | ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* sess, _In_ const OrtMemoryInfo* mem_info, 932 | _Outptr_ OrtAllocator** out); 933 | 934 | // Release instance of OrtAllocator obtained from CreateAllocator API 935 | ORT_CLASS_RELEASE(Allocator); 936 | 937 | ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr); 938 | 939 | // Creates an IoBinding instance that allows one to bind pre-allocated OrtValues 940 | // to input names. Thus if you want to use a raw on device buffer as input or output 941 | // you can avoid extra copy during runtime. 942 | ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* sess, _Outptr_ OrtIoBinding** out); 943 | 944 | // Release instance or OrtIoBinding obtained from CreateIoBinding API 945 | ORT_CLASS_RELEASE(IoBinding); 946 | 947 | /** 948 | * The function will bind the OrtValue to a specified input name. 949 | * The OrtValue must be a Tensor. ORT would use that value in place of input for the specified name. 950 | * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding() 951 | * \param name - name for the model input 952 | * \param val_ptr - OrtValue of Tensor type. 953 | * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success 954 | */ 955 | ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); 956 | 957 | /** 958 | * The function will bind the OrtValue to the specified output name. 959 | * The OrtValue must be a Tensor. ORT would use that value in place of output for the specified name. 960 | * 961 | * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding() 962 | * \param name - name for the model output 963 | * \param val_ptr - OrtValue of Tensor type. 964 | * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success 965 | */ 966 | ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); 967 | 968 | /** 969 | * The function will bind the OrtValue to a device which specification is contained within OrtMemoryInfo 970 | * You can either create an instance of OrtMemoryInfo with a device id or obtain one from the allocator that you are created/using 971 | * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocated and bind a chunk of 972 | * memory within OrtValue ahead of time. 973 | * 974 | * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding() 975 | * \param name - name for the model output 976 | * \param mem_info_ptr - OrtMemoryInfo 977 | * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success 978 | */ 979 | ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* val_ptr); 980 | 981 | /** 982 | * The function returns the names of the outputs in the order they were bound. This is useful after running the model 983 | * with bound outputs because the returned names are in order in which output OrtValues are returned. This API is optional 984 | * to use. If you knew the order of outputs and its names you used for binding you would not need to use this API. 985 | * 986 | * \param binding_ptr - a ptr to an instance of OrtIoBinding created obtained from CreateIoBinding() 987 | * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() 988 | * the specified allocator will be used to allocate continuous buffers for output strings and lengths. 989 | * \param buffer - pointer to a continuous buffer of non-zero terminated UTF-8 encoded strings. The number of strings stored is returned count parameter. 990 | * this buffer will be allocated with the specified allocator and must be freed after it is no longer needed. 991 | * \param lengths - a pointer to a continuous buffer of size_t lengths of strings returned in the buffer. The number of items is returned 992 | * in the count. This buffer is allocated with the specified allocator and must be freed after it is no longer needed. 993 | * \para count - is the number of strings returned. If the instance of OrtIoBiding has no bound outputs, zero is returned, 994 | * no memory allocation is performed and buffer and lengths are nullptr on return. 995 | */ 996 | ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, 997 | _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count); 998 | 999 | /** 1000 | * The function returns an array of pointers to individually allocated OrtValues that contain results of a model execution with RunWithBinding() 1001 | * The array contains the same number of OrtValues and they are in the same order as they were bound with BindOutput() 1002 | * or BindOutputToDevice(). 1003 | * The returned OrtValues must be individually released after they are no longer needed. 1004 | * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after 1005 | * all the OrtValues contained therein are individually released. 1006 | * 1007 | * \param binding_ptr - instance of OrtIoBidning 1008 | * \param allocator - instance of allocator to allocate output array 1009 | * \param output - pointer to the allocated buffer. Returns nullptr if no outputs. 1010 | * \param output_count - pointer to the number of OrtValues returned. Zero if no outputs. 1011 | */ 1012 | ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, 1013 | _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count); 1014 | 1015 | /** Clears any previously specified bindings for inputs/outputs 1016 | */ 1017 | void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; 1018 | void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; 1019 | 1020 | /** 1021 | * Provides element-level access into a tensor. 1022 | * \param location_values a pointer to an array of index values that specify an element's location in the tensor data blob 1023 | * \param location_values_count length of location_values 1024 | * \param out a pointer to the element specified by location_values 1025 | * e.g. 1026 | * Given a tensor with overall shape [3,224,224], an element at 1027 | * location [2,150,128] can be accessed directly. 1028 | * 1029 | * This function only works for numeric tensors. 1030 | * This is a no-copy method whose pointer is only valid until the backing OrtValue is free'd. 1031 | */ 1032 | ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out); 1033 | 1034 | /** 1035 | * Creates an allocator instance and registers it with the env to enable 1036 | * sharing between multiple sessions that use the same env instance. 1037 | * Lifetime of the created allocator will be valid for the duration of the environment. 1038 | * Returns an error if an allocator with the same OrtMemoryInfo is already registered. 1039 | * \param mem_info must be non-null. 1040 | * \param arena_cfg if nullptr defaults will be used. 1041 | * See docs/C_API.md for details. 1042 | */ 1043 | ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, 1044 | _In_ const OrtArenaCfg* arena_cfg); 1045 | 1046 | /** 1047 | * Set the language projection for collecting telemetry data when Env is created 1048 | * \param projection the source projected language. 1049 | */ 1050 | ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection); 1051 | 1052 | /** 1053 | * On some platforms, this timer may not be as precise as nanoseconds 1054 | * For instance, on Windows and MacOS, the precision will be ~100ns 1055 | * \param out is set to the nanoseconds of profiling's start time 1056 | */ 1057 | ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* sess, _Outptr_ uint64_t* out); 1058 | 1059 | /** 1060 | * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools. 1061 | * A value of 0 means ORT will pick the default. 1062 | * A value of 1 means the invoking thread will be used; no threads will be created in the thread pool. 1063 | */ 1064 | ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads); 1065 | ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads); 1066 | 1067 | /** 1068 | * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools. 1069 | * Allow spinning of thread pools when their queues are empty. This API will set the value for both 1070 | * inter_op and intra_op threadpools. 1071 | * \param allow_spinning valid values are 1 and 0. 1072 | * 1: threadpool will spin to wait for queue to become non-empty, 0: it won't spin. 1073 | * Prefer a value of 0 if your CPU usage is very high. 1074 | */ 1075 | ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning); 1076 | 1077 | /** 1078 | * Add a pre-allocated initializer to a session. If a model contains an initializer with a name 1079 | * that is same as the name passed to this API call, ORT will use this initializer instance 1080 | * instead of deserializing one from the model file. This is useful when you want to share 1081 | * the same initializer across sessions. 1082 | * \param name name of the initializer 1083 | * \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be 1084 | * managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object 1085 | * to which it is added. 1086 | */ 1087 | ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, 1088 | _In_ const OrtValue* val); 1089 | 1090 | /** 1091 | * Creates a custom environment with global threadpools and logger that will be shared across sessions. 1092 | * Use this in conjunction with DisablePerSessionThreads API or else the session will use 1093 | * its own thread pools. 1094 | * 1095 | * \param out should be freed by `OrtReleaseEnv` after use 1096 | */ 1097 | ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel logging_level, 1098 | _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); 1099 | 1100 | /** 1101 | * Append CUDA execution provider to the session options 1102 | * If CUDA is not available (due to a non cuda enabled build), this function will return failure. 1103 | */ 1104 | ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA, 1105 | _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); 1106 | 1107 | /** 1108 | * Append OpenVINO execution provider to the session options 1109 | * If OpenVINO is not available (due to the OpenVINO provider shared library or its dependencies not being installed), this function will fail. 1110 | */ 1111 | ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO, 1112 | _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options); 1113 | 1114 | /** 1115 | * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools. 1116 | * When this API is called, flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool. 1117 | * Note that an alternative way not using this option at runtime is to train and export a model without denormals 1118 | * and that's recommended because turning this option on may hurt model accuracy. 1119 | */ 1120 | ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options); 1121 | 1122 | /** 1123 | * Use this API to create the configuration of an arena that can eventually be used to define 1124 | * an arena based allocator's behavior 1125 | * \param max_mem - use 0 to allow ORT to choose the default 1126 | * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested 1127 | * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default 1128 | * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default 1129 | * \param out - a pointer to an OrtArenaCfg instance 1130 | * \return a nullptr in case of success or a pointer to an OrtStatus instance in case of failure 1131 | * See docs/C_API.md for details on what the following parameters mean and how to choose these values 1132 | */ 1133 | ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, 1134 | int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out); 1135 | 1136 | ORT_CLASS_RELEASE(ArenaCfg); 1137 | }; 1138 | 1139 | /* 1140 | * Steps to use a custom op: 1141 | * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops 1142 | * 2 Create an OrtCustomOp structure for each op and add them to the domain 1143 | * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options 1144 | */ 1145 | #define OrtCustomOpApi OrtApi 1146 | 1147 | /* 1148 | * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by 1149 | * the implementor of the custom op. 1150 | */ 1151 | struct OrtCustomOp { 1152 | uint32_t version; // Initialize to ORT_API_VERSION 1153 | 1154 | // This callback creates the kernel, which is a user defined parameter that is passed to the Kernel* callbacks below. 1155 | void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, 1156 | _In_ const OrtKernelInfo* info); 1157 | 1158 | // Returns the name of the op 1159 | const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op); 1160 | 1161 | // Returns the type of the execution provider, return nullptr to use CPU execution provider 1162 | const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op); 1163 | 1164 | // Returns the count and types of the input & output tensors 1165 | ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); 1166 | size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op); 1167 | ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); 1168 | size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op); 1169 | 1170 | // Op kernel callbacks 1171 | void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context); 1172 | void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel); 1173 | }; 1174 | 1175 | /* 1176 | * END EXPERIMENTAL 1177 | */ 1178 | 1179 | #ifdef __cplusplus 1180 | } 1181 | #endif 1182 | -------------------------------------------------------------------------------- /include/onnxruntime_c_api_expanded.h: -------------------------------------------------------------------------------- 1 | #ifdef C2NIM 2 | # dynlib libname 3 | # cdecl 4 | # define libname "libonnxruntime.so" 5 | #endif 6 | 7 | typedef enum ONNXTensorElementDataType 8 | { 9 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, 10 | ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, 11 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, 12 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, 13 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, 14 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, 15 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, 16 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, 17 | ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, 18 | ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, 19 | ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, 20 | ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, 21 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, 22 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, 23 | ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, 24 | ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, 25 | ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 26 | } 27 | 28 | ONNXTensorElementDataType; 29 | typedef enum ONNXType 30 | { 31 | ONNX_TYPE_UNKNOWN, 32 | ONNX_TYPE_TENSOR, 33 | ONNX_TYPE_SEQUENCE, 34 | ONNX_TYPE_MAP, 35 | ONNX_TYPE_OPAQUE, 36 | ONNX_TYPE_SPARSETENSOR, 37 | } 38 | 39 | ONNXType; 40 | typedef enum OrtLoggingLevel 41 | { 42 | ORT_LOGGING_LEVEL_VERBOSE, 43 | ORT_LOGGING_LEVEL_INFO, 44 | ORT_LOGGING_LEVEL_WARNING, 45 | ORT_LOGGING_LEVEL_ERROR, 46 | ORT_LOGGING_LEVEL_FATAL, 47 | } 48 | 49 | OrtLoggingLevel; 50 | typedef enum OrtErrorCode 51 | { 52 | ORT_OK, 53 | ORT_FAIL, 54 | ORT_INVALID_ARGUMENT, 55 | ORT_NO_SUCHFILE, 56 | ORT_NO_MODEL, 57 | ORT_ENGINE_ERROR, 58 | ORT_RUNTIME_EXCEPTION, 59 | ORT_INVALID_PROTOBUF, 60 | ORT_MODEL_LOADED, 61 | ORT_NOT_IMPLEMENTED, 62 | ORT_INVALID_GRAPH, 63 | ORT_EP_FAIL, 64 | } 65 | 66 | OrtErrorCode; 67 | struct OrtEnv; 68 | typedef struct OrtEnv OrtEnv;; 69 | struct OrtStatus; 70 | typedef struct OrtStatus OrtStatus;; 71 | struct OrtMemoryInfo; 72 | typedef struct OrtMemoryInfo OrtMemoryInfo;; 73 | struct OrtIoBinding; 74 | typedef struct OrtIoBinding OrtIoBinding;; 75 | struct OrtSession; 76 | typedef struct OrtSession OrtSession;; 77 | struct OrtValue; 78 | typedef struct OrtValue OrtValue;; 79 | struct OrtRunOptions; 80 | typedef struct OrtRunOptions OrtRunOptions;; 81 | struct OrtTypeInfo; 82 | typedef struct OrtTypeInfo OrtTypeInfo;; 83 | struct OrtTensorTypeAndShapeInfo; 84 | typedef struct OrtTensorTypeAndShapeInfo OrtTensorTypeAndShapeInfo;; 85 | struct OrtSessionOptions; 86 | typedef struct OrtSessionOptions OrtSessionOptions;; 87 | struct OrtCustomOpDomain; 88 | typedef struct OrtCustomOpDomain OrtCustomOpDomain;; 89 | struct OrtMapTypeInfo; 90 | typedef struct OrtMapTypeInfo OrtMapTypeInfo;; 91 | struct OrtSequenceTypeInfo; 92 | typedef struct OrtSequenceTypeInfo OrtSequenceTypeInfo;; 93 | struct OrtModelMetadata; 94 | typedef struct OrtModelMetadata OrtModelMetadata;; 95 | struct OrtThreadPoolParams; 96 | typedef struct OrtThreadPoolParams OrtThreadPoolParams;; 97 | struct OrtThreadingOptions; 98 | typedef struct OrtThreadingOptions OrtThreadingOptions;; 99 | struct OrtArenaCfg; 100 | typedef struct OrtArenaCfg OrtArenaCfg;; 101 | typedef OrtStatus * OrtStatusPtr; 102 | typedef struct OrtAllocator 103 | { 104 | uint32_t version; 105 | void *(*Alloc)(struct OrtAllocator *this, size_t size); 106 | void(*Free)(struct OrtAllocator *this, void *p); 107 | const struct OrtMemoryInfo * (*Info)(const struct OrtAllocator *this); 108 | } 109 | 110 | OrtAllocator; 111 | typedef void(*OrtLoggingFunction)( 112 | void *param, OrtLoggingLevel severity, const char *category, const char *logid, const char *code_location, const char *message); 113 | typedef enum GraphOptimizationLevel 114 | { 115 | ORT_DISABLE_ALL = 0, 116 | ORT_ENABLE_BASIC = 1, 117 | ORT_ENABLE_EXTENDED = 2, 118 | ORT_ENABLE_ALL = 99 119 | } 120 | 121 | GraphOptimizationLevel; 122 | typedef enum ExecutionMode 123 | { 124 | ORT_SEQUENTIAL = 0, 125 | ORT_PARALLEL = 1, 126 | } 127 | 128 | ExecutionMode; 129 | typedef enum OrtLanguageProjection 130 | { 131 | ORT_PROJECTION_C = 0, 132 | ORT_PROJECTION_CPLUSPLUS = 1, 133 | ORT_PROJECTION_CSHARP = 2, 134 | ORT_PROJECTION_PYTHON = 3, 135 | ORT_PROJECTION_JAVA = 4, 136 | ORT_PROJECTION_WINML = 5, 137 | ORT_PROJECTION_NODEJS = 6, 138 | } 139 | 140 | OrtLanguageProjection; 141 | struct OrtKernelInfo; 142 | typedef struct OrtKernelInfo OrtKernelInfo; 143 | struct OrtKernelContext; 144 | typedef struct OrtKernelContext OrtKernelContext; 145 | struct OrtCustomOp; 146 | typedef struct OrtCustomOp OrtCustomOp; 147 | typedef enum OrtAllocatorType 148 | { 149 | Invalid = -1, 150 | OrtDeviceAllocator = 0, 151 | OrtArenaAllocator = 1 152 | } 153 | 154 | OrtAllocatorType; 155 | typedef enum OrtMemType 156 | { 157 | OrtMemTypeCPUInput = -2, 158 | OrtMemTypeCPUOutput = -1, 159 | OrtMemTypeCPU = OrtMemTypeCPUOutput, 160 | OrtMemTypeDefault = 0, 161 | } 162 | 163 | OrtMemType; 164 | typedef enum OrtCudnnConvAlgoSearch 165 | { 166 | EXHAUSTIVE, 167 | HEURISTIC, 168 | DEFAULT, 169 | } 170 | 171 | OrtCudnnConvAlgoSearch; 172 | typedef struct OrtCUDAProviderOptions 173 | { 174 | int device_id; 175 | OrtCudnnConvAlgoSearch cudnn_conv_algo_search; 176 | size_t cuda_mem_limit; 177 | int arena_extend_strategy; 178 | int do_copy_in_default_stream; 179 | } 180 | 181 | OrtCUDAProviderOptions; 182 | typedef struct OrtOpenVINOProviderOptions 183 | { 184 | const char *device_type; 185 | unsigned char enable_vpu_fast_compile; 186 | const char *device_id; 187 | size_t num_of_threads; 188 | } 189 | 190 | OrtOpenVINOProviderOptions; 191 | struct OrtApi; 192 | typedef struct OrtApi OrtApi; 193 | struct OrtApiBase 194 | { 195 | const OrtApi * (*GetApi)(uint32_t version); 196 | const char *(*GetVersionString)(); 197 | }; 198 | typedef struct OrtApiBase OrtApiBase; 199 | const OrtApiBase* OrtGetApiBase(void); 200 | struct OrtApi 201 | { 202 | OrtStatus * (*CreateStatus)(OrtErrorCode code, const char *msg); 203 | OrtErrorCode(*GetErrorCode)(const OrtStatus *status); 204 | const char *(*GetErrorMessage)(const OrtStatus *status); 205 | OrtStatusPtr(*CreateEnv)(OrtLoggingLevel logging_level, const char *logid, OrtEnv **out); 206 | OrtStatusPtr(*CreateEnvWithCustomLogger)(OrtLoggingFunction logging_function, void *logger_param, OrtLoggingLevel logging_level, const char *logid, OrtEnv **out); 207 | OrtStatusPtr(*EnableTelemetryEvents)(const OrtEnv *env); 208 | OrtStatusPtr(*DisableTelemetryEvents)(const OrtEnv *env); 209 | OrtStatusPtr(*CreateSession)(const OrtEnv *env, const char *model_path, const OrtSessionOptions *options, OrtSession **out); 210 | OrtStatusPtr(*CreateSessionFromArray)(const OrtEnv *env, const void *model_data, size_t model_data_length, const OrtSessionOptions *options, OrtSession **out); 211 | OrtStatusPtr(*Run)(OrtSession *sess, const OrtRunOptions *run_options, const char *const *input_names, const OrtValue *const *input, size_t input_len, const char *const *output_names1, size_t output_names_len, OrtValue **output); 212 | OrtStatusPtr(*CreateSessionOptions)(OrtSessionOptions **options); 213 | OrtStatusPtr(*SetOptimizedModelFilePath)(OrtSessionOptions *options, const char *optimized_model_filepath); 214 | OrtStatusPtr(*CloneSessionOptions)(const OrtSessionOptions *in_options, OrtSessionOptions **out_options); 215 | OrtStatusPtr(*SetSessionExecutionMode)(OrtSessionOptions *options, ExecutionMode execution_mode); 216 | OrtStatusPtr(*EnableProfiling)(OrtSessionOptions *options, const char *profile_file_prefix); 217 | OrtStatusPtr(*DisableProfiling)(OrtSessionOptions *options); 218 | OrtStatusPtr(*EnableMemPattern)(OrtSessionOptions *options); 219 | OrtStatusPtr(*DisableMemPattern)(OrtSessionOptions *options); 220 | OrtStatusPtr(*EnableCpuMemArena)(OrtSessionOptions *options); 221 | OrtStatusPtr(*DisableCpuMemArena)(OrtSessionOptions *options); 222 | OrtStatusPtr(*SetSessionLogId)(OrtSessionOptions *options, const char *logid); 223 | OrtStatusPtr(*SetSessionLogVerbosityLevel)(OrtSessionOptions *options, int session_log_verbosity_level); 224 | OrtStatusPtr(*SetSessionLogSeverityLevel)(OrtSessionOptions *options, int session_log_severity_level); 225 | OrtStatusPtr(*SetSessionGraphOptimizationLevel)(OrtSessionOptions *options, GraphOptimizationLevel graph_optimization_level); 226 | OrtStatusPtr(*SetIntraOpNumThreads)(OrtSessionOptions *options, int intra_op_num_threads); 227 | OrtStatusPtr(*SetInterOpNumThreads)(OrtSessionOptions *options, int inter_op_num_threads); 228 | OrtStatusPtr(*CreateCustomOpDomain)(const char *domain, OrtCustomOpDomain **out); 229 | OrtStatusPtr(*CustomOpDomain_Add)(OrtCustomOpDomain *custom_op_domain, const OrtCustomOp *op); 230 | OrtStatusPtr(*AddCustomOpDomain)(OrtSessionOptions *options, OrtCustomOpDomain *custom_op_domain); 231 | OrtStatusPtr(*RegisterCustomOpsLibrary)(OrtSessionOptions *options, const char *library_path, void **library_handle); 232 | OrtStatusPtr(*SessionGetInputCount)(const OrtSession *sess, size_t *out); 233 | OrtStatusPtr(*SessionGetOutputCount)(const OrtSession *sess, size_t *out); 234 | OrtStatusPtr(*SessionGetOverridableInitializerCount)(const OrtSession *sess, size_t *out); 235 | OrtStatusPtr(*SessionGetInputTypeInfo)(const OrtSession *sess, size_t index, OrtTypeInfo **type_info); 236 | OrtStatusPtr(*SessionGetOutputTypeInfo)(const OrtSession *sess, size_t index, OrtTypeInfo **type_info); 237 | OrtStatusPtr(*SessionGetOverridableInitializerTypeInfo)(const OrtSession *sess, size_t index, OrtTypeInfo **type_info); 238 | OrtStatusPtr(*SessionGetInputName)(const OrtSession *sess, size_t index, OrtAllocator *allocator, char **value); 239 | OrtStatusPtr(*SessionGetOutputName)(const OrtSession *sess, size_t index, OrtAllocator *allocator, char **value); 240 | OrtStatusPtr(*SessionGetOverridableInitializerName)(const OrtSession *sess, size_t index, OrtAllocator *allocator, char **value); 241 | OrtStatusPtr(*CreateRunOptions)(OrtRunOptions **out); 242 | OrtStatusPtr(*RunOptionsSetRunLogVerbosityLevel)(OrtRunOptions *options, int value); 243 | OrtStatusPtr(*RunOptionsSetRunLogSeverityLevel)(OrtRunOptions *options, int value); 244 | OrtStatusPtr(*RunOptionsSetRunTag)(OrtRunOptions *, const char *run_tag); 245 | OrtStatusPtr(*RunOptionsGetRunLogVerbosityLevel)(const OrtRunOptions *options, int *out); 246 | OrtStatusPtr(*RunOptionsGetRunLogSeverityLevel)(const OrtRunOptions *options, int *out); 247 | OrtStatusPtr(*RunOptionsGetRunTag)(const OrtRunOptions *, const char **out); 248 | OrtStatusPtr(*RunOptionsSetTerminate)(OrtRunOptions *options); 249 | OrtStatusPtr(*RunOptionsUnsetTerminate)(OrtRunOptions *options); 250 | OrtStatusPtr(*CreateTensorAsOrtValue)(OrtAllocator *allocator, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type, OrtValue **out); 251 | OrtStatusPtr(*CreateTensorWithDataAsOrtValue)(const OrtMemoryInfo *info, void *p_data, size_t p_data_len, const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type, OrtValue **out); 252 | OrtStatusPtr(*IsTensor)(const OrtValue *value, int *out); 253 | OrtStatusPtr(*GetTensorMutableData)(OrtValue *value, void **out); 254 | OrtStatusPtr(*FillStringTensor)(OrtValue *value, const char *const *s, size_t s_len); 255 | OrtStatusPtr(*GetStringTensorDataLength)(const OrtValue *value, size_t *len); 256 | OrtStatusPtr(*GetStringTensorContent)(const OrtValue *value, void *s, size_t s_len, size_t *offsets, size_t offsets_len); 257 | OrtStatusPtr(*CastTypeInfoToTensorInfo)(const OrtTypeInfo *, const OrtTensorTypeAndShapeInfo **out); 258 | OrtStatusPtr(*GetOnnxTypeFromTypeInfo)(const OrtTypeInfo *, enum ONNXType *out); 259 | OrtStatusPtr(*CreateTensorTypeAndShapeInfo)(OrtTensorTypeAndShapeInfo **out); 260 | OrtStatusPtr(*SetTensorElementType)(OrtTensorTypeAndShapeInfo *, enum ONNXTensorElementDataType type); 261 | OrtStatusPtr(*SetDimensions)(OrtTensorTypeAndShapeInfo *info, const int64_t *dim_values, size_t dim_count); 262 | OrtStatusPtr(*GetTensorElementType)(const OrtTensorTypeAndShapeInfo *, enum ONNXTensorElementDataType *out); 263 | OrtStatusPtr(*GetDimensionsCount)(const OrtTensorTypeAndShapeInfo *info, size_t *out); 264 | OrtStatusPtr(*GetDimensions)(const OrtTensorTypeAndShapeInfo *info, int64_t *dim_values, size_t dim_values_length); 265 | OrtStatusPtr(*GetSymbolicDimensions)(const OrtTensorTypeAndShapeInfo *info, const char *dim_params[], size_t dim_params_length); 266 | OrtStatusPtr(*GetTensorShapeElementCount)(const OrtTensorTypeAndShapeInfo *info, size_t *out); 267 | OrtStatusPtr(*GetTensorTypeAndShape)(const OrtValue *value, OrtTensorTypeAndShapeInfo **out); 268 | OrtStatusPtr(*GetTypeInfo)(const OrtValue *value, OrtTypeInfo **out); 269 | OrtStatusPtr(*GetValueType)(const OrtValue *value, enum ONNXType *out); 270 | OrtStatusPtr(*CreateMemoryInfo)(const char *name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, OrtMemoryInfo **out); 271 | OrtStatusPtr(*CreateCpuMemoryInfo)(enum OrtAllocatorType type, enum OrtMemType mem_type1, OrtMemoryInfo **out); 272 | OrtStatusPtr(*CompareMemoryInfo)(const OrtMemoryInfo *info1, const OrtMemoryInfo *info2, int *out); 273 | OrtStatusPtr(*MemoryInfoGetName)(const OrtMemoryInfo *ptr, const char **out); 274 | OrtStatusPtr(*MemoryInfoGetId)(const OrtMemoryInfo *ptr, int *out); 275 | OrtStatusPtr(*MemoryInfoGetMemType)(const OrtMemoryInfo *ptr, OrtMemType *out); 276 | OrtStatusPtr(*MemoryInfoGetType)(const OrtMemoryInfo *ptr, OrtAllocatorType *out); 277 | OrtStatusPtr(*AllocatorAlloc)(OrtAllocator *ptr, size_t size, void **out); 278 | OrtStatusPtr(*AllocatorFree)(OrtAllocator *ptr, void *p); 279 | OrtStatusPtr(*AllocatorGetInfo)(const OrtAllocator *ptr, const struct OrtMemoryInfo **out); 280 | OrtStatusPtr(*GetAllocatorWithDefaultOptions)(OrtAllocator **out); 281 | OrtStatusPtr(*AddFreeDimensionOverride)(OrtSessionOptions *options, const char *dim_denotation, int64_t dim_value); 282 | OrtStatusPtr(*GetValue)(const OrtValue *value, int index, OrtAllocator *allocator, OrtValue **out); 283 | OrtStatusPtr(*GetValueCount)(const OrtValue *value, size_t *out); 284 | OrtStatusPtr(*CreateValue)(const OrtValue *const *in, size_t num_values, enum ONNXType value_type, OrtValue **out); 285 | OrtStatusPtr(*CreateOpaqueValue)(const char *domain_name, const char *type_name, const void *data_container, size_t data_container_size, OrtValue **out); 286 | OrtStatusPtr(*GetOpaqueValue)(const char *domain_name, const char *type_name, const OrtValue *in, void *data_container, size_t data_container_size); 287 | OrtStatusPtr(*KernelInfoGetAttribute_float)(const OrtKernelInfo *info, const char *name, float *out); 288 | OrtStatusPtr(*KernelInfoGetAttribute_int64)(const OrtKernelInfo *info, const char *name, int64_t *out); 289 | OrtStatusPtr(*KernelInfoGetAttribute_string)(const OrtKernelInfo *info, const char *name, char *out, size_t *size); 290 | OrtStatusPtr(*KernelContext_GetInputCount)(const OrtKernelContext *context, size_t *out); 291 | OrtStatusPtr(*KernelContext_GetOutputCount)(const OrtKernelContext *context, size_t *out); 292 | OrtStatusPtr(*KernelContext_GetInput)(const OrtKernelContext *context, size_t index, const OrtValue **out); 293 | OrtStatusPtr(*KernelContext_GetOutput)(OrtKernelContext *context, size_t index, const int64_t *dim_values, size_t dim_count, OrtValue **out); 294 | void(*ReleaseEnv)(OrtEnv *input); 295 | void(*ReleaseStatus)(OrtStatus *input); 296 | void(*ReleaseMemoryInfo)(OrtMemoryInfo *input); 297 | void(*ReleaseSession)(OrtSession *input); 298 | void(*ReleaseValue)(OrtValue *input); 299 | void(*ReleaseRunOptions)(OrtRunOptions *input); 300 | void(*ReleaseTypeInfo)(OrtTypeInfo *input); 301 | void(*ReleaseTensorTypeAndShapeInfo)(OrtTensorTypeAndShapeInfo *input); 302 | void(*ReleaseSessionOptions)(OrtSessionOptions *input); 303 | void(*ReleaseCustomOpDomain)(OrtCustomOpDomain *input); 304 | OrtStatusPtr(*GetDenotationFromTypeInfo)(const OrtTypeInfo *, const char **const denotation, size_t *len); 305 | OrtStatusPtr(*CastTypeInfoToMapTypeInfo)(const OrtTypeInfo *type_info, const OrtMapTypeInfo **out); 306 | OrtStatusPtr(*CastTypeInfoToSequenceTypeInfo)(const OrtTypeInfo *type_info, const OrtSequenceTypeInfo **out); 307 | OrtStatusPtr(*GetMapKeyType)(const OrtMapTypeInfo *map_type_info, enum ONNXTensorElementDataType *out); 308 | OrtStatusPtr(*GetMapValueType)(const OrtMapTypeInfo *map_type_info, OrtTypeInfo **type_info); 309 | OrtStatusPtr(*GetSequenceElementType)(const OrtSequenceTypeInfo *sequence_type_info, OrtTypeInfo **type_info); 310 | void(*ReleaseMapTypeInfo)(OrtMapTypeInfo *input); 311 | void(*ReleaseSequenceTypeInfo)(OrtSequenceTypeInfo *input); 312 | OrtStatusPtr(*SessionEndProfiling)(OrtSession *sess, OrtAllocator *allocator, char **out); 313 | OrtStatusPtr(*SessionGetModelMetadata)(const OrtSession *sess, OrtModelMetadata **out); 314 | OrtStatusPtr(*ModelMetadataGetProducerName)(const OrtModelMetadata *model_metadata, OrtAllocator *allocator, char **value); 315 | OrtStatusPtr(*ModelMetadataGetGraphName)(const OrtModelMetadata *model_metadata, OrtAllocator *allocator, char **value); 316 | OrtStatusPtr(*ModelMetadataGetDomain)(const OrtModelMetadata *model_metadata, OrtAllocator *allocator, char **value); 317 | OrtStatusPtr(*ModelMetadataGetDescription)(const OrtModelMetadata *model_metadata, OrtAllocator *allocator, char **value); 318 | OrtStatusPtr(*ModelMetadataLookupCustomMetadataMap)(const OrtModelMetadata *model_metadata, OrtAllocator *allocator, const char *key, char **value); 319 | OrtStatusPtr(*ModelMetadataGetVersion)(const OrtModelMetadata *model_metadata, int64_t *value); 320 | void(*ReleaseModelMetadata)(OrtModelMetadata *input); 321 | OrtStatusPtr(*CreateEnvWithGlobalThreadPools)(OrtLoggingLevel logging_level, const char *logid, const OrtThreadingOptions *t_options, OrtEnv **out); 322 | OrtStatusPtr(*DisablePerSessionThreads)(OrtSessionOptions *options); 323 | OrtStatusPtr(*CreateThreadingOptions)(OrtThreadingOptions **out); 324 | void(*ReleaseThreadingOptions)(OrtThreadingOptions *input); 325 | OrtStatusPtr(*ModelMetadataGetCustomMetadataMapKeys)(const OrtModelMetadata *model_metadata, OrtAllocator *allocator, char ***keys, int64_t *num_keys); 326 | OrtStatusPtr(*AddFreeDimensionOverrideByName)(OrtSessionOptions *options, const char *dim_name, int64_t dim_value); 327 | OrtStatusPtr(*GetAvailableProviders)(char ***out_ptr, int *provider_length); 328 | OrtStatusPtr(*ReleaseAvailableProviders)(char **ptr, int providers_length); 329 | OrtStatusPtr(*GetStringTensorElementLength)(const OrtValue *value, size_t index, size_t *out); 330 | OrtStatusPtr(*GetStringTensorElement)(const OrtValue *value, size_t s_len, size_t index, void *s); 331 | OrtStatusPtr(*FillStringTensorElement)(OrtValue *value, const char *s, size_t index); 332 | OrtStatusPtr(*AddSessionConfigEntry)(OrtSessionOptions *options, const char *config_key, const char *config_value); 333 | OrtStatusPtr(*CreateAllocator)(const OrtSession *sess, const OrtMemoryInfo *mem_info, OrtAllocator **out); 334 | void(*ReleaseAllocator)(OrtAllocator *input); 335 | OrtStatusPtr(*RunWithBinding)(OrtSession *sess, const OrtRunOptions *run_options, const OrtIoBinding *binding_ptr); 336 | OrtStatusPtr(*CreateIoBinding)(OrtSession *sess, OrtIoBinding **out); 337 | void(*ReleaseIoBinding)(OrtIoBinding *input); 338 | OrtStatusPtr(*BindInput)(OrtIoBinding *binding_ptr, const char *name, const OrtValue *val_ptr); 339 | OrtStatusPtr(*BindOutput)(OrtIoBinding *binding_ptr, const char *name, const OrtValue *val_ptr); 340 | OrtStatusPtr(*BindOutputToDevice)(OrtIoBinding *binding_ptr, const char *name, const OrtMemoryInfo *val_ptr); 341 | OrtStatusPtr(*GetBoundOutputNames)(const OrtIoBinding *binding_ptr, OrtAllocator *allocator, char **buffer, size_t **lengths, size_t *count); 342 | OrtStatusPtr(*GetBoundOutputValues)(const OrtIoBinding *binding_ptr, OrtAllocator *allocator, OrtValue ***output, size_t *output_count); 343 | void(*ClearBoundInputs)(OrtIoBinding *binding_ptr); 344 | void(*ClearBoundOutputs)(OrtIoBinding *binding_ptr); 345 | OrtStatusPtr(*TensorAt)(OrtValue *value, const int64_t *location_values, size_t location_values_count, void **out); 346 | OrtStatusPtr(*CreateAndRegisterAllocator)(OrtEnv *env, const OrtMemoryInfo *mem_info, const OrtArenaCfg *arena_cfg); 347 | OrtStatusPtr(*SetLanguageProjection)(const OrtEnv *ort_env, OrtLanguageProjection projection); 348 | OrtStatusPtr(*SessionGetProfilingStartTimeNs)(const OrtSession *sess, uint64_t *out); 349 | OrtStatusPtr(*SetGlobalIntraOpNumThreads)(OrtThreadingOptions *tp_options, int intra_op_num_threads); 350 | OrtStatusPtr(*SetGlobalInterOpNumThreads)(OrtThreadingOptions *tp_options, int inter_op_num_threads); 351 | OrtStatusPtr(*SetGlobalSpinControl)(OrtThreadingOptions *tp_options, int allow_spinning); 352 | OrtStatusPtr(*AddInitializer)(OrtSessionOptions *options, const char *name, const OrtValue *val); 353 | OrtStatusPtr(*CreateEnvWithCustomLoggerAndGlobalThreadPools)(OrtLoggingFunction logging_function, void *logger_param, OrtLoggingLevel logging_level, const char *logid, const struct OrtThreadingOptions *tp_options, OrtEnv **out); 354 | OrtStatusPtr(*SessionOptionsAppendExecutionProvider_CUDA)(OrtSessionOptions *options, const OrtCUDAProviderOptions *cuda_options); 355 | OrtStatusPtr(*SessionOptionsAppendExecutionProvider_OpenVINO)(OrtSessionOptions *options, const OrtOpenVINOProviderOptions *provider_options); 356 | OrtStatusPtr(*SetGlobalDenormalAsZero)(OrtThreadingOptions *tp_options); 357 | OrtStatusPtr(*CreateArenaCfg)(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk, OrtArenaCfg **out); 358 | void(*ReleaseArenaCfg)(OrtArenaCfg *input); 359 | }; 360 | struct OrtCustomOp 361 | { 362 | uint32_t version; 363 | void *(*CreateKernel)(const struct OrtCustomOp *op, const OrtApi *api, const OrtKernelInfo *info); 364 | const char *(*GetName)(const struct OrtCustomOp *op); 365 | const char *(*GetExecutionProviderType)(const struct OrtCustomOp *op); 366 | ONNXTensorElementDataType(*GetInputType)(const struct OrtCustomOp *op, size_t index); 367 | size_t(*GetInputTypeCount)(const struct OrtCustomOp *op); 368 | ONNXTensorElementDataType(*GetOutputType)(const struct OrtCustomOp *op, size_t index); 369 | size_t(*GetOutputTypeCount)(const struct OrtCustomOp *op); 370 | void(*KernelCompute)(void *op_kernel, OrtKernelContext *context); 371 | void(*KernelDestroy)(void *op_kernel); 372 | }; 373 | -------------------------------------------------------------------------------- /lib/libonnxruntime.so.1.6.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YesDrX/onnxruntime-nim/90ead4efb09c5d897dd0696f6d04a9cbb64892cb/lib/libonnxruntime.so.1.6.0 -------------------------------------------------------------------------------- /onnxruntime.nimble: -------------------------------------------------------------------------------- 1 | version = "0.1.0" 2 | author = "YesDrX" 3 | description = "onnxruntime c Api wrapped for nim." 4 | license = "MIT" 5 | srcDir = "src" 6 | skipDirs = @["sample", "lib", "include"] 7 | -------------------------------------------------------------------------------- /sample/C_Api_Sample.nim: -------------------------------------------------------------------------------- 1 | import onnxruntime 2 | import sequtils 3 | import strformat 4 | 5 | let 6 | api = OrtGetApiBase().GetApi(ORT_API_VERSION) 7 | model_path = "squeezenet.onnx".cstring 8 | 9 | proc `$`(strPtr: ptr cstring): string= 10 | return $(cast[cstringArray](strPtr).cstringArrayToSeq) 11 | 12 | proc checkStatus(status: OrtStatusPtr) = 13 | if status != nil: 14 | let msg = api.GetErrorMessage(status) 15 | echo fmt"[ERROR] : {msg}" 16 | api.ReleaseStatus(status) 17 | quit 1 18 | 19 | when isMainModule: 20 | var 21 | env : ptr OrtEnv 22 | session_options : ptr OrtSessionOptions 23 | session : ptr OrtSession 24 | allocator : ptr OrtAllocator 25 | num_input_nodes : csize_t 26 | input_node_names : seq[cstring] 27 | input_name : cstring 28 | typeinfo : ptr OrtTypeInfo 29 | tensor_info : ptr OrtTensorTypeAndShapeInfo 30 | dataType : ONNXTensorElementDataType 31 | num_dims : csize_t 32 | input_node_dims : seq[int] 33 | input_tensor_size : int = 1 34 | 35 | api.CreateEnv(ORT_LOGGING_LEVEL_WARNING, "test".cstring, env.addr).checkStatus 36 | api.CreateSessionOptions(session_options.addr).checkStatus 37 | api.SetIntraOpNumThreads(session_options, 1).checkStatus 38 | api.SetSessionGraphOptimizationLevel(session_options, ORT_ENABLE_BASIC).checkStatus 39 | 40 | echo fmt"Using Onnxruntime C API" 41 | 42 | api.CreateSession(env, model_path, session_options, session.addr).checkStatus 43 | api.GetAllocatorWithDefaultOptions(allocator.addr).checkStatus 44 | api.SessionGetInputCount(session, num_input_nodes.addr).checkStatus 45 | 46 | input_node_names = newSeq[cstring](num_input_nodes) 47 | echo fmt"Number of inputs = {num_input_nodes}" 48 | 49 | for i in 0 .. num_input_nodes.int - 1: 50 | api.SessionGetInputName(session, 0.csize_t, allocator, input_name.addr).checkStatus 51 | echo fmt"Input {i} : name= {input_name}" 52 | input_node_names[i] = input_name 53 | 54 | api.SessionGetInputTypeInfo(session, i.csize_t, typeinfo.addr).checkStatus 55 | api.CastTypeInfoToTensorInfo(typeinfo, tensor_info.addr).checkStatus 56 | api.GetTensorElementType(tensor_info, dataType.addr).checkStatus 57 | 58 | echo fmt"Input {i} : type = {dataType}" 59 | 60 | api.GetDimensionsCount(tensor_info, num_dims.addr).checkStatus 61 | echo fmt"Input {i} : num_dims = {num_dims.int}" 62 | 63 | input_node_dims = newSeq[int](num_dims) 64 | api.GetDimensions(tensor_info, cast[ptr int64](input_node_dims[0].addr), num_dims).checkStatus 65 | for j in 0 .. num_dims.int - 1: 66 | echo fmt"Input {i} : dim {j} = {input_node_dims[j]}" 67 | 68 | api.ReleaseTypeInfo(typeinfo) 69 | 70 | for i in 0 .. num_dims-1: 71 | input_tensor_size *= input_node_dims[i] 72 | 73 | var 74 | input_tensor_values = newSeq[cfloat](input_tensor_size) 75 | output_node_names = @["softmaxout_1".cstring] 76 | memory_info : ptr OrtMemoryInfo 77 | input_tensor : ptr OrtValue 78 | output_tensor : ptr OrtValue 79 | is_tensor : cint 80 | floatarrPtr: ptr cfloat 81 | 82 | output_node_names.add("softmaxout_1".cstring) 83 | # initialize input data with values in [0.0, 1.0] 84 | for i in 0 .. input_tensor_size - 1: 85 | input_tensor_values[i] = i.float / (input_tensor_size + 1).float 86 | 87 | # create input tensor object from data dim_values 88 | api.CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, memory_info.addr).checkStatus 89 | api.CreateTensorWithDataAsOrtValue(memory_info, cast[pointer](input_tensor_values[0].addr), (input_tensor_size * cfloat.sizeof).csize_t, cast[ptr int64](input_node_dims[0].addr), 4.csize_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, input_tensor.addr).checkStatus 90 | api.IsTensor(input_tensor, is_tensor.addr).checkStatus 91 | assert is_tensor == 1.cint 92 | api.ReleaseMemoryInfo(memory_info) 93 | 94 | # score model & input tensor, get back output tensor 95 | api.Run(session, cast[ptr OrtRunOptions](0), cast[ptr cstring](input_node_names[0].addr), cast[ptr ptr OrtValue](input_tensor.addr), 1.csize_t, cast[ptr cstring](output_node_names[0].addr), 1.csize_t, output_tensor.addr).checkStatus 96 | api.IsTensor(output_tensor, is_tensor.addr).checkStatus 97 | assert is_tensor == 1.cint 98 | 99 | api.GetTensorMutableData(output_tensor, cast[ptr pointer](floatarrPtr.addr)).checkStatus 100 | 101 | var floatarr = cast[ptr UncheckedArray[cfloat]](floatarrPtr) 102 | for i in 0 .. 4: 103 | echo fmt"[Class {i}] : {floatarr[i] : 0.6f}" 104 | 105 | api.ReleaseValue(output_tensor) 106 | api.ReleaseValue(input_tensor) 107 | api.ReleaseSession(session) 108 | api.ReleaseSessionOptions(session_options) 109 | api.ReleaseEnv(env) 110 | 111 | echo "Done!" -------------------------------------------------------------------------------- /sample/squeezenet.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YesDrX/onnxruntime-nim/90ead4efb09c5d897dd0696f6d04a9cbb64892cb/sample/squeezenet.onnx -------------------------------------------------------------------------------- /src/onnxruntime.nim: -------------------------------------------------------------------------------- 1 | {.deadCodeElim: on.} 2 | 3 | when defined(windows): 4 | const libname* = "onnxruntime.dll" 5 | elif defined(macosx): 6 | const libname* = "libonnxruntime.dylib" 7 | else: 8 | const libname* = "libonnxruntime.so" 9 | 10 | const 11 | ORT_API_VERSION* = 6 12 | 13 | type 14 | ONNXTensorElementDataType* {.size: sizeof(cint).} = enum 15 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, 16 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, 17 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, 18 | ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, 19 | ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, 20 | ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, 21 | ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, 22 | ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, 23 | ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, 24 | ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 25 | 26 | ONNXType* {.size: sizeof(cint).} = enum 27 | ONNX_TYPE_UNKNOWN, ONNX_TYPE_TENSOR, ONNX_TYPE_SEQUENCE, ONNX_TYPE_MAP, 28 | ONNX_TYPE_OPAQUE, ONNX_TYPE_SPARSETENSOR 29 | 30 | OrtLoggingLevel* {.size: sizeof(cint).} = enum 31 | ORT_LOGGING_LEVEL_VERBOSE, ORT_LOGGING_LEVEL_INFO, ORT_LOGGING_LEVEL_WARNING, 32 | ORT_LOGGING_LEVEL_ERROR, ORT_LOGGING_LEVEL_FATAL 33 | 34 | OrtErrorCode* {.size: sizeof(cint).} = enum 35 | ORT_OK, ORT_FAIL, ORT_INVALID_ARGUMENT, ORT_NO_SUCHFILE, ORT_NO_MODEL, 36 | ORT_ENGINE_ERROR, ORT_RUNTIME_EXCEPTION, ORT_INVALID_PROTOBUF, 37 | ORT_MODEL_LOADED, ORT_NOT_IMPLEMENTED, ORT_INVALID_GRAPH, ORT_EP_FAIL 38 | 39 | type 40 | OrtEnv* {.bycopy.} = object 41 | 42 | OrtStatus* {.bycopy.} = object 43 | 44 | OrtMemoryInfo* {.bycopy.} = object 45 | 46 | OrtIoBinding* {.bycopy.} = object 47 | 48 | OrtSession* {.bycopy.} = object 49 | 50 | OrtValue* {.bycopy.} = object 51 | 52 | OrtRunOptions* {.bycopy.} = object 53 | 54 | OrtTypeInfo* {.bycopy.} = object 55 | 56 | OrtTensorTypeAndShapeInfo* {.bycopy.} = object 57 | 58 | OrtSessionOptions* {.bycopy.} = object 59 | 60 | OrtCustomOpDomain* {.bycopy.} = object 61 | 62 | OrtMapTypeInfo* {.bycopy.} = object 63 | 64 | OrtSequenceTypeInfo* {.bycopy.} = object 65 | 66 | OrtModelMetadata* {.bycopy.} = object 67 | 68 | OrtThreadPoolParams* {.bycopy.} = object 69 | 70 | OrtThreadingOptions* {.bycopy.} = object 71 | 72 | OrtArenaCfg* {.bycopy.} = object 73 | 74 | OrtStatusPtr* = ptr OrtStatus 75 | 76 | OrtAllocator* {.bycopy.} = object 77 | version*: uint32 78 | Alloc*: proc (this: ptr OrtAllocator, size: csize_t): pointer {.cdecl.} 79 | Free*: proc (this: ptr OrtAllocator, p: pointer) {.cdecl.} 80 | Info*: proc (this: ptr OrtAllocator): ptr OrtMemoryInfo {.cdecl.} 81 | 82 | OrtLoggingFunction* = proc (param: pointer, severity: OrtLoggingLevel, category: cstring, logid: cstring,code_location: cstring, message: cstring) {.cdecl.} 83 | 84 | GraphOptimizationLevel* {.size: sizeof(cint).} = enum 85 | ORT_DISABLE_ALL = 0, ORT_ENABLE_BASIC = 1, ORT_ENABLE_EXTENDED = 2, 86 | ORT_ENABLE_ALL = 99 87 | 88 | ExecutionMode* {.size: sizeof(cint).} = enum 89 | ORT_SEQUENTIAL = 0, ORT_PARALLEL = 1 90 | 91 | OrtLanguageProjection* {.size: sizeof(cint).} = enum 92 | ORT_PROJECTION_C = 0, ORT_PROJECTION_CPLUSPLUS = 1, ORT_PROJECTION_CSHARP = 2, 93 | ORT_PROJECTION_PYTHON = 3, ORT_PROJECTION_JAVA = 4, ORT_PROJECTION_WINML = 5, 94 | ORT_PROJECTION_NODEJS = 6 95 | 96 | type 97 | OrtKernelInfo* {.bycopy.} = object 98 | 99 | OrtKernelContext* {.bycopy.} = object 100 | 101 | OrtAllocatorType* {.size: sizeof(cint).} = enum 102 | Invalid = -1, OrtDeviceAllocator = 0, OrtArenaAllocator = 1 103 | 104 | OrtMemType* {.size: sizeof(cint).} = enum 105 | OrtMemTypeCPUInput = -2, OrtMemTypeCPUOutput = -1, OrtMemTypeDefault = 0 106 | 107 | OrtCudnnConvAlgoSearch* {.size: sizeof(cint).} = enum 108 | EXHAUSTIVE, HEURISTIC, DEFAULT 109 | 110 | OrtCUDAProviderOptions* {.bycopy.} = object 111 | device_id*: cint 112 | cudnn_conv_algo_search*: OrtCudnnConvAlgoSearch 113 | cuda_mem_limit*: csize_t 114 | arena_extend_strategy*: cint 115 | do_copy_in_default_stream*: cint 116 | 117 | OrtOpenVINOProviderOptions* {.bycopy.} = object 118 | device_type*: cstring 119 | enable_vpu_fast_compile*: cuchar 120 | device_id*: cstring 121 | num_of_threads*: csize_t 122 | 123 | const 124 | OrtMemTypeCPU = OrtMemTypeCPUOutput 125 | 126 | type 127 | OrtApi* {.bycopy.} = object 128 | CreateStatus*: proc (code: OrtErrorCode, msg: cstring): ptr OrtStatus {.cdecl.} 129 | GetErrorCode*: proc (status: ptr OrtStatus): OrtErrorCode {.cdecl.} 130 | GetErrorMessage*: proc (status: ptr OrtStatus): cstring {.cdecl.} 131 | CreateEnv*: proc (logging_level: OrtLoggingLevel, logid: cstring,`out`: ptr ptr OrtEnv): OrtStatusPtr {.cdecl.} 132 | CreateEnvWithCustomLogger*: proc (logging_function: OrtLoggingFunction, logger_param: pointer, logging_level: OrtLoggingLevel, logid: cstring, `out`: ptr ptr OrtEnv): OrtStatusPtr {.cdecl.} 133 | EnableTelemetryEvents*: proc (env: ptr OrtEnv): OrtStatusPtr {.cdecl.} 134 | DisableTelemetryEvents*: proc (env: ptr OrtEnv): OrtStatusPtr {.cdecl.} 135 | CreateSession*: proc (env: ptr OrtEnv, model_path: cstring, options: ptr OrtSessionOptions, `out`: ptr ptr OrtSession): OrtStatusPtr {.cdecl.} 136 | CreateSessionFromArray*: proc (env: ptr OrtEnv, model_data: pointer, model_data_length: csize_t, options: ptr OrtSessionOptions, `out`: ptr ptr OrtSession): OrtStatusPtr {.cdecl.} 137 | Run*: proc (sess: ptr OrtSession, run_options: ptr OrtRunOptions, input_names: ptr cstring, input: ptr ptr OrtValue, input_len: csize_t, output_names1: ptr cstring, output_names_len: csize_t, output: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 138 | CreateSessionOptions*: proc (options: ptr ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 139 | SetOptimizedModelFilePath*: proc (options: ptr OrtSessionOptions, optimized_model_filepath: cstring): OrtStatusPtr {.cdecl.} 140 | CloneSessionOptions*: proc (in_options: ptr OrtSessionOptions, out_options: ptr ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 141 | SetSessionExecutionMode*: proc (options: ptr OrtSessionOptions, execution_mode: ExecutionMode): OrtStatusPtr {.cdecl.} 142 | EnableProfiling*: proc (options: ptr OrtSessionOptions, profile_file_prefix: cstring): OrtStatusPtr {.cdecl.} 143 | DisableProfiling*: proc (options: ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 144 | EnableMemPattern*: proc (options: ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 145 | DisableMemPattern*: proc (options: ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 146 | EnableCpuMemArena*: proc (options: ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 147 | DisableCpuMemArena*: proc (options: ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 148 | SetSessionLogId*: proc (options: ptr OrtSessionOptions, logid: cstring): OrtStatusPtr {.cdecl.} 149 | SetSessionLogVerbosityLevel*: proc (options: ptr OrtSessionOptions, session_log_verbosity_level: cint): OrtStatusPtr {.cdecl.} 150 | SetSessionLogSeverityLevel*: proc (options: ptr OrtSessionOptions, session_log_severity_level: cint): OrtStatusPtr {.cdecl.} 151 | SetSessionGraphOptimizationLevel*: proc (options: ptr OrtSessionOptions,graph_optimization_level: GraphOptimizationLevel): OrtStatusPtr {.cdecl.} 152 | SetIntraOpNumThreads*: proc (options: ptr OrtSessionOptions, intra_op_num_threads: cint): OrtStatusPtr {.cdecl.} 153 | SetInterOpNumThreads*: proc (options: ptr OrtSessionOptions, inter_op_num_threads: cint): OrtStatusPtr {.cdecl.} 154 | CreateCustomOpDomain*: proc (domain: cstring, `out`: ptr ptr OrtCustomOpDomain): OrtStatusPtr {.cdecl.} 155 | CustomOpDomain_Add*: proc (custom_op_domain: ptr OrtCustomOpDomain, op: ptr OrtCustomOp): OrtStatusPtr {.cdecl.} 156 | AddCustomOpDomain*: proc (options: ptr OrtSessionOptions, custom_op_domain: ptr OrtCustomOpDomain): OrtStatusPtr {.cdecl.} 157 | RegisterCustomOpsLibrary*: proc (options: ptr OrtSessionOptions, library_path: cstring, library_handle: ptr pointer): OrtStatusPtr {.cdecl.} 158 | SessionGetInputCount*: proc (sess: ptr OrtSession, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 159 | SessionGetOutputCount*: proc (sess: ptr OrtSession, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 160 | SessionGetOverridableInitializerCount*: proc (sess: ptr OrtSession,`out`: ptr csize_t): OrtStatusPtr {.cdecl.} 161 | SessionGetInputTypeInfo*: proc (sess: ptr OrtSession, index: csize_t, type_info: ptr ptr OrtTypeInfo): OrtStatusPtr {.cdecl.} 162 | SessionGetOutputTypeInfo*: proc (sess: ptr OrtSession, index: csize_t, type_info: ptr ptr OrtTypeInfo): OrtStatusPtr {.cdecl.} 163 | SessionGetOverridableInitializerTypeInfo*: proc (sess: ptr OrtSession,index: csize_t, type_info: ptr ptr OrtTypeInfo): OrtStatusPtr {.cdecl.} 164 | SessionGetInputName*: proc (sess: ptr OrtSession, index: csize_t, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 165 | SessionGetOutputName*: proc (sess: ptr OrtSession, index: csize_t, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 166 | SessionGetOverridableInitializerName*: proc (sess: ptr OrtSession,index: csize_t, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 167 | CreateRunOptions*: proc (`out`: ptr ptr OrtRunOptions): OrtStatusPtr {.cdecl.} 168 | RunOptionsSetRunLogVerbosityLevel*: proc (options: ptr OrtRunOptions, value: cint): OrtStatusPtr {.cdecl.} 169 | RunOptionsSetRunLogSeverityLevel*: proc (options: ptr OrtRunOptions, value: cint): OrtStatusPtr {.cdecl.} 170 | RunOptionsSetRunTag*: proc (a1: ptr OrtRunOptions, run_tag: cstring): OrtStatusPtr {.cdecl.} 171 | RunOptionsGetRunLogVerbosityLevel*: proc (options: ptr OrtRunOptions,`out`: ptr cint): OrtStatusPtr {.cdecl.} 172 | RunOptionsGetRunLogSeverityLevel*: proc (options: ptr OrtRunOptions,`out`: ptr cint): OrtStatusPtr {.cdecl.} 173 | RunOptionsGetRunTag*: proc (a1: ptr OrtRunOptions, `out`: ptr cstring): OrtStatusPtr {.cdecl.} 174 | RunOptionsSetTerminate*: proc (options: ptr OrtRunOptions): OrtStatusPtr {.cdecl.} 175 | RunOptionsUnsetTerminate*: proc (options: ptr OrtRunOptions): OrtStatusPtr {.cdecl.} 176 | CreateTensorAsOrtValue*: proc (allocator: ptr OrtAllocator, shape: ptr int64, shape_len: csize_t, `type`: ONNXTensorElementDataType, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 177 | CreateTensorWithDataAsOrtValue*: proc (info: ptr OrtMemoryInfo, p_data: pointer,p_data_len: csize_t, shape: ptr int64, shape_len: csize_t,`type`: ONNXTensorElementDataType, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 178 | IsTensor*: proc (value: ptr OrtValue, `out`: ptr cint): OrtStatusPtr {.cdecl.} 179 | GetTensorMutableData*: proc (value: ptr OrtValue, `out`: ptr pointer): OrtStatusPtr {.cdecl.} 180 | FillStringTensor*: proc (value: ptr OrtValue, s: ptr cstring, s_len: csize_t): OrtStatusPtr {.cdecl.} 181 | GetStringTensorDataLength*: proc (value: ptr OrtValue, len: ptr csize_t): OrtStatusPtr {.cdecl.} 182 | GetStringTensorContent*: proc (value: ptr OrtValue, s: pointer, s_len: csize_t, offsets: ptr csize_t, offsets_len: csize_t): OrtStatusPtr {.cdecl.} 183 | CastTypeInfoToTensorInfo*: proc (a1: ptr OrtTypeInfo, `out`: ptr ptr OrtTensorTypeAndShapeInfo): OrtStatusPtr {.cdecl.} 184 | GetOnnxTypeFromTypeInfo*: proc (a1: ptr OrtTypeInfo, `out`: ptr ONNXType): OrtStatusPtr {.cdecl.} 185 | CreateTensorTypeAndShapeInfo*: proc (`out`: ptr ptr OrtTensorTypeAndShapeInfo): OrtStatusPtr {.cdecl.} 186 | SetTensorElementType*: proc (a1: ptr OrtTensorTypeAndShapeInfo, `type`: ONNXTensorElementDataType): OrtStatusPtr {.cdecl.} 187 | SetDimensions*: proc (info: ptr OrtTensorTypeAndShapeInfo, dim_values: ptr int64, dim_count: csize_t): OrtStatusPtr {.cdecl.} 188 | GetTensorElementType*: proc (a1: ptr OrtTensorTypeAndShapeInfo, `out`: ptr ONNXTensorElementDataType): OrtStatusPtr {.cdecl.} 189 | GetDimensionsCount*: proc (info: ptr OrtTensorTypeAndShapeInfo, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 190 | GetDimensions*: proc (info: ptr OrtTensorTypeAndShapeInfo, dim_values: ptr int64, dim_values_length: csize_t): OrtStatusPtr {.cdecl.} 191 | GetSymbolicDimensions*: proc (info: ptr OrtTensorTypeAndShapeInfo, dim_params: ptr cstring, dim_params_length: csize_t): OrtStatusPtr {.cdecl.} 192 | GetTensorShapeElementCount*: proc (info: ptr OrtTensorTypeAndShapeInfo, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 193 | GetTensorTypeAndShape*: proc (value: ptr OrtValue, `out`: ptr ptr OrtTensorTypeAndShapeInfo): OrtStatusPtr {.cdecl.} 194 | GetTypeInfo*: proc (value: ptr OrtValue, `out`: ptr ptr OrtTypeInfo): OrtStatusPtr {.cdecl.} 195 | GetValueType*: proc (value: ptr OrtValue, `out`: ptr ONNXType): OrtStatusPtr {.cdecl.} 196 | CreateMemoryInfo*: proc (name1: cstring, `type`: OrtAllocatorType, id1: cint,mem_type1: OrtMemType, `out`: ptr ptr OrtMemoryInfo): OrtStatusPtr {.cdecl.} 197 | CreateCpuMemoryInfo*: proc (`type`: OrtAllocatorType, mem_type1: OrtMemType, `out`: ptr ptr OrtMemoryInfo): OrtStatusPtr {.cdecl.} 198 | CompareMemoryInfo*: proc (info1: ptr OrtMemoryInfo, info2: ptr OrtMemoryInfo, `out`: ptr cint): OrtStatusPtr {.cdecl.} 199 | MemoryInfoGetName*: proc (`ptr`: ptr OrtMemoryInfo, `out`: ptr cstring): OrtStatusPtr {.cdecl.} 200 | MemoryInfoGetId*: proc (`ptr`: ptr OrtMemoryInfo, `out`: ptr cint): OrtStatusPtr {.cdecl.} 201 | MemoryInfoGetMemType*: proc (`ptr`: ptr OrtMemoryInfo, `out`: ptr OrtMemType): OrtStatusPtr {.cdecl.} 202 | MemoryInfoGetType*: proc (`ptr`: ptr OrtMemoryInfo, `out`: ptr OrtAllocatorType): OrtStatusPtr {.cdecl.} 203 | AllocatorAlloc*: proc (`ptr`: ptr OrtAllocator, size: csize_t, `out`: ptr pointer): OrtStatusPtr {.cdecl.} 204 | AllocatorFree*: proc (`ptr`: ptr OrtAllocator, p: pointer): OrtStatusPtr {.cdecl.} 205 | AllocatorGetInfo*: proc (`ptr`: ptr OrtAllocator, `out`: ptr ptr OrtMemoryInfo): OrtStatusPtr {.cdecl.} 206 | GetAllocatorWithDefaultOptions*: proc (`out`: ptr ptr OrtAllocator): OrtStatusPtr {.cdecl.} 207 | AddFreeDimensionOverride*: proc (options: ptr OrtSessionOptions, dim_denotation: cstring, dim_value: int64): OrtStatusPtr {.cdecl.} 208 | GetValue*: proc (value: ptr OrtValue, index: cint, allocator: ptr OrtAllocator, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 209 | GetValueCount*: proc (value: ptr OrtValue, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 210 | CreateValue*: proc (`in`: ptr ptr OrtValue, num_values: csize_t, value_type: ONNXType, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 211 | CreateOpaqueValue*: proc (domain_name: cstring, type_name: cstring, data_container: pointer, data_container_size: csize_t, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 212 | GetOpaqueValue*: proc (domain_name: cstring, type_name: cstring, `in`: ptr OrtValue, data_container: pointer, data_container_size: csize_t): OrtStatusPtr {.cdecl.} 213 | KernelInfoGetAttribute_float*: proc (info: ptr OrtKernelInfo, name: cstring, `out`: ptr cfloat): OrtStatusPtr {.cdecl.} 214 | KernelInfoGetAttribute_int64*: proc (info: ptr OrtKernelInfo, name: cstring, `out`: ptr int64): OrtStatusPtr {.cdecl.} 215 | KernelInfoGetAttribute_string*: proc (info: ptr OrtKernelInfo, name: cstring, `out`: cstring, size: ptr csize_t): OrtStatusPtr {.cdecl.} 216 | KernelContext_GetInputCount*: proc (context: ptr OrtKernelContext, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 217 | KernelContext_GetOutputCount*: proc (context: ptr OrtKernelContext, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 218 | KernelContext_GetInput*: proc (context: ptr OrtKernelContext, index: csize_t, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 219 | KernelContext_GetOutput*: proc (context: ptr OrtKernelContext, index: csize_t, dim_values: ptr int64, dim_count: csize_t, `out`: ptr ptr OrtValue): OrtStatusPtr {.cdecl.} 220 | ReleaseEnv*: proc (input: ptr OrtEnv) {.cdecl.} 221 | ReleaseStatus*: proc (input: ptr OrtStatus) {.cdecl.} 222 | ReleaseMemoryInfo*: proc (input: ptr OrtMemoryInfo) {.cdecl.} 223 | ReleaseSession*: proc (input: ptr OrtSession) {.cdecl.} 224 | ReleaseValue*: proc (input: ptr OrtValue) {.cdecl.} 225 | ReleaseRunOptions*: proc (input: ptr OrtRunOptions) {.cdecl.} 226 | ReleaseTypeInfo*: proc (input: ptr OrtTypeInfo) {.cdecl.} 227 | ReleaseTensorTypeAndShapeInfo*: proc (input: ptr OrtTensorTypeAndShapeInfo) {.cdecl.} 228 | ReleaseSessionOptions*: proc (input: ptr OrtSessionOptions) {.cdecl.} 229 | ReleaseCustomOpDomain*: proc (input: ptr OrtCustomOpDomain) {.cdecl.} 230 | GetDenotationFromTypeInfo*: proc (a1: ptr OrtTypeInfo, denotation: ptr cstring, len: ptr csize_t): OrtStatusPtr {.cdecl.} 231 | CastTypeInfoToMapTypeInfo*: proc (type_info: ptr OrtTypeInfo, `out`: ptr ptr OrtMapTypeInfo): OrtStatusPtr {.cdecl.} 232 | CastTypeInfoToSequenceTypeInfo*: proc (type_info: ptr OrtTypeInfo,`out`: ptr ptr OrtSequenceTypeInfo): OrtStatusPtr {.cdecl.} 233 | GetMapKeyType*: proc (map_type_info: ptr OrtMapTypeInfo, `out`: ptr ONNXTensorElementDataType): OrtStatusPtr {.cdecl.} 234 | GetMapValueType*: proc (map_type_info: ptr OrtMapTypeInfo, type_info: ptr ptr OrtTypeInfo): OrtStatusPtr {.cdecl.} 235 | GetSequenceElementType*: proc (sequence_type_info: ptr OrtSequenceTypeInfo, type_info: ptr ptr OrtTypeInfo): OrtStatusPtr {.cdecl.} 236 | ReleaseMapTypeInfo*: proc (input: ptr OrtMapTypeInfo) {.cdecl.} 237 | ReleaseSequenceTypeInfo*: proc (input: ptr OrtSequenceTypeInfo) {.cdecl.} 238 | SessionEndProfiling*: proc (sess: ptr OrtSession, allocator: ptr OrtAllocator, `out`: ptr cstring): OrtStatusPtr {.cdecl.} 239 | SessionGetModelMetadata*: proc (sess: ptr OrtSession, `out`: ptr ptr OrtModelMetadata): OrtStatusPtr {.cdecl.} 240 | ModelMetadataGetProducerName*: proc (model_metadata: ptr OrtModelMetadata, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 241 | ModelMetadataGetGraphName*: proc (model_metadata: ptr OrtModelMetadata, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 242 | ModelMetadataGetDomain*: proc (model_metadata: ptr OrtModelMetadata, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 243 | ModelMetadataGetDescription*: proc (model_metadata: ptr OrtModelMetadata, allocator: ptr OrtAllocator, value: ptr cstring): OrtStatusPtr {.cdecl.} 244 | ModelMetadataLookupCustomMetadataMap*: proc (model_metadata: ptr OrtModelMetadata, allocator: ptr OrtAllocator,key: cstring, value: ptr cstring): OrtStatusPtr {.cdecl.} 245 | ModelMetadataGetVersion*: proc (model_metadata: ptr OrtModelMetadata, value: ptr int64): OrtStatusPtr {.cdecl.} 246 | ReleaseModelMetadata*: proc (input: ptr OrtModelMetadata) {.cdecl.} 247 | CreateEnvWithGlobalThreadPools*: proc (logging_level: OrtLoggingLevel,logid: cstring, t_options: ptr OrtThreadingOptions, `out`: ptr ptr OrtEnv): OrtStatusPtr {.cdecl.} 248 | DisablePerSessionThreads*: proc (options: ptr OrtSessionOptions): OrtStatusPtr {.cdecl.} 249 | CreateThreadingOptions*: proc (`out`: ptr ptr OrtThreadingOptions): OrtStatusPtr {.cdecl.} 250 | ReleaseThreadingOptions*: proc (input: ptr OrtThreadingOptions) {.cdecl.} 251 | ModelMetadataGetCustomMetadataMapKeys*: proc (model_metadata: ptr OrtModelMetadata, allocator: ptr OrtAllocator,keys: ptr ptr cstring, num_keys: ptr int64): OrtStatusPtr {.cdecl.} 252 | AddFreeDimensionOverrideByName*: proc (options: ptr OrtSessionOptions,dim_name: cstring, dim_value: int64): OrtStatusPtr {.cdecl.} 253 | GetAvailableProviders*: proc (out_ptr: ptr ptr cstring, provider_length: ptr cint): OrtStatusPtr {.cdecl.} 254 | ReleaseAvailableProviders*: proc (`ptr`: ptr cstring, providers_length: cint): OrtStatusPtr {.cdecl.} 255 | GetStringTensorElementLength*: proc (value: ptr OrtValue, index: csize_t, `out`: ptr csize_t): OrtStatusPtr {.cdecl.} 256 | GetStringTensorElement*: proc (value: ptr OrtValue, s_len: csize_t, index: csize_t, s: pointer): OrtStatusPtr {.cdecl.} 257 | FillStringTensorElement*: proc (value: ptr OrtValue, s: cstring, index: csize_t): OrtStatusPtr {.cdecl.} 258 | AddSessionConfigEntry*: proc (options: ptr OrtSessionOptions, config_key: cstring, config_value: cstring): OrtStatusPtr {.cdecl.} 259 | CreateAllocator*: proc (sess: ptr OrtSession, mem_info: ptr OrtMemoryInfo, `out`: ptr ptr OrtAllocator): OrtStatusPtr {.cdecl.} 260 | ReleaseAllocator*: proc (input: ptr OrtAllocator) {.cdecl.} 261 | RunWithBinding*: proc (sess: ptr OrtSession, run_options: ptr OrtRunOptions, binding_ptr: ptr OrtIoBinding): OrtStatusPtr {.cdecl.} 262 | CreateIoBinding*: proc (sess: ptr OrtSession, `out`: ptr ptr OrtIoBinding): OrtStatusPtr {.cdecl.} 263 | ReleaseIoBinding*: proc (input: ptr OrtIoBinding) {.cdecl.} 264 | BindInput*: proc (binding_ptr: ptr OrtIoBinding, name: cstring,val_ptr: ptr OrtValue): OrtStatusPtr {.cdecl.} 265 | BindOutput*: proc (binding_ptr: ptr OrtIoBinding, name: cstring, val_ptr: ptr OrtValue): OrtStatusPtr {.cdecl.} 266 | BindOutputToDevice*: proc (binding_ptr: ptr OrtIoBinding, name: cstring, val_ptr: ptr OrtMemoryInfo): OrtStatusPtr {.cdecl.} 267 | GetBoundOutputNames*: proc (binding_ptr: ptr OrtIoBinding, allocator: ptr OrtAllocator, buffer: ptr cstring, lengths: ptr ptr csize_t, count: ptr csize_t): OrtStatusPtr {.cdecl.} 268 | GetBoundOutputValues*: proc (binding_ptr: ptr OrtIoBinding, allocator: ptr OrtAllocator, output: ptr ptr ptr OrtValue, output_count: ptr csize_t): OrtStatusPtr {.cdecl.} 269 | ClearBoundInputs*: proc (binding_ptr: ptr OrtIoBinding) {.cdecl.} 270 | ClearBoundOutputs*: proc (binding_ptr: ptr OrtIoBinding) {.cdecl.} 271 | TensorAt*: proc (value: ptr OrtValue, location_values: ptr int64, location_values_count: csize_t, `out`: ptr pointer): OrtStatusPtr {.cdecl.} 272 | CreateAndRegisterAllocator*: proc (env: ptr OrtEnv, mem_info: ptr OrtMemoryInfo, arena_cfg: ptr OrtArenaCfg): OrtStatusPtr {.cdecl.} 273 | SetLanguageProjection*: proc (ort_env: ptr OrtEnv, projection: OrtLanguageProjection): OrtStatusPtr {.cdecl.} 274 | SessionGetProfilingStartTimeNs*: proc (sess: ptr OrtSession, `out`: ptr uint64): OrtStatusPtr {.cdecl.} 275 | SetGlobalIntraOpNumThreads*: proc (tp_options: ptr OrtThreadingOptions, intra_op_num_threads: cint): OrtStatusPtr {.cdecl.} 276 | SetGlobalInterOpNumThreads*: proc (tp_options: ptr OrtThreadingOptions, inter_op_num_threads: cint): OrtStatusPtr {.cdecl.} 277 | SetGlobalSpinControl*: proc (tp_options: ptr OrtThreadingOptions, allow_spinning: cint): OrtStatusPtr {.cdecl.} 278 | AddInitializer*: proc (options: ptr OrtSessionOptions, name: cstring, val: ptr OrtValue): OrtStatusPtr {.cdecl.} 279 | CreateEnvWithCustomLoggerAndGlobalThreadPools*: proc (logging_function: OrtLoggingFunction, logger_param: pointer,logging_level: OrtLoggingLevel, logid: cstring,tp_options: ptr OrtThreadingOptions, `out`: ptr ptr OrtEnv): OrtStatusPtr {.cdecl.} 280 | SessionOptionsAppendExecutionProvider_CUDA*: proc (options: ptr OrtSessionOptions, cuda_options: ptr OrtCUDAProviderOptions): OrtStatusPtr {.cdecl.} 281 | SessionOptionsAppendExecutionProvider_OpenVINO*: proc (options: ptr OrtSessionOptions,provider_options: ptr OrtOpenVINOProviderOptions): OrtStatusPtr {.cdecl.} 282 | SetGlobalDenormalAsZero*: proc (tp_options: ptr OrtThreadingOptions): OrtStatusPtr {.cdecl.} 283 | CreateArenaCfg*: proc (max_mem: csize_t, arena_extend_strategy: cint, initial_chunk_size_bytes: cint, max_dead_bytes_per_chunk: cint, `out`: ptr ptr OrtArenaCfg): OrtStatusPtr {.cdecl.} 284 | ReleaseArenaCfg*: proc (input: ptr OrtArenaCfg) {.cdecl.} 285 | 286 | OrtCustomOp* {.bycopy.} = object 287 | version*: uint32 288 | CreateKernel*: proc (op: ptr OrtCustomOp, api: ptr OrtApi, info: ptr OrtKernelInfo): pointer {.cdecl.} 289 | GetName*: proc (op: ptr OrtCustomOp): cstring {.cdecl.} 290 | GetExecutionProviderType*: proc (op: ptr OrtCustomOp): cstring {.cdecl.} 291 | GetInputType*: proc (op: ptr OrtCustomOp, index: csize_t): ONNXTensorElementDataType {.cdecl.} 292 | GetInputTypeCount*: proc (op: ptr OrtCustomOp): csize_t {.cdecl.} 293 | GetOutputType*: proc (op: ptr OrtCustomOp, index: csize_t): ONNXTensorElementDataType {.cdecl.} 294 | GetOutputTypeCount*: proc (op: ptr OrtCustomOp): csize_t {.cdecl.} 295 | KernelCompute*: proc (op_kernel: pointer, context: ptr OrtKernelContext) {.cdecl.} 296 | KernelDestroy*: proc (op_kernel: pointer) {.cdecl.} 297 | 298 | type 299 | OrtApiBase* {.bycopy.} = object 300 | GetApi*: proc (version: uint32): ptr OrtApi {.cdecl.} 301 | GetVersionString*: proc (): cstring {.cdecl.} 302 | 303 | proc OrtGetApiBase*(): ptr OrtApiBase {.cdecl, importc: "OrtGetApiBase", dynlib: libname.} 304 | 305 | let 306 | apiBase = OrtGetApiBase() 307 | onxApi* = apiBase.GetApi(ORT_API_VERSION) 308 | 309 | echo "Using Onnxruntime C Api : " & $apiBase.GetVersionString() --------------------------------------------------------------------------------