├── .gitignore ├── LICENSE ├── README.md ├── c2numpy.h ├── commonblock ├── DemoAnalyzer.cc ├── NumpyCommonBlock.h ├── commonblock-demo.ipynb ├── commonblock.py └── pythonrun.py ├── examples └── CMSSW-with-C-interface │ ├── Demo │ └── DemoAnalyzer │ │ ├── interface │ │ └── c2numpy.h │ │ ├── plugins │ │ ├── BuildFile.xml │ │ └── DemoAnalyzer.cc │ │ └── python │ │ ├── CfiFile_cfi.py │ │ ├── ConfFile_cfg.py │ │ └── __init__.py │ ├── README.md │ └── output │ ├── trackparams0.npy │ ├── trackparams1.npy │ ├── trackparams2.npy │ ├── trackparams3.npy │ ├── trackparams4.npy │ ├── trackparams5.npy │ └── trackparams6.npy └── test.c /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # c2numpy 2 | 3 | Write Numpy (.npy) files from C++ for analysis in [Numpy](http://www.numpy.org/), [Scipy](https://www.scipy.org/), [Scikit-Learn](http://scikit-learn.org/stable/), [Pandas](http://pandas.pydata.org/), etc. 4 | 5 | Fills a collection of .npy files with a maximum size, a common prefix and a rotating number (like rotating log files). Each file contains one [structured array](http://docs.scipy.org/doc/numpy/user/basics.rec.html), consisting of named, typed columns (numbers and fixed-size strings) and many rows. In Python, you access rows and columns with string and integer indexing: 6 | 7 | ```python 8 | myarray["column1"] # one column, all rows 9 | myarray[3:5] # all columns, slice of rows 10 | # etc. 11 | ``` 12 | 13 | This project does not support _reading_ of Numpy files in C++. 14 | 15 | ## Installation 16 | 17 | Put `c2numpy.h` in your C++ project and compile. No libraries are required. Earlier versions of this worked with strict C99, but this project now requires C++. 18 | 19 | For an example and testing, `test.c` is provided. Compile and run it with 20 | 21 | ```bash 22 | gcc -std=c99 test.c -o testme && ./testme 23 | ``` 24 | 25 | and view the results with 26 | 27 | ```bash 28 | python -c "import numpy; print numpy.load(open('testout0.npy'));" 29 | python -c "import numpy; print numpy.load(open('testout1.npy'));" 30 | ``` 31 | 32 | ## C++ example 33 | 34 | ```c++ 35 | // declare writer 36 | c2numpy_writer writer; 37 | 38 | // initialize it and add columns 39 | c2numpy_init(&writer, "testout", 5); 40 | c2numpy_addcolumn(&writer, "one", C2NUMPY_INTC); 41 | c2numpy_addcolumn(&writer, "two", C2NUMPY_FLOAT64); 42 | c2numpy_addcolumn(&writer, "three", C2NUMPY_STRING + 5); 43 | 44 | // first row 45 | c2numpy_intc(&writer, 1); 46 | c2numpy_float64(&writer, 1.1); 47 | c2numpy_string(&writer, "ONE"); 48 | 49 | // second row 50 | c2numpy_intc(&writer, 2); 51 | c2numpy_float64(&writer, 2.2); 52 | c2numpy_string(&writer, "TWO"); 53 | 54 | // third row 55 | c2numpy_intc(&writer, 3); 56 | c2numpy_float64(&writer, 3.3); 57 | c2numpy_string(&writer, "THREE"); 58 | 59 | // close file 60 | c2numpy_close(&writer); 61 | ``` 62 | 63 | ## C-like API 64 | 65 | The original version of this project could be used in pure C projects, and hence it has a pure C API. However, now that the internals require C++, this API will be replaced by a C++ API. This documentation will always be in sync with the codebase (in the same branch of GitHub). 66 | 67 | ### Enumeration constants for Numpy types: `c2numpy_type` 68 | 69 | See [number type definitions](http://docs.scipy.org/doc/numpy/user/basics.types.html) in the Numpy documentation. 70 | 71 | ```c++ 72 | C2NUMPY_BOOL // Boolean (True or False) stored as a byte 73 | C2NUMPY_INT // Default integer type (same as C long; normally either int64 or int32) 74 | C2NUMPY_INTC // Identical to C int (normally int32 or int64) 75 | C2NUMPY_INTP // Integer used for indexing (same as C ssize_t; normally either int32 or int64) 76 | C2NUMPY_INT8 // Byte (-128 to 127) 77 | C2NUMPY_INT16 // Integer (-32768 to 32767) 78 | C2NUMPY_INT32 // Integer (-2147483648 to 2147483647) 79 | C2NUMPY_INT64 // Integer (-9223372036854775808 to 9223372036854775807) 80 | C2NUMPY_UINT8 // Unsigned integer (0 to 255) 81 | C2NUMPY_UINT16 // Unsigned integer (0 to 65535) 82 | C2NUMPY_UINT32 // Unsigned integer (0 to 4294967295) 83 | C2NUMPY_UINT64 // Unsigned integer (0 to 18446744073709551615) 84 | C2NUMPY_FLOAT // Shorthand for float64. 85 | C2NUMPY_FLOAT16 // Half precision float: sign bit, 5 bits exponent, 10 bits mantissa 86 | C2NUMPY_FLOAT32 // Single precision float: sign bit, 8 bits exponent, 23 bits mantissa 87 | C2NUMPY_FLOAT64 // Double precision float: sign bit, 11 bits exponent, 52 bits mantissa 88 | C2NUMPY_COMPLEX // Shorthand for complex128. 89 | C2NUMPY_COMPLEX64 // Complex number, represented by two 32-bit floats (real and imaginary components) 90 | C2NUMPY_COMPLEX128 // Complex number, represented by two 64-bit floats (real and imaginary components) 91 | C2NUMPY_STRING = 100 // strings are C2NUMPY_STRING + their fixed size (up to 155) 92 | ``` 93 | 94 | Strings are fixed-width only, so the type for strings with 12 characters is `C2NUMPY_STRING + 12`. 95 | 96 | Not currently supported: 97 | 98 | * `C2NUMPY_FLOAT16` 99 | * `C2NUMPY_COMPLEX` 100 | * `C2NUMPY_COMPLEX64` 101 | * `C2NUMPY_COMPLEX128` 102 | * strings 155 characters or longer 103 | 104 | ### Writer object: `c2numpy_writer` 105 | 106 | A writer contains the following fields. Some of them are internal, and all of them should be treated as read-only. Use the associated functions to manipulate. 107 | 108 | ```c++ 109 | typedef struct { 110 | char buffer[16]; // (internal) used for temporary copies in c2numpy_row 111 | 112 | FILE *file; // output file handle 113 | std::string outputFilePrefix; // output file name, not including the rotating number and .npy 114 | int64_t sizeSeekPosition; // (internal) keep track of number of rows to modify before closing 115 | int64_t sizeSeekSize; // (internal) 116 | 117 | int32_t numColumns; // number of columns in the record array 118 | std::vector columnNames; // column names 119 | std::vector columnTypes; // column types 120 | 121 | int32_t numRowsPerFile; // maximum number of rows per file 122 | int32_t currentColumn; // current column number 123 | int32_t currentRowInFile; // current row number in the current file 124 | int32_t currentFileNumber; // current file number 125 | } c2numpy_writer; 126 | ``` 127 | 128 | ### Numpy description string from type: `c2numpy_descr` 129 | 130 | ```c++ 131 | const char *c2numpy_descr(c2numpy_type type); 132 | ``` 133 | 134 | Rarely needed by typical users; converts a `c2numpy_type` to the corresponding Numpy "descr" string. **Returns** `NULL` if the `type` is invalid. 135 | 136 | ### Initialize a writer object: `c2numpy_init` 137 | 138 | ```c++ 139 | int c2numpy_init(c2numpy_writer *writer, const char *outputFilePrefix, int32_t numRowsPerFile); 140 | ``` 141 | 142 | This is the first function you should call on a new writer. After this, call `c2numpy_addcolumn` to add column descriptions. 143 | 144 | * `writer`: a new writer object. 145 | * `outputFilePrefix`: name of the output files, not including the rotating file number or the `.npy` suffix. This can include directories. 146 | * `numRowsPerFile`: number of rows to write before starting a new file. 147 | * **returns:** 0 if successful, -1 otherwise 148 | 149 | **Copies** the string `outputFilePrefix`, so you are responsible for deleting the original if necessary. 150 | 151 | ### Add a column to the writer: `c2numpy_addcolumn` 152 | 153 | ```c++ 154 | int c2numpy_addcolumn(c2numpy_writer *writer, const char *name, c2numpy_type type); 155 | ``` 156 | 157 | This is the second function you should call on a new writer. Call it once for each column you wish to add. 158 | 159 | * `writer`: the writer object, already initialized. 160 | * `name`: the name of the column to add. 161 | * `type`: the type of the column to add (see enumeration constants above). 162 | * **returns:** 0 if successful, -1 otherwise 163 | 164 | **Copies** the string `name`, so you are responsible for deleting the original if necessary. 165 | 166 | ### Optional open file: `c2numpy_open` 167 | 168 | ```c++ 169 | int c2numpy_open(c2numpy_writer *writer); 170 | ``` 171 | 172 | Open a file and write its header to disk. If you don't call this explicitly, writing data will automatically open the first file. You might want to call it before an expensive initialization to learn about I/O errors early, though. 173 | 174 | **Returns:** 0 if successful and -1 otherwise. 175 | 176 | ### Write an item of data: `c2numpy_*` 177 | 178 | The following suite of functions push one datum (item in a row/column) to the writer. They check the requested data type against the expected data type for the current column, but cannot prevent column-misalignment if all data types are the same. 179 | 180 | ```c++ 181 | int c2numpy_bool(c2numpy_writer *writer, int8_t data); // "bool" is just a byte 182 | int c2numpy_int(c2numpy_writer *writer, int64_t data); // Numpy's default int is 64-bit 183 | int c2numpy_intc(c2numpy_writer *writer, int data); // the built-in C int 184 | int c2numpy_intp(c2numpy_writer *writer, size_t data); // intp is Numpy's way of saying size_t 185 | int c2numpy_int8(c2numpy_writer *writer, int8_t data); 186 | int c2numpy_int16(c2numpy_writer *writer, int16_t data); 187 | int c2numpy_int32(c2numpy_writer *writer, int32_t data); 188 | int c2numpy_int64(c2numpy_writer *writer, int64_t data); 189 | int c2numpy_uint8(c2numpy_writer *writer, uint8_t data); 190 | int c2numpy_uint16(c2numpy_writer *writer, uint16_t data); 191 | int c2numpy_uint32(c2numpy_writer *writer, uint32_t data); 192 | int c2numpy_uint64(c2numpy_writer *writer, uint64_t data); 193 | int c2numpy_float(c2numpy_writer *writer, double data); // Numpy's "float" is a double 194 | // int c2numpy_float16(c2numpy_writer *writer, ??? data); // how to do float16 in C? 195 | int c2numpy_float32(c2numpy_writer *writer, float data); 196 | int c2numpy_float64(c2numpy_writer *writer, double data); 197 | // int c2numpy_complex(c2numpy_writer *writer, ??? data); // how to do complex in C? 198 | // int c2numpy_complex64(c2numpy_writer *writer, ??? data); 199 | // int c2numpy_complex128(c2numpy_writer *writer, ??? data); 200 | int c2numpy_string(c2numpy_writer *writer, const char *data); 201 | ``` 202 | 203 | **Returns:** 0 if successful and -1 otherwise. 204 | 205 | The string form, `c2numpy_string`, **only writes** the string `data`, so you are responsible for deleting the original if necessary. The full width of the string is written every time, even if this means writing uninitialized data past a termination character or truncating the string before its termination character. 206 | 207 | ### Required close file: `c2numpy_close` 208 | 209 | ```c++ 210 | int c2numpy_close(c2numpy_writer *writer); 211 | ``` 212 | 213 | If you do not explicitly close the writer, your last file may be corrupted. Be sure to do this after your loop over data. 214 | 215 | **Returns:** 0 if successful and -1 otherwise. 216 | 217 | ## To do 218 | 219 | * Add convenience function to calculate number of rows for a target file size. 220 | * Compressed (.npz) files. 221 | * System independence (currently assumes little endian with 32-bit `int` and 64-bit `size_t`). 222 | * Faster guessing of header size and column types. 223 | * Float16 and complex numbers. 224 | * Distinct return values for different errors and documentation of those errors. 225 | * C++ API. 226 | -------------------------------------------------------------------------------- /c2numpy.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Jim Pivarski 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef C2NUMPY 16 | #define C2NUMPY 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | const char* C2NUMPY_VERSION = "1.2"; 27 | 28 | // http://docs.scipy.org/doc/numpy/user/basics.types.html 29 | typedef enum { 30 | C2NUMPY_BOOL, // Boolean (True or False) stored as a byte 31 | C2NUMPY_INT, // Default integer type (same as C long; normally either int64 or int32) 32 | C2NUMPY_INTC, // Identical to C int (normally int32 or int64) 33 | C2NUMPY_INTP, // Integer used for indexing (same as C ssize_t; normally either int32 or int64) 34 | C2NUMPY_INT8, // Byte (-128 to 127) 35 | C2NUMPY_INT16, // Integer (-32768 to 32767) 36 | C2NUMPY_INT32, // Integer (-2147483648 to 2147483647) 37 | C2NUMPY_INT64, // Integer (-9223372036854775808 to 9223372036854775807) 38 | C2NUMPY_UINT8, // Unsigned integer (0 to 255) 39 | C2NUMPY_UINT16, // Unsigned integer (0 to 65535) 40 | C2NUMPY_UINT32, // Unsigned integer (0 to 4294967295) 41 | C2NUMPY_UINT64, // Unsigned integer (0 to 18446744073709551615) 42 | C2NUMPY_FLOAT, // Shorthand for float64. 43 | C2NUMPY_FLOAT16, // Half precision float: sign bit, 5 bits exponent, 10 bits mantissa 44 | C2NUMPY_FLOAT32, // Single precision float: sign bit, 8 bits exponent, 23 bits mantissa 45 | C2NUMPY_FLOAT64, // Double precision float: sign bit, 11 bits exponent, 52 bits mantissa 46 | C2NUMPY_COMPLEX, // Shorthand for complex128. 47 | C2NUMPY_COMPLEX64, // Complex number, represented by two 32-bit floats (real and imaginary components) 48 | C2NUMPY_COMPLEX128, // Complex number, represented by two 64-bit floats (real and imaginary components) 49 | 50 | C2NUMPY_STRING = 100, // strings are C2NUMPY_STRING + their fixed size (up to 155) 51 | C2NUMPY_END = 255 // ensure that c2numpy_type is at least a byte 52 | } c2numpy_type; 53 | 54 | // a Numpy writer object 55 | typedef struct { 56 | FILE *file; // output file handle 57 | std::string outputFilePrefix; // output file name, not including the rotating number and .npy 58 | int64_t sizeSeekPosition; // (internal) keep track of number of rows to modify before closing 59 | int64_t sizeSeekSize; // (internal) 60 | 61 | int32_t numColumns; // number of columns in the record array 62 | std::vector columnNames; // column names 63 | std::vector columnTypes; // column types 64 | 65 | int32_t numRowsPerFile; // maximum number of rows per file 66 | int32_t currentColumn; // current column number 67 | int32_t currentRowInFile; // current row number in the current file 68 | int32_t currentFileNumber; // current file number 69 | } c2numpy_writer; 70 | 71 | const char *c2numpy_descr(c2numpy_type type) { 72 | // FIXME: all of the "<" signs should be system-dependent (they mean little endian) 73 | static const char *c2numpy_bool = "|b1"; 74 | static const char *c2numpy_int = "file = NULL; 144 | writer->outputFilePrefix = outputFilePrefix; 145 | writer->sizeSeekPosition = 0; 146 | writer->sizeSeekSize = 0; 147 | 148 | writer->numColumns = 0; 149 | 150 | writer->numRowsPerFile = numRowsPerFile; 151 | writer->currentColumn = 0; 152 | writer->currentRowInFile = 0; 153 | writer->currentFileNumber = 0; 154 | 155 | return 0; 156 | } 157 | 158 | int c2numpy_addcolumn(c2numpy_writer *writer, const std::string name, c2numpy_type type) { 159 | writer->numColumns += 1; 160 | writer->columnNames.push_back(name); 161 | writer->columnTypes.push_back(type); 162 | return 0; 163 | } 164 | 165 | int c2numpy_open(c2numpy_writer *writer) { 166 | std::stringstream fileNameStream; 167 | fileNameStream << writer->outputFilePrefix; 168 | fileNameStream << writer->currentFileNumber; 169 | fileNameStream << ".npy"; 170 | std::string fileName = fileNameStream.str(); 171 | writer->file = fopen(fileName.c_str(), "wb"); 172 | 173 | std::stringstream headerStream; 174 | headerStream << "{'descr': ["; 175 | 176 | int column; 177 | for (column = 0; column < writer->numColumns; ++column) { 178 | headerStream << "('" << writer->columnNames[column] << "', '" << c2numpy_descr(writer->columnTypes[column]) << "')"; 179 | if (column < writer->numColumns - 1) 180 | headerStream << ", "; 181 | } 182 | 183 | headerStream << "], 'fortran_order': False, 'shape': ("; 184 | 185 | writer->sizeSeekPosition = headerStream.str().size(); 186 | 187 | headerStream << writer->numRowsPerFile; 188 | 189 | writer->sizeSeekSize = headerStream.str().size() - writer->sizeSeekPosition; 190 | 191 | headerStream << ",), }"; 192 | 193 | int headerSize = headerStream.str().size(); 194 | char version = 1; 195 | 196 | if (headerSize > 65535) version = 2; 197 | while ((6 + 2 + (version == 1 ? 2 : 4) + headerSize) % 16 != 0) { 198 | headerSize += 1; 199 | headerStream << " "; 200 | if (headerSize > 65535) version = 2; 201 | } 202 | 203 | fwrite("\x93NUMPY", 1, 6, writer->file); 204 | if (version == 1) { 205 | fwrite("\x01\x00", 1, 2, writer->file); 206 | fwrite(&headerSize, 1, 2, writer->file); 207 | writer->sizeSeekPosition += 6 + 2 + 2; 208 | } 209 | else { 210 | fwrite("\x02\x00", 1, 2, writer->file); 211 | fwrite(&headerSize, 1, 4, writer->file); 212 | writer->sizeSeekPosition += 6 + 2 + 4; 213 | } 214 | 215 | std::string header = headerStream.str(); 216 | fwrite(header.c_str(), 1, header.size(), writer->file); 217 | 218 | return 0; 219 | } 220 | 221 | #define C2NUMPY_CHECK_ITEM { \ 222 | if (writer->file == NULL) { \ 223 | int status = c2numpy_open(writer); \ 224 | if (status != 0) \ 225 | return status; \ 226 | } \ 227 | } 228 | 229 | #define C2NUMPY_INCREMENT_ITEM { \ 230 | if (writer->currentColumn == 0) { \ 231 | writer->currentRowInFile += 1; \ 232 | if (writer->currentRowInFile == writer->numRowsPerFile) { \ 233 | fclose(writer->file); \ 234 | writer->file = NULL; \ 235 | writer->currentRowInFile = 0; \ 236 | writer->currentFileNumber += 1; \ 237 | } \ 238 | } \ 239 | return 0; \ 240 | } 241 | 242 | int c2numpy_bool(c2numpy_writer *writer, int8_t data) { // "bool" is just a byte 243 | C2NUMPY_CHECK_ITEM 244 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_BOOL) return -1; 245 | fwrite(&data, sizeof(int8_t), 1, writer->file); 246 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 247 | C2NUMPY_INCREMENT_ITEM 248 | } 249 | 250 | int c2numpy_int(c2numpy_writer *writer, int64_t data) { // Numpy's default int is 64-bit 251 | C2NUMPY_CHECK_ITEM 252 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT) return -1; 253 | fwrite(&data, sizeof(int64_t), 1, writer->file); 254 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 255 | C2NUMPY_INCREMENT_ITEM 256 | } 257 | 258 | int c2numpy_intc(c2numpy_writer *writer, int data) { // the built-in C int 259 | C2NUMPY_CHECK_ITEM 260 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INTC) return -1; 261 | fwrite(&data, sizeof(int), 1, writer->file); 262 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 263 | C2NUMPY_INCREMENT_ITEM 264 | } 265 | 266 | int c2numpy_intp(c2numpy_writer *writer, size_t data) { // intp is Numpy's way of saying size_t 267 | C2NUMPY_CHECK_ITEM 268 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INTP) return -1; 269 | fwrite(&data, sizeof(size_t), 1, writer->file); 270 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 271 | C2NUMPY_INCREMENT_ITEM 272 | } 273 | 274 | int c2numpy_int8(c2numpy_writer *writer, int8_t data) { 275 | C2NUMPY_CHECK_ITEM 276 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT8) return -1; 277 | fwrite(&data, sizeof(int8_t), 1, writer->file); 278 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 279 | C2NUMPY_INCREMENT_ITEM 280 | } 281 | 282 | int c2numpy_int16(c2numpy_writer *writer, int16_t data) { 283 | C2NUMPY_CHECK_ITEM 284 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT16) return -1; 285 | fwrite(&data, sizeof(int16_t), 1, writer->file); 286 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 287 | C2NUMPY_INCREMENT_ITEM 288 | } 289 | 290 | int c2numpy_int32(c2numpy_writer *writer, int32_t data) { 291 | C2NUMPY_CHECK_ITEM 292 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT32) return -1; 293 | fwrite(&data, sizeof(int32_t), 1, writer->file); 294 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 295 | C2NUMPY_INCREMENT_ITEM 296 | } 297 | 298 | int c2numpy_int64(c2numpy_writer *writer, int64_t data) { 299 | C2NUMPY_CHECK_ITEM 300 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT64) return -1; 301 | fwrite(&data, sizeof(int64_t), 1, writer->file); 302 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 303 | C2NUMPY_INCREMENT_ITEM 304 | } 305 | 306 | int c2numpy_uint8(c2numpy_writer *writer, uint8_t data) { 307 | C2NUMPY_CHECK_ITEM 308 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT8) return -1; 309 | fwrite(&data, sizeof(uint8_t), 1, writer->file); 310 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 311 | C2NUMPY_INCREMENT_ITEM 312 | } 313 | 314 | int c2numpy_uint16(c2numpy_writer *writer, uint16_t data) { 315 | C2NUMPY_CHECK_ITEM 316 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT16) return -1; 317 | fwrite(&data, sizeof(uint16_t), 1, writer->file); 318 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 319 | C2NUMPY_INCREMENT_ITEM 320 | } 321 | 322 | int c2numpy_uint32(c2numpy_writer *writer, uint32_t data) { 323 | C2NUMPY_CHECK_ITEM 324 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT32) return -1; 325 | fwrite(&data, sizeof(uint32_t), 1, writer->file); 326 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 327 | C2NUMPY_INCREMENT_ITEM 328 | } 329 | 330 | int c2numpy_uint64(c2numpy_writer *writer, uint64_t data) { 331 | C2NUMPY_CHECK_ITEM 332 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT64) return -1; 333 | fwrite(&data, sizeof(uint64_t), 1, writer->file); 334 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 335 | C2NUMPY_INCREMENT_ITEM 336 | } 337 | 338 | int c2numpy_float(c2numpy_writer *writer, double data) { // Numpy's "float" is a double 339 | C2NUMPY_CHECK_ITEM 340 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT) return -1; 341 | fwrite(&data, sizeof(double), 1, writer->file); 342 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 343 | C2NUMPY_INCREMENT_ITEM 344 | } 345 | 346 | // int c2numpy_float16(c2numpy_writer *writer, ??? data) { // how to do float16 in C? 347 | // C2NUMPY_CHECK_ITEM 348 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT16) return -1; 349 | // fwrite(&data, sizeof(???), 1, writer->file); 350 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 351 | // C2NUMPY_INCREMENT_ITEM 352 | // } 353 | 354 | int c2numpy_float32(c2numpy_writer *writer, float data) { 355 | C2NUMPY_CHECK_ITEM 356 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT32) return -1; 357 | fwrite(&data, sizeof(float), 1, writer->file); 358 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 359 | C2NUMPY_INCREMENT_ITEM 360 | } 361 | 362 | int c2numpy_float64(c2numpy_writer *writer, double data) { 363 | C2NUMPY_CHECK_ITEM 364 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT64) return -1; 365 | fwrite(&data, sizeof(double), 1, writer->file); 366 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 367 | C2NUMPY_INCREMENT_ITEM 368 | } 369 | 370 | // int c2numpy_complex(c2numpy_writer *writer, ??? data) { // how to do complex in C? 371 | // C2NUMPY_CHECK_ITEM 372 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX) return -1; 373 | // fwrite(&data, sizeof(???), 1, writer->file); 374 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 375 | // C2NUMPY_INCREMENT_ITEM 376 | // } 377 | 378 | // int c2numpy_complex64(c2numpy_writer *writer, ??? data) { 379 | // C2NUMPY_CHECK_ITEM 380 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX64) return -1; 381 | // fwrite(&data, sizeof(???), 1, writer->file); 382 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 383 | // C2NUMPY_INCREMENT_ITEM 384 | // } 385 | 386 | // int c2numpy_complex128(c2numpy_writer *writer, ??? data) { 387 | // C2NUMPY_CHECK_ITEM 388 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX128) return -1; 389 | // fwrite(&data, sizeof(???), 1, writer->file); 390 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 391 | // C2NUMPY_INCREMENT_ITEM 392 | // } 393 | 394 | int c2numpy_string(c2numpy_writer *writer, const char *data) { 395 | C2NUMPY_CHECK_ITEM 396 | 397 | int stringlength = writer->columnTypes[writer->currentColumn] - C2NUMPY_STRING; 398 | if (0 < stringlength && stringlength < 155) 399 | fwrite(data, 1, stringlength, writer->file); 400 | else 401 | return -1; 402 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 403 | 404 | C2NUMPY_INCREMENT_ITEM 405 | } 406 | 407 | int c2numpy_close(c2numpy_writer *writer) { 408 | if (writer->file != NULL) { 409 | // we wrote fewer rows than we promised 410 | if (writer->currentRowInFile < writer->numRowsPerFile) { 411 | // so go back to the part of the header where that was written 412 | fseek(writer->file, writer->sizeSeekPosition, SEEK_SET); 413 | // overwrite it with spaces 414 | int i; 415 | for (i = 0; i < writer->sizeSeekSize; ++i) 416 | fputc(' ', writer->file); 417 | // now go back and write it again (it MUST be fewer or an equal number of digits) 418 | fseek(writer->file, writer->sizeSeekPosition, SEEK_SET); 419 | fprintf(writer->file, "%d", writer->currentRowInFile); 420 | } 421 | // now close it 422 | fclose(writer->file); 423 | } 424 | 425 | return 0; 426 | } 427 | 428 | #endif // C2NUMPY 429 | -------------------------------------------------------------------------------- /commonblock/DemoAnalyzer.cc: -------------------------------------------------------------------------------- 1 | // -*- C++ -*- 2 | // 3 | // Package: Demo/DemoAnalyzer 4 | // Class: DemoAnalyzer 5 | // 6 | /**\class DemoAnalyzer DemoAnalyzer.cc Demo/DemoAnalyzer/plugins/DemoAnalyzer.cc 7 | 8 | Description: [one line class summary] 9 | 10 | Implementation: 11 | [Notes on implementation] 12 | */ 13 | // 14 | // Original Author: James Pivarski 15 | // Created: Wed, 10 May 2017 16:16:22 GMT 16 | // 17 | // 18 | 19 | // system include files 20 | #include 21 | #include 22 | 23 | // user include files 24 | #include "FWCore/Framework/interface/Frameworkfwd.h" 25 | #include "FWCore/Framework/interface/one/EDAnalyzer.h" 26 | 27 | #include "FWCore/Framework/interface/Event.h" 28 | #include "FWCore/Framework/interface/MakerMacros.h" 29 | 30 | #include "FWCore/ParameterSet/interface/ParameterSet.h" 31 | 32 | #include "DataFormats/TrackReco/interface/TrackFwd.h" 33 | #include "DataFormats/TrackReco/interface/Track.h" 34 | #include "DataFormats/TrackingRecHit/interface/TrackingRecHitFwd.h" 35 | #include "DataFormats/TrackingRecHit/interface/TrackingRecHit.h" 36 | 37 | #include "Demo/DemoAnalyzer/interface/NumpyCommonBlock.h" 38 | 39 | // 40 | // class declaration 41 | // 42 | 43 | // If the analyzer does not use TFileService, please remove 44 | // the template argument to the base class so the class inherits 45 | // from edm::one::EDAnalyzer<> and also remove the line from 46 | // constructor "usesResource("TFileService");" 47 | // This will improve performance in multithreaded jobs. 48 | 49 | class DemoAnalyzer : public edm::one::EDAnalyzer { 50 | public: 51 | explicit DemoAnalyzer(const edm::ParameterSet&); 52 | ~DemoAnalyzer(); 53 | 54 | static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); 55 | 56 | private: 57 | virtual void beginJob() override; 58 | virtual void analyze(const edm::Event&, const edm::EventSetup&) override; 59 | virtual void endJob() override; 60 | 61 | // ----------member data --------------------------- 62 | 63 | edm::InputTag trackTag; 64 | edm::EDGetTokenT trackToken; 65 | 66 | edm::InputTag muonTag; 67 | edm::EDGetTokenT muonToken; 68 | 69 | NumpyCommonBlock *tracksBlock; 70 | NumpyCommonBlock *hitsBlock; 71 | 72 | NumpyCommonBlockAccessor *trackermu_qoverp; 73 | NumpyCommonBlockAccessor *trackermu_qoverp_err; 74 | NumpyCommonBlockAccessor *trackermu_phi; 75 | NumpyCommonBlockAccessor *trackermu_eta; 76 | NumpyCommonBlockAccessor *trackermu_dxy; 77 | NumpyCommonBlockAccessor *trackermu_dz; 78 | NumpyCommonBlockAccessor *globalmu_qoverp; 79 | NumpyCommonBlockAccessor *globalmu_qoverp_err; 80 | 81 | NumpyCommonBlockAccessor *detid; 82 | NumpyCommonBlockAccessor *localx; 83 | NumpyCommonBlockAccessor *localy; 84 | NumpyCommonBlockAccessor *localx_err; 85 | NumpyCommonBlockAccessor *localy_err; 86 | 87 | uint64_t tracksIndex = 0; 88 | uint64_t hitsIndex = 0; 89 | }; 90 | 91 | // 92 | // constants, enums and typedefs 93 | // 94 | 95 | // 96 | // static data member definitions 97 | // 98 | 99 | // 100 | // constructors and destructor 101 | // 102 | DemoAnalyzer::DemoAnalyzer(const edm::ParameterSet& iConfig) 103 | : trackTag(edm::InputTag("ALCARECOMuAlZMuMu", "TrackerOnly")), trackToken(consumes(trackTag)) 104 | , muonTag(edm::InputTag("ALCARECOMuAlZMuMu", "GlobalMuon")), muonToken(consumes(muonTag)) 105 | { 106 | //now do what ever initialization is needed 107 | // usesResource("TFileService"); 108 | 109 | tracksBlock = (NumpyCommonBlock*)iConfig.getParameter("tracks"); 110 | hitsBlock = (NumpyCommonBlock*)iConfig.getParameter("hits"); 111 | 112 | trackermu_qoverp = tracksBlock->newAccessor("trackermu_qoverp"); 113 | trackermu_qoverp_err = tracksBlock->newAccessor("trackermu_qoverp_err"); 114 | trackermu_phi = tracksBlock->newAccessor("trackermu_phi"); 115 | trackermu_eta = tracksBlock->newAccessor("trackermu_eta"); 116 | trackermu_dxy = tracksBlock->newAccessor("trackermu_dxy"); 117 | trackermu_dz = tracksBlock->newAccessor("trackermu_dz"); 118 | globalmu_qoverp = tracksBlock->newAccessor("globalmu_qoverp"); 119 | globalmu_qoverp_err = tracksBlock->newAccessor("globalmu_qoverp_err"); 120 | 121 | detid = hitsBlock->newAccessor("detid"); 122 | localx = hitsBlock->newAccessor("localx"); 123 | localy = hitsBlock->newAccessor("localy"); 124 | localx_err = hitsBlock->newAccessor("localx_err"); 125 | localy_err = hitsBlock->newAccessor("localy_err"); 126 | } 127 | 128 | 129 | DemoAnalyzer::~DemoAnalyzer() 130 | { 131 | 132 | // do anything here that needs to be done at desctruction time 133 | // (e.g. close files, deallocate resources etc.) 134 | 135 | } 136 | 137 | 138 | // 139 | // member functions 140 | // 141 | 142 | // ------------ method called for each event ------------ 143 | void 144 | DemoAnalyzer::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) 145 | { 146 | using namespace edm; 147 | 148 | edm::Handle tracks; 149 | iEvent.getByLabel("ALCARECOMuAlZMuMu", "TrackerOnly", tracks); 150 | 151 | if (tracks->size() != 2) 152 | return; 153 | 154 | edm::Handle muons; 155 | iEvent.getByLabel("ALCARECOMuAlZMuMu", "GlobalMuon", muons); 156 | 157 | if (muons->size() != 2) 158 | return; 159 | 160 | // auto t0 = (*tracks)[0].momentum(); 161 | // auto t1 = (*tracks)[1].momentum(); 162 | // auto m0 = (*muons)[0].momentum(); 163 | // auto m1 = (*muons)[1].momentum(); 164 | // double normal = t0.Dot(m0) / sqrt(t0.Mag2()) / sqrt(m0.Mag2()) + t1.Dot(m1) / sqrt(t1.Mag2()) / sqrt(m1.Mag2()); 165 | // double inverted = t0.Dot(m1) / sqrt(t0.Mag2()) / sqrt(m1.Mag2()) + t1.Dot(m0) / sqrt(t1.Mag2()) / sqrt(m0.Mag2()); 166 | // if (inverted > normal) 167 | // return; 168 | 169 | reco::TrackCollection::const_iterator track = tracks->begin(); 170 | reco::TrackCollection::const_iterator muon = muons->begin(); 171 | 172 | for (int i = 0; i < 2; i++) { 173 | if (tracksIndex < trackermu_qoverp->size()) { 174 | trackermu_qoverp->set(tracksIndex, track->qoverp()); 175 | trackermu_qoverp_err->set(tracksIndex, track->qoverpError()); 176 | trackermu_phi->set(tracksIndex, track->phi()); 177 | trackermu_eta->set(tracksIndex, track->eta()); 178 | trackermu_dxy->set(tracksIndex, track->dxy()); 179 | trackermu_dz->set(tracksIndex, track->dz()); 180 | globalmu_qoverp->set(tracksIndex, muon->qoverp()); 181 | globalmu_qoverp_err->set(tracksIndex, muon->qoverpError()); 182 | tracksIndex++; 183 | if (tracksIndex == trackermu_qoverp->size()) 184 | tracksBlock->notify(1); 185 | } 186 | 187 | for (trackingRecHit_iterator hit = muon->recHitsBegin(); hit != muon->recHitsEnd(); ++hit) { 188 | if ((*hit)->isValid() && (*hit)->rawId() > 600000000) { // only muon hits have localpositions 189 | if (hitsIndex < detid->size()) { 190 | detid->set(hitsIndex, (*hit)->rawId()); 191 | localx->set(hitsIndex, (*hit)->localPosition().x()); 192 | localy->set(hitsIndex, (*hit)->localPosition().y()); 193 | localx_err->set(hitsIndex, (*hit)->localPositionError().xx()); 194 | localy_err->set(hitsIndex, (*hit)->localPositionError().yy()); 195 | hitsIndex++; 196 | if (hitsIndex == detid->size()) 197 | hitsBlock->notify(1); 198 | } 199 | } 200 | } 201 | 202 | ++track; 203 | ++muon; 204 | } 205 | } 206 | 207 | 208 | // ------------ Method called once each job just before starting event loop ------------ 209 | void 210 | DemoAnalyzer::beginJob() 211 | { 212 | } 213 | 214 | // ------------ method called once each job just after ending the event loop ------------ 215 | void 216 | DemoAnalyzer::endJob() 217 | { 218 | } 219 | 220 | // ------------ method fills 'descriptions' with the allowed parameters for the module ------------ 221 | void 222 | DemoAnalyzer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { 223 | //The following says we do not know what parameters are allowed so do no validation 224 | // Please change this to state exactly what you do use, even if it is no parameters 225 | edm::ParameterSetDescription desc; 226 | desc.setUnknown(); 227 | descriptions.addDefault(desc); 228 | } 229 | 230 | //define this as a plug-in 231 | DEFINE_FWK_MODULE(DemoAnalyzer); 232 | -------------------------------------------------------------------------------- /commonblock/NumpyCommonBlock.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Jim Pivarski 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | template class NumpyCommonBlockAccessor; 22 | 23 | class NumpyCommonBlock { 24 | template friend class NumpyCommonBlockAccessor; 25 | 26 | public: 27 | template NumpyCommonBlockAccessor accessor(std::string name) { 28 | uint64_t which = 0; 29 | while (which < numArrays) { 30 | if (std::string(names[which]) == name) 31 | break; 32 | which++; 33 | }; 34 | assert(which < numArrays); 35 | 36 | return NumpyCommonBlockAccessor(this, which); 37 | } 38 | 39 | template NumpyCommonBlockAccessor* newAccessor(std::string name) { 40 | uint64_t which = 0; 41 | while (which < numArrays) { 42 | if (std::string(names[which]) == name) 43 | break; 44 | which++; 45 | }; 46 | assert(which < numArrays); 47 | 48 | return new NumpyCommonBlockAccessor(this, which); 49 | } 50 | 51 | inline void wait(uint64_t forstate) { 52 | while (pthread_rwlock_rdlock(statelock) != 0) usleep(1); 53 | uint64_t current = state; 54 | pthread_rwlock_unlock(statelock); 55 | 56 | while (current != forstate) { 57 | usleep(1); 58 | while (pthread_rwlock_rdlock(statelock) != 0) usleep(1); 59 | current = state; 60 | pthread_rwlock_unlock(statelock); 61 | } 62 | } 63 | 64 | inline void waitmask(uint64_t formask) { 65 | while (pthread_rwlock_rdlock(statelock) != 0) usleep(1); 66 | uint64_t current = state; 67 | pthread_rwlock_unlock(statelock); 68 | 69 | while (!(current & formask)) { 70 | usleep(1); 71 | while (pthread_rwlock_rdlock(statelock) != 0) usleep(1); 72 | current = state; 73 | pthread_rwlock_unlock(statelock); 74 | } 75 | } 76 | 77 | inline void notify(uint64_t newstate) { 78 | while (pthread_rwlock_wrlock(statelock) != 0) usleep(1); 79 | state = newstate; 80 | pthread_rwlock_unlock(statelock); 81 | } 82 | 83 | private: 84 | NumpyCommonBlock() { } // can't create them 85 | uint64_t numArrays; 86 | char **names; 87 | char **types; 88 | void **data; 89 | uint64_t *lengths; 90 | pthread_rwlock_t **locks; 91 | pthread_rwlock_t *statelock; 92 | uint64_t state; 93 | }; 94 | 95 | template class NumpyCommonBlockAccessor { 96 | public: 97 | NumpyCommonBlockAccessor(NumpyCommonBlock *cb, uint64_t which) { 98 | std::string type = std::string(cb->types[which]); 99 | 100 | if (type == std::string("bool")) 101 | assert(typeid(T) == typeid(bool)); 102 | 103 | else if (type == std::string("int8")) 104 | assert(typeid(T) == typeid(int8_t)); 105 | 106 | else if (type == std::string("uint8")) 107 | assert(typeid(T) == typeid(uint8_t)); 108 | 109 | else if (type == std::string("int16")) 110 | assert(typeid(T) == typeid(int16_t)); 111 | 112 | else if (type == std::string("uint16")) 113 | assert(typeid(T) == typeid(uint16_t)); 114 | 115 | else if (type == std::string("int32")) 116 | assert(typeid(T) == typeid(int32_t)); 117 | 118 | else if (type == std::string("uint32")) 119 | assert(typeid(T) == typeid(uint32_t)); 120 | 121 | else if (type == std::string("int64")) 122 | assert(typeid(T) == typeid(int64_t)); 123 | 124 | else if (type == std::string("uint64")) 125 | assert(typeid(T) == typeid(uint64_t)); 126 | 127 | else if (type == std::string("float32")) 128 | assert(typeid(T) == typeid(float)); 129 | 130 | else if (type == std::string("float64")) 131 | assert(typeid(T) == typeid(double)); 132 | 133 | else 134 | assert(false); 135 | 136 | lock = cb->locks[which]; 137 | data = (T*)(cb->data[which]); 138 | length = cb->lengths[which]; 139 | } 140 | 141 | inline T get(uint64_t index) { 142 | assert(index < length); 143 | return data[index]; 144 | } 145 | 146 | inline void set(uint64_t index, T value) { 147 | data[index] = value; 148 | } 149 | 150 | inline T safeget(uint64_t index) { 151 | assert(index < length); 152 | while (pthread_rwlock_rdlock(lock) != 0) usleep(1); 153 | T out = data[index]; 154 | pthread_rwlock_unlock(lock); 155 | return out; 156 | } 157 | 158 | inline void safeset(uint64_t index, T value) { 159 | assert(index < length); 160 | while (pthread_rwlock_wrlock(lock) != 0) usleep(1); 161 | data[index] = value; 162 | pthread_rwlock_unlock(lock); 163 | } 164 | 165 | inline uint64_t size() { 166 | return length; 167 | } 168 | 169 | private: 170 | pthread_rwlock_t *lock; 171 | T *data; 172 | uint64_t length; 173 | }; 174 | -------------------------------------------------------------------------------- /commonblock/commonblock.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Jim Pivarski 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import ctypes 16 | import time 17 | 18 | import numpy 19 | import prwlock 20 | 21 | class NumpyCommonBlock(object): 22 | class struct(ctypes.Structure): 23 | _fields_ = [ 24 | ("numArrays", ctypes.c_uint64), 25 | ("names", ctypes.POINTER(ctypes.c_char_p)), 26 | ("types", ctypes.POINTER(ctypes.c_char_p)), 27 | ("data", ctypes.POINTER(ctypes.POINTER(None))), 28 | ("lengths", ctypes.POINTER(ctypes.c_uint64)), 29 | ("locks", ctypes.POINTER(ctypes.POINTER(None))), 30 | ("statelock", ctypes.POINTER(None)), 31 | ("state", ctypes.c_uint64)] 32 | 33 | def __init__(self, *order, **arrays): 34 | self._arrays = arrays 35 | 36 | numArrays = len(arrays) 37 | 38 | self._order = [] 39 | remaining = set(arrays) 40 | for name in order: 41 | if name in remaining: 42 | self._order.append(name) 43 | remaining.discard(name) 44 | self._order.extend(sorted(remaining)) 45 | 46 | c_names = ctypes.ARRAY(ctypes.c_char_p, numArrays)(*[ctypes.c_char_p(x) for x in self._order]) 47 | c_types = ctypes.ARRAY(ctypes.c_char_p, numArrays)(*[ctypes.c_char_p(bytes(arrays[x].dtype)) for x in self._order]) 48 | c_data = ctypes.ARRAY(ctypes.POINTER(None), numArrays)(*[ 49 | arrays[x].ctypes.data_as(ctypes.POINTER(None)) for x in self._order]) 50 | c_lengths = ctypes.ARRAY(ctypes.c_uint64, numArrays)(*[ctypes.c_uint64(numpy.product(arrays[x].shape)) for x in self._order]) 51 | self._locks = [prwlock.RWLock() for x in self._order] 52 | c_locks = ctypes.ARRAY(ctypes.POINTER(None), numArrays)(*[ctypes.cast(x._lock, ctypes.POINTER(None)) for x in self._locks]) 53 | self._statelock = prwlock.RWLock() 54 | c_statelock = ctypes.cast(self._statelock._lock, ctypes.POINTER(None)) 55 | 56 | self._struct = self.struct(ctypes.c_uint64(numArrays), c_names, c_types, c_data, c_lengths, c_locks, c_statelock, ctypes.c_uint64(0)) 57 | 58 | def pointer(self): 59 | return ctypes.addressof(self._struct) 60 | 61 | def pandas(self): 62 | import pandas 63 | for lock in self._locks: 64 | lock.acquire_read() 65 | try: 66 | return pandas.DataFrame(self._arrays, columns=self._order) 67 | finally: 68 | for lock in self._locks: 69 | lock.release() 70 | 71 | class Accessor(object): 72 | def __init__(self, lock, array): 73 | self.lock = lock 74 | self.array = array 75 | 76 | def __getitem__(self, slice): 77 | self.lock.acquire_read() 78 | try: 79 | return self.array[slice] 80 | finally: 81 | self.lock.release() 82 | 83 | def __setitem__(self, slice, value): 84 | self.lock.acquire_write() 85 | try: 86 | self.array[slice] = value 87 | finally: 88 | self.lock.release() 89 | 90 | def size(self): 91 | return len(self.array) 92 | 93 | def accessor(self, name): 94 | return self.Accessor(self._locks[self._order.index(name)], self._arrays[name]) 95 | 96 | def _wait(self, condition): 97 | self._statelock.acquire_read() 98 | try: 99 | current = self._struct.state 100 | finally: 101 | self._statelock.release() 102 | 103 | while condition(current): 104 | time.sleep(0.001) 105 | self._statelock.acquire_read() 106 | try: 107 | current = self._struct.state 108 | finally: 109 | self._statelock.release() 110 | 111 | def wait(self, forstate): 112 | self._wait(lambda current: current != forstate) 113 | 114 | def waitmask(self, formask): 115 | self._wait(lambda current: not (current & formask)) 116 | 117 | def notify(self, newstate): 118 | self._statelock.acquire_write() 119 | try: 120 | self._struct.state = newstate 121 | finally: 122 | self._statelock.release() 123 | -------------------------------------------------------------------------------- /commonblock/pythonrun.py: -------------------------------------------------------------------------------- 1 | # sharable data 2 | 3 | import numpy 4 | import commonblock 5 | 6 | tracks = commonblock.NumpyCommonBlock( 7 | trackermu_qoverp = numpy.zeros(1000, dtype=numpy.double), 8 | trackermu_qoverp_err = numpy.zeros(1000, dtype=numpy.double), 9 | trackermu_phi = numpy.zeros(1000, dtype=numpy.double), 10 | trackermu_eta = numpy.zeros(1000, dtype=numpy.double), 11 | trackermu_dxy = numpy.zeros(1000, dtype=numpy.double), 12 | trackermu_dz = numpy.zeros(1000, dtype=numpy.double), 13 | globalmu_qoverp = numpy.zeros(1000, dtype=numpy.double), 14 | globalmu_qoverp_err = numpy.zeros(1000, dtype=numpy.double)) 15 | 16 | hits = commonblock.NumpyCommonBlock( 17 | detid = numpy.zeros(5000, dtype=numpy.uint64), 18 | localx = numpy.zeros(5000, dtype=numpy.double), 19 | localy = numpy.zeros(5000, dtype=numpy.double), 20 | localx_err = numpy.zeros(5000, dtype=numpy.double), 21 | localy_err = numpy.zeros(5000, dtype=numpy.double)) 22 | 23 | # CMSSW configuration 24 | 25 | import FWCore.ParameterSet.Config as cms 26 | 27 | process = cms.Process("Demo") 28 | 29 | process.load("FWCore.MessageService.MessageLogger_cfi") 30 | 31 | process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(1000)) 32 | 33 | process.source = cms.Source( 34 | "PoolSource", fileNames = cms.untracked.vstring("file:MuAlZMuMu-2016H-002590494DA0.root")) 35 | 36 | process.demo = cms.EDAnalyzer( 37 | "DemoAnalyzer", 38 | tracks = cms.uint64(tracks.pointer()), # pass the arrays to C++ as a pointer 39 | hits = cms.uint64(hits.pointer())) 40 | 41 | process.p = cms.Path(process.demo) 42 | 43 | # run it in Python 44 | 45 | import threading 46 | import libFWCorePythonFramework 47 | import libFWCorePythonParameterSet 48 | 49 | class CMSSWThread(threading.Thread): 50 | def __init__(self, process): 51 | super(CMSSWThread, self).__init__() 52 | self.process = process 53 | 54 | def run(self): 55 | processDesc = libFWCorePythonParameterSet.ProcessDesc() 56 | self.process.fillProcessDesc(processDesc.pset()) 57 | 58 | cppProcessor = libFWCorePythonFramework.PythonEventProcessor(processDesc) 59 | cppProcessor.run() 60 | print "CMSSW finished" 61 | 62 | cmsswThread = CMSSWThread(process) 63 | cmsswThread.start() 64 | 65 | print "running" 66 | 67 | tracks.wait(1) # CMSSW notifies that it has filled the tracks array 68 | print tracks.pandas() 69 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/Demo/DemoAnalyzer/interface/c2numpy.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Jim Pivarski 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef C2NUMPY 16 | #define C2NUMPY 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | const char* C2NUMPY_VERSION = "1.1"; 23 | 24 | // http://docs.scipy.org/doc/numpy/user/basics.types.html 25 | typedef enum { 26 | C2NUMPY_BOOL, // Boolean (True or False) stored as a byte 27 | C2NUMPY_INT, // Default integer type (same as C long; normally either int64 or int32) 28 | C2NUMPY_INTC, // Identical to C int (normally int32 or int64) 29 | C2NUMPY_INTP, // Integer used for indexing (same as C ssize_t; normally either int32 or int64) 30 | C2NUMPY_INT8, // Byte (-128 to 127) 31 | C2NUMPY_INT16, // Integer (-32768 to 32767) 32 | C2NUMPY_INT32, // Integer (-2147483648 to 2147483647) 33 | C2NUMPY_INT64, // Integer (-9223372036854775808 to 9223372036854775807) 34 | C2NUMPY_UINT8, // Unsigned integer (0 to 255) 35 | C2NUMPY_UINT16, // Unsigned integer (0 to 65535) 36 | C2NUMPY_UINT32, // Unsigned integer (0 to 4294967295) 37 | C2NUMPY_UINT64, // Unsigned integer (0 to 18446744073709551615) 38 | C2NUMPY_FLOAT, // Shorthand for float64. 39 | C2NUMPY_FLOAT16, // Half precision float: sign bit, 5 bits exponent, 10 bits mantissa 40 | C2NUMPY_FLOAT32, // Single precision float: sign bit, 8 bits exponent, 23 bits mantissa 41 | C2NUMPY_FLOAT64, // Double precision float: sign bit, 11 bits exponent, 52 bits mantissa 42 | C2NUMPY_COMPLEX, // Shorthand for complex128. 43 | C2NUMPY_COMPLEX64, // Complex number, represented by two 32-bit floats (real and imaginary components) 44 | C2NUMPY_COMPLEX128, // Complex number, represented by two 64-bit floats (real and imaginary components) 45 | 46 | C2NUMPY_STRING = 100, // strings are C2NUMPY_STRING + their fixed size (up to 155) 47 | C2NUMPY_END = 255 // ensure that c2numpy_type is at least a byte 48 | } c2numpy_type; 49 | 50 | // a Numpy writer object 51 | typedef struct { 52 | char buffer[16]; // (internal) used for temporary copies in c2numpy_row 53 | 54 | FILE *file; // output file handle 55 | char *outputFilePrefix; // output file name, not including the rotating number and .npy 56 | int64_t sizeSeekPosition; // (internal) keep track of number of rows to modify before closing 57 | int64_t sizeSeekSize; // (internal) 58 | 59 | int32_t numColumns; // number of columns in the record array 60 | char **columnNames; // column names 61 | c2numpy_type *columnTypes; // column types 62 | 63 | int32_t numRowsPerFile; // maximum number of rows per file 64 | int32_t currentColumn; // current column number 65 | int32_t currentRowInFile; // current row number in the current file 66 | int32_t currentFileNumber; // current file number 67 | } c2numpy_writer; 68 | 69 | const char *c2numpy_descr(c2numpy_type type) { 70 | // FIXME: all of the "<" signs should be system-dependent (they mean little endian) 71 | static const char *c2numpy_bool = "|b1"; 72 | static const char *c2numpy_int = "file = NULL; 142 | writer->outputFilePrefix = (char*)malloc(strlen(outputFilePrefix) + 1); 143 | strcpy(writer->outputFilePrefix, outputFilePrefix); 144 | writer->sizeSeekPosition = 0; 145 | writer->sizeSeekSize = 0; 146 | 147 | writer->numColumns = 0; 148 | writer->columnNames = NULL; 149 | writer->columnTypes = NULL; 150 | 151 | writer->numRowsPerFile = numRowsPerFile; 152 | writer->currentColumn = 0; 153 | writer->currentRowInFile = 0; 154 | writer->currentFileNumber = 0; 155 | 156 | return 0; 157 | } 158 | 159 | int c2numpy_addcolumn(c2numpy_writer *writer, const char *name, c2numpy_type type) { 160 | writer->numColumns += 1; 161 | 162 | char *newColumnName = (char*)malloc(strlen(name) + 1); 163 | strcpy(newColumnName, name); 164 | 165 | char **oldColumnNames = writer->columnNames; 166 | writer->columnNames = (char**)malloc(writer->numColumns * sizeof(char*)); 167 | for (int column = 0; column < writer->numColumns - 1; ++column) 168 | writer->columnNames[column] = oldColumnNames[column]; 169 | writer->columnNames[writer->numColumns - 1] = newColumnName; 170 | if (oldColumnNames != NULL) 171 | free(oldColumnNames); 172 | 173 | c2numpy_type *oldColumnTypes = writer->columnTypes; 174 | writer->columnTypes = (c2numpy_type*)malloc(writer->numColumns * sizeof(c2numpy_type)); 175 | for (int column = 0; column < writer->numColumns - 1; ++column) 176 | writer->columnTypes[column] = oldColumnTypes[column]; 177 | writer->columnTypes[writer->numColumns - 1] = type; 178 | if (oldColumnTypes != NULL) 179 | free(oldColumnTypes); 180 | 181 | return 0; 182 | } 183 | 184 | int c2numpy_open(c2numpy_writer *writer) { 185 | char *fileName = (char*)malloc(strlen(writer->outputFilePrefix) + 15); 186 | sprintf(fileName, "%s%d.npy", writer->outputFilePrefix, writer->currentFileNumber); 187 | writer->file = fopen(fileName, "wb"); 188 | 189 | // FIXME: better initial guess about header size before going in 128 byte increments 190 | char *header = NULL; 191 | for (int64_t headerSize = 128; headerSize <= 4294967295; headerSize += 128) { 192 | if (header != NULL) free(header); 193 | header = (char*)malloc(headerSize + 1); 194 | 195 | char version1 = headerSize <= 65535; 196 | uint32_t descrSize; 197 | if (version1) 198 | descrSize = headerSize - 10; 199 | else 200 | descrSize = headerSize - 12; 201 | 202 | header[0] = 147; // magic 203 | header[1] = 'N'; 204 | header[2] = 'U'; 205 | header[3] = 'M'; 206 | header[4] = 'P'; 207 | header[5] = 'Y'; 208 | if (version1) { 209 | header[6] = 1; // format version 1.0 210 | header[7] = 0; 211 | const uint16_t descrSize2 = descrSize; 212 | *(uint16_t*)(header + 8) = descrSize2; // version 1.0 has a 16-byte descrSize 213 | } 214 | else { 215 | header[6] = 2; // format version 2.0 216 | header[7] = 0; 217 | *(uint32_t*)(header + 8) = descrSize; // version 2.0 has a 32-byte descrSize 218 | } 219 | 220 | int64_t offset = headerSize - descrSize; 221 | offset += snprintf((header + offset), headerSize - offset + 1, "{'descr': ["); 222 | if (offset >= headerSize) continue; 223 | 224 | for (int column = 0; column < writer->numColumns; ++column) { 225 | offset += snprintf((header + offset), headerSize - offset + 1, "('%s', '%s')", 226 | writer->columnNames[column], 227 | c2numpy_descr(writer->columnTypes[column])); 228 | if (offset >= headerSize) continue; 229 | 230 | if (column < writer->numColumns - 1) 231 | offset += snprintf((header + offset), headerSize - offset + 1, ", "); 232 | if (offset >= headerSize) continue; 233 | } 234 | 235 | offset += snprintf((header + offset), headerSize - offset + 1, "], 'fortran_order': False, 'shape': ("); 236 | if (offset >= headerSize) continue; 237 | 238 | writer->sizeSeekPosition = offset; 239 | writer->sizeSeekSize = snprintf((header + offset), headerSize - offset + 1, "%d", writer->numRowsPerFile); 240 | offset += writer->sizeSeekSize; 241 | if (offset >= headerSize) continue; 242 | 243 | offset += snprintf((header + offset), headerSize - offset + 1, ",), }"); 244 | if (offset >= headerSize) continue; 245 | 246 | while (offset < headerSize) { 247 | if (offset < headerSize - 1) 248 | header[offset] = ' '; 249 | else 250 | header[offset] = '\n'; 251 | offset += 1; 252 | } 253 | header[headerSize] = 0; 254 | 255 | fwrite(header, 1, headerSize, writer->file); 256 | 257 | return 0; 258 | } 259 | 260 | return -1; 261 | } 262 | 263 | #define C2NUMPY_CHECK_ITEM { \ 264 | if (writer->file == NULL) { \ 265 | int status = c2numpy_open(writer); \ 266 | if (status != 0) \ 267 | return status; \ 268 | } \ 269 | } 270 | 271 | #define C2NUMPY_INCREMENT_ITEM { \ 272 | if (writer->currentColumn == 0) { \ 273 | writer->currentRowInFile += 1; \ 274 | if (writer->currentRowInFile == writer->numRowsPerFile) { \ 275 | fclose(writer->file); \ 276 | writer->file = NULL; \ 277 | writer->currentRowInFile = 0; \ 278 | writer->currentFileNumber += 1; \ 279 | } \ 280 | } \ 281 | return 0; \ 282 | } 283 | 284 | int c2numpy_bool(c2numpy_writer *writer, int8_t data) { // "bool" is just a byte 285 | C2NUMPY_CHECK_ITEM 286 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_BOOL) return -1; 287 | fwrite(&data, sizeof(int8_t), 1, writer->file); 288 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 289 | C2NUMPY_INCREMENT_ITEM 290 | } 291 | 292 | int c2numpy_int(c2numpy_writer *writer, int64_t data) { // Numpy's default int is 64-bit 293 | C2NUMPY_CHECK_ITEM 294 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT) return -1; 295 | fwrite(&data, sizeof(int64_t), 1, writer->file); 296 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 297 | C2NUMPY_INCREMENT_ITEM 298 | } 299 | 300 | int c2numpy_intc(c2numpy_writer *writer, int data) { // the built-in C int 301 | C2NUMPY_CHECK_ITEM 302 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INTC) return -1; 303 | fwrite(&data, sizeof(int), 1, writer->file); 304 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 305 | C2NUMPY_INCREMENT_ITEM 306 | } 307 | 308 | int c2numpy_intp(c2numpy_writer *writer, size_t data) { // intp is Numpy's way of saying size_t 309 | C2NUMPY_CHECK_ITEM 310 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INTP) return -1; 311 | fwrite(&data, sizeof(size_t), 1, writer->file); 312 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 313 | C2NUMPY_INCREMENT_ITEM 314 | } 315 | 316 | int c2numpy_int8(c2numpy_writer *writer, int8_t data) { 317 | C2NUMPY_CHECK_ITEM 318 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT8) return -1; 319 | fwrite(&data, sizeof(int8_t), 1, writer->file); 320 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 321 | C2NUMPY_INCREMENT_ITEM 322 | } 323 | 324 | int c2numpy_int16(c2numpy_writer *writer, int16_t data) { 325 | C2NUMPY_CHECK_ITEM 326 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT16) return -1; 327 | fwrite(&data, sizeof(int16_t), 1, writer->file); 328 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 329 | C2NUMPY_INCREMENT_ITEM 330 | } 331 | 332 | int c2numpy_int32(c2numpy_writer *writer, int32_t data) { 333 | C2NUMPY_CHECK_ITEM 334 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT32) return -1; 335 | fwrite(&data, sizeof(int32_t), 1, writer->file); 336 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 337 | C2NUMPY_INCREMENT_ITEM 338 | } 339 | 340 | int c2numpy_int64(c2numpy_writer *writer, int64_t data) { 341 | C2NUMPY_CHECK_ITEM 342 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_INT64) return -1; 343 | fwrite(&data, sizeof(int64_t), 1, writer->file); 344 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 345 | C2NUMPY_INCREMENT_ITEM 346 | } 347 | 348 | int c2numpy_uint8(c2numpy_writer *writer, uint8_t data) { 349 | C2NUMPY_CHECK_ITEM 350 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT8) return -1; 351 | fwrite(&data, sizeof(uint8_t), 1, writer->file); 352 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 353 | C2NUMPY_INCREMENT_ITEM 354 | } 355 | 356 | int c2numpy_uint16(c2numpy_writer *writer, uint16_t data) { 357 | C2NUMPY_CHECK_ITEM 358 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT16) return -1; 359 | fwrite(&data, sizeof(uint16_t), 1, writer->file); 360 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 361 | C2NUMPY_INCREMENT_ITEM 362 | } 363 | 364 | int c2numpy_uint32(c2numpy_writer *writer, uint32_t data) { 365 | C2NUMPY_CHECK_ITEM 366 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT32) return -1; 367 | fwrite(&data, sizeof(uint32_t), 1, writer->file); 368 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 369 | C2NUMPY_INCREMENT_ITEM 370 | } 371 | 372 | int c2numpy_uint64(c2numpy_writer *writer, uint64_t data) { 373 | C2NUMPY_CHECK_ITEM 374 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_UINT64) return -1; 375 | fwrite(&data, sizeof(uint64_t), 1, writer->file); 376 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 377 | C2NUMPY_INCREMENT_ITEM 378 | } 379 | 380 | int c2numpy_float(c2numpy_writer *writer, double data) { // Numpy's "float" is a double 381 | C2NUMPY_CHECK_ITEM 382 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT) return -1; 383 | fwrite(&data, sizeof(double), 1, writer->file); 384 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 385 | C2NUMPY_INCREMENT_ITEM 386 | } 387 | 388 | // int c2numpy_float16(c2numpy_writer *writer, ??? data) { // how to do float16 in C? 389 | // C2NUMPY_CHECK_ITEM 390 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT16) return -1; 391 | // fwrite(&data, sizeof(???), 1, writer->file); 392 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 393 | // C2NUMPY_INCREMENT_ITEM 394 | // } 395 | 396 | int c2numpy_float32(c2numpy_writer *writer, float data) { 397 | C2NUMPY_CHECK_ITEM 398 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT32) return -1; 399 | fwrite(&data, sizeof(float), 1, writer->file); 400 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 401 | C2NUMPY_INCREMENT_ITEM 402 | } 403 | 404 | int c2numpy_float64(c2numpy_writer *writer, double data) { 405 | C2NUMPY_CHECK_ITEM 406 | if (writer->columnTypes[writer->currentColumn] != C2NUMPY_FLOAT64) return -1; 407 | fwrite(&data, sizeof(double), 1, writer->file); 408 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 409 | C2NUMPY_INCREMENT_ITEM 410 | } 411 | 412 | // int c2numpy_complex(c2numpy_writer *writer, ??? data) { // how to do complex in C? 413 | // C2NUMPY_CHECK_ITEM 414 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX) return -1; 415 | // fwrite(&data, sizeof(???), 1, writer->file); 416 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 417 | // C2NUMPY_INCREMENT_ITEM 418 | // } 419 | 420 | // int c2numpy_complex64(c2numpy_writer *writer, ??? data) { 421 | // C2NUMPY_CHECK_ITEM 422 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX64) return -1; 423 | // fwrite(&data, sizeof(???), 1, writer->file); 424 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 425 | // C2NUMPY_INCREMENT_ITEM 426 | // } 427 | 428 | // int c2numpy_complex128(c2numpy_writer *writer, ??? data) { 429 | // C2NUMPY_CHECK_ITEM 430 | // if (writer->columnTypes[writer->currentColumn] != C2NUMPY_COMPLEX128) return -1; 431 | // fwrite(&data, sizeof(???), 1, writer->file); 432 | // writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 433 | // C2NUMPY_INCREMENT_ITEM 434 | // } 435 | 436 | int c2numpy_string(c2numpy_writer *writer, const char *data) { 437 | C2NUMPY_CHECK_ITEM 438 | 439 | int stringlength = writer->columnTypes[writer->currentColumn] - C2NUMPY_STRING; 440 | if (0 < stringlength && stringlength < 155) 441 | fwrite(data, 1, stringlength, writer->file); 442 | else 443 | return -1; 444 | writer->currentColumn = (writer->currentColumn + 1) % writer->numColumns; 445 | 446 | C2NUMPY_INCREMENT_ITEM 447 | } 448 | 449 | int c2numpy_close(c2numpy_writer *writer) { 450 | if (writer->file != NULL) { 451 | // we wrote fewer rows than we promised 452 | if (writer->currentRowInFile < writer->numRowsPerFile) { 453 | // so go back to the part of the header where that was written 454 | fseek(writer->file, writer->sizeSeekPosition, SEEK_SET); 455 | // overwrite it with spaces 456 | for (int i = 0; i < writer->sizeSeekSize; ++i) 457 | fputc(' ', writer->file); 458 | // now go back and write it again (it MUST be fewer or an equal number of digits) 459 | fseek(writer->file, writer->sizeSeekPosition, SEEK_SET); 460 | fprintf(writer->file, "%d", writer->currentRowInFile); 461 | } 462 | // now close it 463 | fclose(writer->file); 464 | } 465 | 466 | // and clear the malloc'ed memory 467 | free(writer->outputFilePrefix); 468 | for (int column = 0; column < writer->numColumns; ++column) 469 | free(writer->columnNames[column]); 470 | free(writer->columnNames); 471 | free(writer->columnTypes); 472 | 473 | return 0; 474 | } 475 | 476 | #endif // C2NUMPY 477 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/Demo/DemoAnalyzer/plugins/BuildFile.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/Demo/DemoAnalyzer/plugins/DemoAnalyzer.cc: -------------------------------------------------------------------------------- 1 | // -*- C++ -*- 2 | // 3 | // Package: Demo/DemoAnalyzer 4 | // Class: DemoAnalyzer 5 | // 6 | /**\class DemoAnalyzer DemoAnalyzer.cc Demo/DemoAnalyzer/plugins/DemoAnalyzer.cc 7 | 8 | Description: [one line class summary] 9 | 10 | Implementation: 11 | [Notes on implementation] 12 | */ 13 | // 14 | // Original Author: Valentin Y Kuznetsov 15 | // Created: Wed, 03 Aug 2016 19:31:32 GMT 16 | // 17 | // 18 | // Code examples used to write this code 19 | // https://github.com/cms-sw/cmssw/blob/CMSSW_8_1_X/DataFormats/TrackReco/interface/TrackBase.h 20 | // http://cmslxr.fnal.gov/source/Fireworks/Tracks/plugins/FWTracksRecHitsProxyBuilder.cc 21 | // http://cmslxr.fnal.gov/source/Fireworks/Tracks/src/TrackUtils.cc#0422 22 | // http://cmslxr.fnal.gov/source/Fireworks/Geometry/src/FWRecoGeometryESProducer.cc#0075 23 | // http://cmslxr.fnal.gov/source/Fireworks/Geometry/interface/FWRecoGeometryESProducer.h#0059 24 | 25 | 26 | // system include files 27 | #include 28 | 29 | // user include files 30 | #include "FWCore/Framework/interface/Frameworkfwd.h" 31 | #include "FWCore/Framework/interface/one/EDAnalyzer.h" 32 | 33 | #include "FWCore/Framework/interface/Event.h" 34 | #include "FWCore/Framework/interface/MakerMacros.h" 35 | 36 | #include "FWCore/ParameterSet/interface/ParameterSet.h" 37 | 38 | // track includes 39 | #include "DataFormats/TrackReco/interface/Track.h" 40 | #include "DataFormats/TrackReco/interface/TrackFwd.h" 41 | #include "FWCore/MessageLogger/interface/MessageLogger.h" 42 | 43 | // c2numpy convertion include 44 | #include "Demo/DemoAnalyzer/interface/c2numpy.h" 45 | 46 | 47 | // fireworks and geometry includes 48 | #include "TEveGeoShape.h" 49 | #include "TEvePointSet.h" 50 | 51 | #include "Fireworks/Core/interface/FWSimpleProxyBuilderTemplate.h" 52 | #include "Fireworks/Core/interface/FWGeometry.h" 53 | #include "Fireworks/Core/interface/FWEventItem.h" 54 | #include "Fireworks/Tracks/interface/TrackUtils.h" 55 | #include "Fireworks/Core/interface/fwLog.h" 56 | 57 | #include "DataFormats/TrackerRecHit2D/interface/SiPixelRecHit.h" 58 | 59 | #include 60 | using namespace std; 61 | 62 | // local function to get Pixel Detector Hits, see 63 | // http://cmslxr.fnal.gov/source/Fireworks/Tracks/src/TrackUtils.cc#0611 64 | void 65 | pixelHits( std::vector &pixelPoints, FWGeometry *geom, const reco::Track &t ); 66 | void 67 | pixelHits( std::vector &pixelPoints, FWGeometry *geom, const reco::Track &t ) 68 | { 69 | for( trackingRecHit_iterator it = t.recHitsBegin(), itEnd = t.recHitsEnd(); it != itEnd; ++it ) 70 | { 71 | const TrackingRecHit* rh = &(**it); 72 | // -- get position of center of wafer, assuming (0,0,0) is the center 73 | DetId id = (*it)->geographicalId(); 74 | if( ! geom->contains( id )) 75 | { 76 | fwLog( fwlog::kError ) 77 | << "failed to get geometry of Tracker Det with raw id: " 78 | << id.rawId() << std::endl; 79 | 80 | continue; 81 | } 82 | 83 | // -- in which detector are we? 84 | unsigned int subdet = (unsigned int)id.subdetId(); 85 | 86 | if(( subdet == PixelSubdetector::PixelBarrel ) || ( subdet == PixelSubdetector::PixelEndcap )) 87 | { 88 | if( const SiPixelRecHit* pixel = dynamic_cast( rh )) 89 | { 90 | const SiPixelCluster& c = *( pixel->cluster()); 91 | fireworks::pushPixelCluster( pixelPoints, *geom, id, c, geom->getParameters( id )); 92 | } 93 | } 94 | } 95 | } 96 | 97 | // local function to get SiStripClusters, see 98 | // http://cmslxr.fnal.gov/source/Fireworks/Tracks/src/TrackUtils.cc#0422 99 | void 100 | SiStripClusters( std::vector &points, FWGeometry *geom, const reco::Track &t ); 101 | void 102 | SiStripClusters( std::vector &points, FWGeometry *geom, const reco::Track &t ) 103 | { 104 | // const edmNew::DetSetVector * allClusters = 0; 105 | for( trackingRecHit_iterator it = t.recHitsBegin(), itEnd = t.recHitsEnd(); it != itEnd; ++it ) 106 | { 107 | unsigned int rawid = (*it)->geographicalId(); 108 | if( ! geom->contains( rawid )) 109 | { 110 | fwLog( fwlog::kError ) 111 | << "failed to get geometry of SiStripCluster with detid: " 112 | << rawid << std::endl; 113 | 114 | continue; 115 | } 116 | 117 | const float* pars = geom->getParameters( rawid ); 118 | 119 | // -- get phi from SiStripHit 120 | auto rechitRef = *it; 121 | const TrackingRecHit *rechit = &( *rechitRef ); 122 | const SiStripCluster *cluster = fireworks::extractClusterFromTrackingRecHit( rechit ); 123 | if (cluster) 124 | { 125 | short firststrip = cluster->firstStrip(); 126 | float localTop[3] = { 0.0, 0.0, 0.0 }; 127 | float localBottom[3] = { 0.0, 0.0, 0.0 }; 128 | 129 | fireworks::localSiStrip( firststrip, localTop, localBottom, pars, rawid ); 130 | 131 | float globalTop[3]; 132 | float globalBottom[3]; 133 | geom->localToGlobal( rawid, localTop, globalTop, localBottom, globalBottom ); 134 | TVector3 pt( globalTop[0], globalTop[1], globalTop[2] ); 135 | points.push_back( pt ); 136 | TVector3 pb( globalBottom[0], globalBottom[1], globalBottom[2] ); 137 | points.push_back( pb ); 138 | } 139 | } 140 | } 141 | 142 | // 143 | // class declaration 144 | // 145 | 146 | // If the analyzer does not use TFileService, please remove 147 | // the template argument to the base class so the class inherits 148 | // from edm::one::EDAnalyzer<> and also remove the line from 149 | // constructor "usesResource("TFileService");" 150 | // This will improve performance in multithreaded jobs. 151 | 152 | class DemoAnalyzer : public edm::one::EDAnalyzer { 153 | public: 154 | explicit DemoAnalyzer(const edm::ParameterSet&); 155 | ~DemoAnalyzer(); 156 | 157 | static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); 158 | 159 | 160 | private: 161 | virtual void beginJob() override; 162 | virtual void analyze(const edm::Event&, const edm::EventSetup&) override; 163 | virtual void endJob() override; 164 | 165 | // ----------member data --------------------------- 166 | // c2numpy 167 | c2numpy_writer writer; 168 | 169 | // FWGeometry 170 | FWGeometry *geom; 171 | 172 | // hits constrains 173 | int max_pxhits = 5; 174 | int max_sihits = 50; 175 | }; 176 | 177 | // 178 | // constants, enums and typedefs 179 | // 180 | 181 | // 182 | // static data member definitions 183 | // 184 | 185 | // 186 | // constructors and destructor 187 | // 188 | DemoAnalyzer::DemoAnalyzer(const edm::ParameterSet& iConfig) 189 | 190 | { 191 | //now do what ever initialization is needed 192 | // usesResource("TFileService"); 193 | // 194 | // load geometry 195 | geom = new FWGeometry(); 196 | const char* filename="cmsGeom10.root"; 197 | geom->loadMap(filename); 198 | 199 | // c2numpy 200 | c2numpy_init(&writer, "output/params", 1000); 201 | 202 | c2numpy_addcolumn(&writer, "run", C2NUMPY_INTC); 203 | c2numpy_addcolumn(&writer, "evt", C2NUMPY_INTC); 204 | c2numpy_addcolumn(&writer, "lumi", C2NUMPY_INTC); 205 | c2numpy_addcolumn(&writer, "TrackId", C2NUMPY_INTC); 206 | c2numpy_addcolumn(&writer, "charge", C2NUMPY_INTC); 207 | 208 | c2numpy_addcolumn(&writer, "chi2", C2NUMPY_FLOAT64); 209 | c2numpy_addcolumn(&writer, "ndof", C2NUMPY_FLOAT64); 210 | c2numpy_addcolumn(&writer, "normalizedChi2", C2NUMPY_FLOAT64); 211 | c2numpy_addcolumn(&writer, "qoverp", C2NUMPY_FLOAT64); 212 | c2numpy_addcolumn(&writer, "theta", C2NUMPY_FLOAT64); 213 | c2numpy_addcolumn(&writer, "lambda", C2NUMPY_FLOAT64); 214 | c2numpy_addcolumn(&writer, "dxy", C2NUMPY_FLOAT64); 215 | c2numpy_addcolumn(&writer, "d0", C2NUMPY_FLOAT64); 216 | c2numpy_addcolumn(&writer, "dsz", C2NUMPY_FLOAT64); 217 | c2numpy_addcolumn(&writer, "dz", C2NUMPY_FLOAT64); 218 | c2numpy_addcolumn(&writer, "p", C2NUMPY_FLOAT64); 219 | c2numpy_addcolumn(&writer, "pt", C2NUMPY_FLOAT64); 220 | c2numpy_addcolumn(&writer, "px", C2NUMPY_FLOAT64); 221 | c2numpy_addcolumn(&writer, "py", C2NUMPY_FLOAT64); 222 | c2numpy_addcolumn(&writer, "pz", C2NUMPY_FLOAT64); 223 | c2numpy_addcolumn(&writer, "eta", C2NUMPY_FLOAT64); 224 | c2numpy_addcolumn(&writer, "phi", C2NUMPY_FLOAT64); 225 | c2numpy_addcolumn(&writer, "vx", C2NUMPY_FLOAT64); 226 | c2numpy_addcolumn(&writer, "vy", C2NUMPY_FLOAT64); 227 | c2numpy_addcolumn(&writer, "vz", C2NUMPY_FLOAT64); 228 | 229 | for (auto i = 0; i < max_pxhits; ++i) { // number of pixel hits 230 | std::ostringstream name; 231 | name << "pix_" << i; 232 | for (auto j = 0; j < 3; ++j) { // 3 coordinates 233 | std::ostringstream cname; 234 | if(j==0) cname << name.str() << "_x"; 235 | if(j==1) cname << name.str() << "_y"; 236 | if(j==2) cname << name.str() << "_z"; 237 | c2numpy_addcolumn(&writer, cname.str().c_str(), C2NUMPY_FLOAT64); 238 | } 239 | } 240 | 241 | for (auto i = 0; i < max_sihits; ++i) { // number of sistrip clusters 242 | std::ostringstream name; 243 | name << "sis_" << i; 244 | for (auto j = 0; j < 3; ++j) { // 3 coordinates 245 | std::ostringstream cname; 246 | if(j==0) cname << name.str() << "_x"; 247 | if(j==1) cname << name.str() << "_y"; 248 | if(j==2) cname << name.str() << "_z"; 249 | c2numpy_addcolumn(&writer, cname.str().c_str(), C2NUMPY_FLOAT64); 250 | } 251 | } 252 | 253 | // a module register what data it will request from the Event, Chris' suggestion 254 | consumes(edm::InputTag("generalTracks")); 255 | 256 | } 257 | 258 | 259 | DemoAnalyzer::~DemoAnalyzer() 260 | { 261 | 262 | // do anything here that needs to be done at desctruction time 263 | // (e.g. close files, deallocate resources etc.) 264 | geom->clear(); 265 | delete geom; 266 | 267 | } 268 | 269 | 270 | // 271 | // member functions 272 | // 273 | 274 | // ------------ method called for each event ------------ 275 | void 276 | DemoAnalyzer::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) 277 | { 278 | using namespace edm; 279 | 280 | 281 | Handle tracks; 282 | iEvent.getByLabel("generalTracks", tracks); 283 | LogInfo("Demo") << "number of tracks "<size(); 284 | 285 | // get event id 286 | auto eid = iEvent.id(); 287 | 288 | // c2numpy 289 | int tidx = 0; 290 | for (auto track = tracks->cbegin(); track != tracks->end(); ++track, ++tidx) { 291 | // extract track parameters 292 | 293 | c2numpy_intc(&writer, eid.run()); 294 | c2numpy_intc(&writer, eid.event()); 295 | c2numpy_intc(&writer, eid.luminosityBlock()); 296 | c2numpy_intc(&writer, tidx); 297 | c2numpy_intc(&writer, track->charge()); 298 | 299 | c2numpy_float64(&writer, track->chi2()); 300 | c2numpy_float64(&writer, track->ndof()); 301 | c2numpy_float64(&writer, track->normalizedChi2()); 302 | c2numpy_float64(&writer, track->qoverp()); 303 | c2numpy_float64(&writer, track->theta()); 304 | c2numpy_float64(&writer, track->lambda()); 305 | c2numpy_float64(&writer, track->dxy()); 306 | c2numpy_float64(&writer, track->d0()); 307 | c2numpy_float64(&writer, track->dsz()); 308 | c2numpy_float64(&writer, track->dz()); 309 | c2numpy_float64(&writer, track->p()); 310 | c2numpy_float64(&writer, track->pt()); 311 | c2numpy_float64(&writer, track->px()); 312 | c2numpy_float64(&writer, track->py()); 313 | c2numpy_float64(&writer, track->pz()); 314 | c2numpy_float64(&writer, track->eta()); 315 | c2numpy_float64(&writer, track->phi()); 316 | c2numpy_float64(&writer, track->vx()); 317 | c2numpy_float64(&writer, track->vy()); 318 | c2numpy_float64(&writer, track->vz()); 319 | 320 | // extract Pixel hits 321 | int npxhits = 0; 322 | std::vector pxpoints; 323 | pixelHits( pxpoints, geom, *track ); 324 | for( auto it = pxpoints.begin(), itEnd = pxpoints.end(); it != itEnd; ++it, ++npxhits) { 325 | if (npxhitsx()); 327 | c2numpy_float64(&writer, it->y()); 328 | c2numpy_float64(&writer, it->z()); 329 | } 330 | } 331 | // fill the rest 332 | for(auto i=npxhits; i < max_pxhits; ++i){ 333 | c2numpy_float64(&writer, 0.); // init x 334 | c2numpy_float64(&writer, 0.); // init y 335 | c2numpy_float64(&writer, 0.); // init z 336 | } 337 | 338 | // extract SiStripClusters 339 | int nsihits = 0; 340 | std::vector sipoints; 341 | SiStripClusters(sipoints, geom, *track); 342 | for( auto it = sipoints.begin(), itEnd = sipoints.end(); it != itEnd; ++it, ++nsihits) { 343 | if (nsihitsx()); 345 | c2numpy_float64(&writer, it->y()); 346 | c2numpy_float64(&writer, it->z()); 347 | } 348 | } 349 | // fill the rest 350 | for(auto i=nsihits; i < max_sihits; ++i){ 351 | c2numpy_float64(&writer, 0.); // init x 352 | c2numpy_float64(&writer, 0.); // init y 353 | c2numpy_float64(&writer, 0.); // init z 354 | } 355 | LogInfo("Demo") << "track "<< tidx << "# pixel hits" << npxhits << "# sihits" << nsihits; 356 | 357 | } 358 | 359 | } 360 | 361 | 362 | // ------------ method called once each job just before starting event loop ------------ 363 | void 364 | DemoAnalyzer::beginJob() 365 | { 366 | } 367 | 368 | // ------------ method called once each job just after ending the event loop ------------ 369 | void 370 | DemoAnalyzer::endJob() 371 | { 372 | // c2numpy 373 | c2numpy_close(&writer); 374 | } 375 | 376 | // ------------ method fills 'descriptions' with the allowed parameters for the module ------------ 377 | void 378 | DemoAnalyzer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { 379 | //The following says we do not know what parameters are allowed so do no validation 380 | // Please change this to state exactly what you do use, even if it is no parameters 381 | edm::ParameterSetDescription desc; 382 | desc.setUnknown(); 383 | descriptions.addDefault(desc); 384 | } 385 | 386 | //define this as a plug-in 387 | DEFINE_FWK_MODULE(DemoAnalyzer); 388 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/Demo/DemoAnalyzer/python/CfiFile_cfi.py: -------------------------------------------------------------------------------- 1 | import FWCore.ParameterSet.Config as cms 2 | 3 | demo = cms.EDAnalyzer('DemoAnalyzer' 4 | ) 5 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/Demo/DemoAnalyzer/python/ConfFile_cfg.py: -------------------------------------------------------------------------------- 1 | import FWCore.ParameterSet.Config as cms 2 | 3 | process = cms.Process("Demo") 4 | 5 | # initialize MessageLogger and output report 6 | process.load("FWCore.MessageLogger.MessageLogger_cfi") 7 | process.MessageLogger.cerr.threshold = 'INFO' 8 | process.MessageLogger.categories.append('Demo') 9 | process.MessageLogger.cerr.INFO = cms.untracked.PSet( 10 | limit = cms.untracked.int32(-1) 11 | ) 12 | process.options = cms.untracked.PSet( wantSummary = cms.untracked.bool(True) ) 13 | 14 | # process all events 15 | #process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) ) 16 | # process 10 events 17 | process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(10) ) 18 | 19 | process.source = cms.Source("PoolSource", 20 | # replace 'myfile.root' with the source file you want to use 21 | fileNames = cms.untracked.vstring( 22 | 'file:/afs/cern.ch/cms/Tutorials/TWIKI_DATA/TTJets_8TeV_53X.root' 23 | ) 24 | ) 25 | 26 | process.dump=cms.EDAnalyzer('EventContentAnalyzer') 27 | 28 | process.demo = cms.EDAnalyzer('DemoAnalyzer' 29 | ) 30 | 31 | 32 | # process.p = cms.Path(process.demo) 33 | process.p = cms.Path(process.demo*process.dump) 34 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/Demo/DemoAnalyzer/python/__init__.py: -------------------------------------------------------------------------------- 1 | #Automatically created by SCRAM 2 | import os 3 | __path__.append(os.path.dirname(os.path.abspath(__file__).rsplit('/Demo/DemoAnalyzer/',1)[0])+'/cfipython/slc6_amd64_gcc530/Demo/DemoAnalyzer') 4 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/README.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | 3 | This example shows the use of c2numpy 1.1 (C interface) in CMSSW framework. The directory contains files and directories generated by the `mkedanlzr` script of CMSSW; we assume readers are familiar with that environment. If not you can follow up the CMSSW standard procedure [how to write EDAnalyzer](https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookWriteFrameworkModule) 4 | 5 | ## Getting started 6 | 7 | To reproduce it, run 8 | 9 | ```bash 10 | cmsrel CMSSW_8_0_16 11 | cd CMSSW_8_0_16/src 12 | cmsenv 13 | ``` 14 | 15 | and copy in the `Demo` directory with its contents. Then 16 | 17 | ```bash 18 | scram build 19 | ``` 20 | 21 | to compile it. 22 | 23 | ## What was added 24 | 25 | We added c2numpy to this analyzer by copying the header file into `Demo/DemoAnalyzer/interface/c2numpy.h` (for some projects, you need to create the `interface` directory first). Then it was called by 26 | 27 | 1. Adding `#include "Demo/DemoAnalyzer/interface/c2numpy.h"` at the top of `Demo/DemoAnalyzer/plugins/DemoAnalyzer.cc`. 28 | 29 | 2. Adding `c2numpy_writer writer;` as member data in the `DemoAnalyzer` class. 30 | 31 | 3. Adding a 32 | 33 | ```c++ 34 | c2numpy_init(&writer, "output/params", 1000); 35 | c2numpy_addcolumn(&writer, "pt", C2NUMPY_FLOAT64); 36 | c2numpy_addcolumn(&writer, "eta", C2NUMPY_FLOAT64); 37 | c2numpy_addcolumn(&writer, "phi", C2NUMPY_FLOAT64); 38 | c2numpy_addcolumn(&writer, "dxy", C2NUMPY_FLOAT64); 39 | c2numpy_addcolumn(&writer, "dz", C2NUMPY_FLOAT64); 40 | ``` 41 | 42 | block in the `DemoAnalyzer` constructor. 43 | 4. Adding a 44 | 45 | ```c++ 46 | for (auto track = tracks->cbegin(); track != tracks->end(); ++track) { 47 | c2numpy_float64(&writer, track->pt()); 48 | c2numpy_float64(&writer, track->eta()); 49 | c2numpy_float64(&writer, track->phi()); 50 | c2numpy_float64(&writer, track->dxy()); 51 | c2numpy_float64(&writer, track->dz()); 52 | } 53 | ``` 54 | block in the `DemoAnalyzer::analyze` method. 55 | 5. **This is important!** Adding a `c2numpy_close(&writer);` statement in `DemoAnalyzer::endJob()`. 56 | 57 | The future C++ API will make this (easily forgotten) step 5 unnecessary. 58 | 59 | ## How to run it 60 | 61 | If you're testing this on CERN LXPLUS or another AFS mounted machine (to see the dataset at `/afs/cern.ch/cms/Tutorials/TWIKI_DATA/TTJets_8TeV_53X.root`), simply do 62 | 63 | ```bash 64 | mkdir output 65 | cmsRun Demo/DemoAnalyzer/python/ConfFile_cfg.py 66 | ``` 67 | 68 | When CMSSW runs over the 10-event dataset, it produces seven files in `output` named `params0.npy` through `params6.npy`. To make one file, increase the number of rows per file (currently 1000) in the `c2numpy_init` call, recompile, and rerun. 69 | 70 | These files should match the ones in this repository. 71 | 72 | ## What to do with the output 73 | 74 | You can load any of the files in a Python session. Start a Python prompt and do 75 | 76 | ```python 77 | import numpy as np 78 | >>> params0 = np.load("output/params0.npy") 79 | ``` 80 | 81 | to get the recarray. This array represents a table of data with typed, named columns (just like a flat ROOT NTuple or a Pandas DataFrame). You can access it a column at a time: 82 | 83 | ```python 84 | >>> params0["pt"] 85 | array([ 0.58615864, 0.47248294, 1.74911408, 1.04643586, 86 | 0.47096605, 0.59780441, 0.88029248, 3.06881774, 87 | 0.66100567, 1.33790879, 0.65042406, 0.60015372, 88 | 0.86379886, 0.57943386, 0.53491224, 1.0408088 , 89 | 1.55475394, 1.02831484, 0.47554129, 1.45042064, 90 | ... 91 | 92 | # to see array shape we can use built in numpy functionality 93 | >>> print np.shape(params0) 94 | (1000,) 95 | ``` 96 | 97 | a row at a time: 98 | 99 | ```python 100 | >>> params0[100] 101 | (1.384070332555011, 0.18779495889500655, -1.6946575614328503, 0.19390893548491872, 0.39640133641298614) 102 | >>> params0.dtype.names 103 | ('pt', 'eta', 'phi', 'dxy', 'dz') 104 | >>> params0.dtype["pt"] 105 | dtype('float64') 106 | ``` 107 | 108 | or any combination: 109 | 110 | ```python 111 | >>> params0["pt"][100:150] 112 | array([ 1.38407033, 0.5997181 , 2.64444269, 1.17549855, 2.81446767, 113 | 1.41189475, 2.28960126, 0.56264374, 1.43745931, 0.78332034, 114 | 1.34970658, 0.81638258, 1.52328245, 0.60757518, 1.1785421 , 115 | 0.81772201, 0.74880223, 0.69483397, 1.61993328, 0.59333001, 116 | 1.15131582, 0.49265859, 2.57502517, 1.07178172, 0.67694687, 117 | 0.463539 , 1.4918132 , 0.80275165, 0.86756021, 0.66508061, 118 | 0.46361261, 1.27049738, 0.80131647, 0.59497201, 0.48336115, 119 | 0.44257204, 0.47842197, 0.79463124, 0.77059007, 1.04439877, 120 | 0.78301175, 0.53878087, 0.66803512, 1.03883587, 1.72249824, 121 | 1.22609581, 0.96652589, 0.45579263, 0.62061272, 0.60283977]) 122 | ``` 123 | 124 | ## Importing to a Pandas DataFrame 125 | 126 | Let's convert this into pandas DataFrame and see how we can access the data 127 | 128 | ```python 129 | >>> import pandas as pd 130 | >>> df = pd.DataFrame(params0) 131 | >>> # access first tree rows 132 | >>> print df.iloc[:3] 133 | dxy dz 134 | 0 0.586159 0.088209 -3.124979 -0.374269 1.817592 135 | 1 0.472483 0.482395 -2.954974 -0.396636 9.588100 136 | 2 1.749114 -0.549223 -3.007367 -0.363395 -8.003905 137 | ``` 138 | 139 | Pandas dataframe provides a convenient way to work with dataframes, 140 | for instance you may inspect the dataframe and see its shape, columns, etc. 141 | 142 | ```python 143 | >>> print df.columns 144 | Index([u'pt', u'eta', u'phi', u'dxy', u'dz'], dtype='object') 145 | ``` 146 | 147 | ## Ready for Machine Learning (ML) challenge 148 | 149 | Now, let's use our dataframe for ML studies, such as classification problem. 150 | To do that we need a *labels* which will be used by ML classifier. 151 | For simplicity we'll randomly generate it: 152 | 153 | ```python 154 | >>> import random 155 | >>> # we have 1000 rows, therefore we need 1000 labels 156 | >>> labels = [random.randint(0,1) for _ in range(1000)] 157 | ``` 158 | 159 | At this step we're ready to apply any ML techniques and for demonstration 160 | purposes we'll use well-known python [scikit-learn library](http://scikit-learn.org/). 161 | 162 | ```python 163 | >>> # import scikit learn RandomForest classifier 164 | >>> from sklearn.ensemble import RandomForestClassifier 165 | >>> from sklearn.cross_validation import train_test_split 166 | 167 | >>> # split data into train/validation/test sets 168 | >>> x_train, x_rest, y_train, y_rest = train_test_split(df, labels, test_size=0.3, random_state=12345) 169 | 170 | >>> # train the model and generate predictions 171 | >>> fit = clf.fit(x_train, y_train) 172 | >>> predictions = fit.predict(x_rest) 173 | ``` 174 | 175 | At this step our predictions are ready. Of course, in this simple example 176 | they're meaningless, but this example shows full pipeline how someone 177 | can go from CMSSW root based data into ML pipeline in very simple steps. 178 | -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams0.npy -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams1.npy -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams2.npy -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams3.npy -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams4.npy -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams5.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams5.npy -------------------------------------------------------------------------------- /examples/CMSSW-with-C-interface/output/trackparams6.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diana-hep/c2numpy/579a5bf4467cdcb946bdc46add056b68cd4568f3/examples/CMSSW-with-C-interface/output/trackparams6.npy -------------------------------------------------------------------------------- /test.c: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Jim Pivarski 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "c2numpy.h" 19 | 20 | int main(int argc, char **argv) { 21 | printf("start\n"); 22 | 23 | c2numpy_writer writer; 24 | 25 | c2numpy_init(&writer, "testout", 1000); 26 | c2numpy_addcolumn(&writer, "one", C2NUMPY_INTC); 27 | c2numpy_addcolumn(&writer, "two", C2NUMPY_FLOAT64); 28 | c2numpy_addcolumn(&writer, "three", (c2numpy_type)((int)C2NUMPY_STRING + 5)); 29 | 30 | // completely optional: writing will open a file if not explicitly called 31 | c2numpy_open(&writer); 32 | 33 | printf("row %d by separate calls\n", writer.currentRowInFile); 34 | printf(" col %d\n", writer.currentColumn); 35 | c2numpy_intc(&writer, 3); 36 | printf(" col %d\n", writer.currentColumn); 37 | c2numpy_float64(&writer, 3.3); 38 | printf(" col %d\n", writer.currentColumn); 39 | c2numpy_string(&writer, "THREE"); 40 | 41 | printf("row %d by separate calls\n", writer.currentRowInFile); 42 | printf(" col %d\n", writer.currentColumn); 43 | c2numpy_intc(&writer, 4); 44 | printf(" col %d\n", writer.currentColumn); 45 | c2numpy_float64(&writer, 4.4); 46 | printf(" col %d\n", writer.currentColumn); 47 | c2numpy_string(&writer, "FOUR"); 48 | 49 | printf("row %d by separate calls\n", writer.currentRowInFile); 50 | printf(" col %d\n", writer.currentColumn); 51 | c2numpy_intc(&writer, 8); 52 | printf(" col %d\n", writer.currentColumn); 53 | c2numpy_float64(&writer, 8.8); 54 | printf(" col %d\n", writer.currentColumn); 55 | c2numpy_string(&writer, "EIGHT"); 56 | 57 | printf("close\n"); 58 | c2numpy_close(&writer); 59 | 60 | printf("end\n"); 61 | } 62 | --------------------------------------------------------------------------------