├── .github └── workflows │ ├── py-in-mem.yml │ └── pyext.yml ├── .gitignore ├── LICENSE ├── README.md ├── first-contact └── README.md ├── grpc ├── Dockerfile ├── README.md ├── client.go ├── client_test.go ├── gen.go ├── go.mod ├── go.sum ├── outliers.proto ├── pb │ └── outliers.pb.go └── py │ ├── Makefile │ ├── outliers_pb2.py │ ├── outliers_pb2_grpc.py │ ├── requirements.txt │ └── server.py ├── lisp ├── README-1.md ├── collatz.clj ├── collatz.go └── collatz.py ├── py-bindings └── outliers │ ├── README.md │ ├── build.sh │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── pkg-config │ └── py385-Darwin-x86_64 │ │ ├── README │ │ └── python3.pc │ ├── pyoutliers │ ├── __init__.py │ └── detect.py │ └── set_env.sh ├── py-in-mem ├── .gitignore ├── Dockerfile.test ├── Makefile ├── README.md ├── bench.ipy ├── bench.txt ├── doc.go ├── example.go ├── glue.c ├── glue.h ├── go.mod ├── go.sum ├── images │ ├── data-flow.json │ ├── data-flow.png │ ├── data-ownership.json │ ├── data-ownership.png │ ├── go-py-stack.json │ ├── go-py-stack.png │ ├── go-to-py.json │ ├── go-to-py.png │ ├── py-go-stack.json │ ├── py-go-stack.png │ ├── py-to-go.json │ └── py-to-go.png ├── outliers.go ├── outliers.py └── outliers_test.go └── pyext ├── .gitignore ├── Dockerfile.test-a ├── Dockerfile.test-b ├── MANIFEST.in ├── Makefile ├── README-A.md ├── README-B.md ├── README.md ├── bench ├── .gitignore ├── Makefile ├── bench.ipy ├── bench.proto ├── client.py ├── go.mod ├── go.sum ├── server.go └── so.go ├── checksig.go ├── checksig.py ├── checksig_test.go ├── example.py ├── export.go ├── func-call.json ├── func-call.png ├── go.mod ├── go.sum ├── image.md ├── pip-install.json ├── pip-install.png ├── py_session.py ├── setup.py ├── test_checksig.py └── testdata └── logs ├── httpd-00.log ├── httpd-01.log ├── httpd-02.log ├── httpd-03.log ├── httpd-04.log ├── httpd-05.log ├── httpd-06.log ├── httpd-07.log ├── httpd-08.log ├── httpd-09.log └── sha1sum.txt /.github/workflows/py-in-mem.yml: -------------------------------------------------------------------------------- 1 | name: py-in-mem 2 | on: [push] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v2 8 | - name: Run test docker 9 | working-directory: ./py-in-mem 10 | run: docker build -f Dockerfile.test . 11 | -------------------------------------------------------------------------------- /.github/workflows/pyext.yml: -------------------------------------------------------------------------------- 1 | name: pyext 2 | on: [push] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v2 8 | - name: Run test docker A 9 | working-directory: ./pyext 10 | run: docker build -f Dockerfile.test-a . 11 | - name: Run test docker B 12 | working-directory: ./pyext 13 | run: docker build -f Dockerfile.test-b . 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # py-bindings executable 18 | py-bindings/outliers/outliers 19 | py-bindings/outliers/outliers.exe 20 | 21 | # python 22 | *.pyc 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-go 2 | Training material on how to leverage Python and Go together to solve different problems. 3 | -------------------------------------------------------------------------------- /first-contact/README.md: -------------------------------------------------------------------------------- 1 | # First Contact With Data 2 | 3 | > Every single company I've worked at and talked to has the same problem without a single exception so far - poor data quality, especially tracking data. Either there's incomplete data, missing tracking data, duplicative tracking data. 4 | > - DJ Patil 5 | 6 | I spend a lot of my time digging in various companies' data. Every time, I am surprised by what I'm seeing there, and every time, the engineers and analysts at the company are surprised as well. I've seen missing data, bad data, data nobody knows what it means and many other oddities. 7 | 8 | As a data scientist, the quality of the data you work with is crucial to your success. The old GIGO acronym, which stands for "garbage in, garbage out" is very true. In this blog post we'll discuss some methods and practices that will help you with your first contact with data that will save you a lot of grief down the road. 9 | 10 | I'm going to assume you'll be using [pandas](https://pandas.pydata.org/) to process the data. I'll be using pandas version 1.1 and Python 3.5. The code shown here is an [IPython](https://ipython.org/) session. 11 | 12 | ### Schema 13 | 14 | All data have a schema, it is either formally written somewhere or in about 1,000 places in the code. Try to find the schema for the data you're working with, ask people about it - and don't trust it until you compare it with the real data. 15 | 16 | Let's have a look at sample weather data from [NOAA](https://www.noaa.gov/weather). The data was originally in CSV format, and I've indented it for readability. 17 | 18 | **Listing 1: Weather Data** 19 | ``` 20 | DATE SNOW TMAX TMIN PGTM 21 | 2000-01-01 0 100 11 1337 22 | 2000-01-02 0 156 61 2313 23 | 2000-01-03 0 178 106 320 24 | 2000-01-04 0 156 78 1819 25 | 2000-01-05 0 83 -17 843 26 | ``` 27 | 28 | Without a schema, it's hard to know what's going on here. The `DATE` column is obvious, but the rest are unclear: 29 | 30 | - What is `SNOW`? How much fell? If so is it inches? centimeters? ... Maybe it's a boolean for yes/no? 31 | - `TMAX` and `TMIN` are probably the maximal and minimal temperature at the day. What are the units? Both Celsius and Fahrenheit don't make sense - an 89 difference in one day? 32 | - And what does `PGTM` stand for? What are these numbers? 33 | 34 | It's clear that types (string, integer, float ...) are not enough. We need units and maybe more to understand what's going one 35 | 36 | **Listing 2: Weather Schema** 37 | 38 | ``` 39 | TMAX - Maximum temperature (tenths of degrees C) 40 | TMIN - Minimum temperature (tenths of degrees C) 41 | SNOW - Snowfall (mm) 42 | PGTM - Peak gust time (hours and minutes, i.e., HHMM) 43 | ``` 44 | 45 | One we read the schema, things become clear. `TMAX` and `TMIN` values make sense, `SNOW` is in millimeters and `PGTM` values are actually time of day. 46 | 47 | If you have a say in your company, try to see that all data have a formal written schema, and that this schema is kept up to date. 48 | 49 | Even if your company keeps schemas and updates them, data will still have errors. As agent Mulder used to say: "Trust no one!". Always look at the raw data and see that it matches your assumptions about it. 50 | 51 | ### Size Matters 52 | 53 | pandas is built to work in memory and by default will load the whole data into memory. Some datasets are too big to fit in memory, and once you exhaust the computer's physical memory and start to swap to disk - performance goes down the drain. 54 | 55 | Load a small part of the data initially, and then extrapolate to guess the final size it'll take in memory. If you're working with a database that's pretty easy - add a `LIMIT` clause to your `SELECT` statement and you're done. If the data is in file - you'll need to work harder. 56 | 57 | I'm to look at part of the [NYC Taxi Dataset](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). The data comes as a compressed CSV and I'd like to find out how much memory it'll take once loaded to a pandas DataFrame. 58 | 59 | The first thing I start with is to look at size on disk. 60 | 61 | **Listing 3: Disk Size** 62 | 63 | ``` 64 | In [1]: csv_file = 'yellow_tripdata_2018-05.csv.bz2' 65 | In [2]: from pathlib import Path 66 | In [3]: MB = 2**20 67 | In [4]: Path(csv_file).stat().st_size / MB 68 | Out[4]: 85.04909038543701 69 | ``` 70 | 71 | About 85MB compressed on disk. From my experience bz2 compresses text to about 10-15% from its original size. Uncompressed this data will be around 780MB on disk. 72 | 73 | The next thing is to find out how many lines there are. 74 | 75 | **List 4: Line Count** 76 | 77 | ``` 78 | In [5]: import bz2 79 | In [6]: with bz2.open(csv_file, 'rt') as fp: 80 | ...: num_lines = sum(1 for _ in fp) 81 | In [7]: num_lines 82 | Out[7]: 9224065 83 | In [8]: f'{num_lines:,}' 84 | Out[8]: '9,224,065' 85 | ``` 86 | 87 | On `5` we import the `bz2` and on `6` we sum a generator expression to get the number of lines in the file, this took about 40 seconds on my machine. 88 | On `8` I use an `f-string` to print the number in a more human readable format. 89 | 90 | We have 9.2 million lines in the file. Let's load 10,000, into a DataFrame, measure the size and calculate the whole size. 91 | 92 | **Listing 5: Calculating Size** 93 | 94 | ``` 95 | In [9]: import pandas as pd 96 | In [10]: nrows = 10_000 97 | In [11]: df = pd.read_csv(csv_file, nrows=10_000) 98 | In [12]: df.memory_usage(deep=True).sum() / MB 99 | Out[12]: 3.070953369140625 100 | In [13]: Out[12] * (num_lines / nrows) 101 | Out[13]: 2832.667348892212 102 | ``` 103 | 104 | On `11` we load 10,000 rows to a DataFrame. On `12` we calculate how much memory the DataFrame is consuming in MB and on `13` we calculate the total memory consumption for the whole data - about 2.8GB. The cat can fit in memory. 105 | 106 | _Note: If the data doesn't fit in your computer's memory, don't despair! There are way to load parts of data and reduce memory consumption. But probably the most effective solution is to lease a cloud machine with a lot of memory. Some cloud providers have machines with several **terabytes** of memory._ 107 | 108 | ### Raw Data 109 | 110 | Before you load the data, it's a good idea to look at it in it's raw format and see if it matches your assumptions about it. 111 | 112 | **Listing 6: Raw Data** 113 | 114 | ``` 115 | In [14]: with bz2.open(csv_file, 'rt') as fp: 116 | ...: for i, line in enumerate(fp): 117 | ...: print(line.strip()) 118 | ...: if i == 3: 119 | ...: break 120 | ...: 121 | VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount 122 | 123 | 1,2018-05-01 00:13:56,2018-05-01 00:22:46,1,1.60,1,N,230,50,1,8,0.5,0.5,1.85,0,0.3,11.15 124 | 1,2018-05-01 00:23:26,2018-05-01 00:29:56,1,1.70,1,N,263,239,1,7.5,0.5,0.5,2,0,0.3,10.8 125 | ``` 126 | 127 | Looks like a CSV with a header line. Let's use the `csv` module to get rows. 128 | 129 | **Listing 7: Raw Data - CSV** 130 | 131 | ``` 132 | In [15]: from pprint import pprint 133 | In [16]: with bz2.open(csv_file, 'rt') as fp: 134 | ...: rdr = csv.DictReader(fp) 135 | ...: for i, row in enumerate(rdr): 136 | ...: pprint(row) 137 | ...: if i == 3: 138 | ...: break 139 | ...: 140 | {'DOLocationID': '50', 141 | 'PULocationID': '230', 142 | 'RatecodeID': '1', 143 | 'VendorID': '1', 144 | 'extra': '0.5', 145 | 'fare_amount': '8', 146 | 'improvement_surcharge': '0.3', 147 | 'mta_tax': '0.5', 148 | 'passenger_count': '1', 149 | 'payment_type': '1', 150 | 'store_and_fwd_flag': 'N', 151 | 'tip_amount': '1.85', 152 | 'tolls_amount': '0', 153 | 'total_amount': '11.15', 154 | 'tpep_dropoff_datetime': '2018-05-01 00:22:46', 155 | 'tpep_pickup_datetime': '2018-05-01 00:13:56', 156 | 'trip_distance': '1.60'} 157 | ... 158 | ``` 159 | 160 | On `15` we load the `pprint` module for more human readable printing. Then we use `csv.DictReader` to read 3 records and print them. Looking at the data it seems OK: datetime fields look like data & time, amounts look like floating point numbers etc. 161 | 162 | ### Data Types 163 | 164 | Once you see the raw data, and verify you can load the data to memory - you can load the data into a DataFrame. However remember that in CSV everything is text and pandas is guessing the types for you - so you need to check it. 165 | 166 | 167 | **Listing 8: Checking Types** 168 | 169 | ``` 170 | In [16]: df = pd.read_csv(csv_file) 171 | In [17]: df.dtypes 172 | Out[17]: 173 | VendorID int64 174 | tpep_pickup_datetime object 175 | tpep_dropoff_datetime object 176 | passenger_count int64 177 | trip_distance float64 178 | RatecodeID int64 179 | store_and_fwd_flag object 180 | PULocationID int64 181 | DOLocationID int64 182 | payment_type int64 183 | fare_amount float64 184 | extra float64 185 | mta_tax float64 186 | tip_amount float64 187 | tolls_amount float64 188 | improvement_surcharge float64 189 | total_amount float64 190 | dtype: object 191 | ``` 192 | 193 | On `16` we load the whole data into a DataFrame, this took about 45 seconds on my machine. On `17` we print out the data type for each column. 194 | 195 | Most of the column types seem OK, but `tpep_pickup_datetime` and `tpep_dropoff_datetime` are `object`. The `object` type usually means a string, and we'd like to have a time stamp here. This is a case where pandas need some help figuring out types. 196 | 197 | _Note: I hate the CSV format with a passion - there's no type information, no formal specification, and don't get me started on Unicode ... If you have a say - pick a different format which has type information. My default storage format is [SQlite](https://www.sqlite.org/) which is a one-file SQL database._ 198 | 199 | Let's help pandas figure out the types. 200 | 201 | **Listing 9: Fixing Types** 202 | 203 | ``` 204 | In [18]: time_cols = ['tpep_pickup_datetime', 'tpep_dropoff_datetime'] 205 | In [19]: df = pd.read_csv(csv_file, parse_dates=time_cols) 206 | In [20]: df.dtypes 207 | Out[20]: 208 | VendorID int64 209 | tpep_pickup_datetime datetime64[ns] 210 | tpep_dropoff_datetime datetime64[ns] 211 | passenger_count int64 212 | trip_distance float64 213 | RatecodeID int64 214 | store_and_fwd_flag object 215 | PULocationID int64 216 | DOLocationID int64 217 | payment_type int64 218 | fare_amount float64 219 | extra float64 220 | mta_tax float64 221 | tip_amount float64 222 | tolls_amount float64 223 | improvement_surcharge float64 224 | total_amount float64 225 | dtype: object 226 | ``` 227 | 228 | On `19` we tell pandas to parse the two time columns as dates. And by looking at the output of `20` we see now that we have the right types. 229 | 230 | ### Looking for Outliers 231 | 232 | Once the data is loaded and in the right types, it's time to look for bad values. The definition of "bad data" depends on the data you're working with. For example if you have a `temperature` column, the maximal value probably shouldn't be more than 60°C (the highest temperature ever recorded was 56.7°C). But - what if we're talking about engine temperature? 233 | 234 | One of the easy ways to start is to use the DataFrame's `describe` method. Since our DataFrame has many columns, I'm going to look at a subset of the columns. 235 | 236 | **Listing 10: Looking for Outliers** 237 | 238 | ``` 239 | In [21]: df[['trip_distance', 'total_amount', 'passenger_count']].describe() 240 | Out[21]: 241 | trip_distance total_amount passenger_count 242 | count 9.224063e+06 9.224063e+06 9.224063e+06 243 | mean 3.014031e+00 1.681252e+01 1.596710e+00 244 | std 3.886332e+00 7.864489e+01 1.245703e+00 245 | min 0.000000e+00 -4.858000e+02 0.000000e+00 246 | 25% 1.000000e+00 8.760000e+00 1.000000e+00 247 | 50% 1.650000e+00 1.225000e+01 1.000000e+00 248 | 75% 3.100000e+00 1.835000e+01 2.000000e+00 249 | max 9.108000e+02 2.346327e+05 9.000000e+00 250 | ``` 251 | 252 | Right away we see some fishy data: 253 | 254 | - The minimal `total_amount` is negative 255 | - The maximal `trip_distance` is 910 miles 256 | - The are rides with 0 passengers 257 | 258 | Sometimes you'll need to run a calculation to find outliers 259 | 260 | **Listing 11: Trip Duration** 261 | 262 | ``` 263 | In [22]: (df['tpep_dropoff_datetime'] - df['tpep_pickup_datetime']).describe() 264 | Out[22]: 265 | count 9224063 266 | mean 0 days 00:16:29.874877155 267 | std 2 days 10:35:51.816665095 268 | min -7410 days +13:10:08 269 | 25% 0 days 00:06:46 270 | 50% 0 days 00:11:28 271 | 75% 0 days 00:19:01 272 | max 0 days 23:59:59 273 | dtype: object 274 | ``` 275 | 276 | On `22` we calculate the trip duration and use `describe` to display statistics on it. 277 | 278 | - The minimal duration is negative (maybe someone invented a time machine?) 279 | - The maximal duration is a full day 280 | 281 | ### Conclusion 282 | 283 | I haven't met real data that didn't have errors in it. I've learned to keep my eyes open and challenge everything I *think* I know about the data before starting to process it. I urge you to follow these steps every time you start working with new data: 284 | 285 | - Find out the schema 286 | - Calculate data size 287 | - Look at the raw data 288 | - Check data types 289 | - Look for outliers 290 | 291 | This might seem like a lot of work, but I guarantee it'll save you much more work down the road when the models you’ve worked hard to develop start to misbehave. 292 | 293 | I'd love to hear your data horror stories, and how you handled them. Reach out to me at miki@353solutions and amaze me. 294 | 295 | 296 | -------------------------------------------------------------------------------- /grpc/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.14-buster 2 | WORKDIR /code 3 | COPY . . 4 | RUN go build client.go 5 | -------------------------------------------------------------------------------- /grpc/README.md: -------------------------------------------------------------------------------- 1 | # Go ↔ Python: Part I - gRPC 2 | 3 | Blog post [here](https://www.ardanlabs.com/blog/2020/06/python-go-grpc.html) 4 | 5 | ### Introduction 6 | 7 | Like tools, programming languages tend to solve problems they are designed to. You _can_ use a knife to tighten a screw, but it's better to use a screwdriver. Plus there is less chance of you getting hurt in the process. 8 | 9 | The Go programming language shines when writing high throughput services, and Python shines when used for data science. In this series of blog posts, we're going to explore how you can use each language to do the part it's better and explore various methods of communication between Go & Python. 10 | 11 | _Note: Using more than one language in a project has its cost. If you can write everything in Go or Python only - by all means do that. However, there are cases where using the right language for the job reduces the overall engineering overhead for readability, maintenance and performance._ 12 | 13 | In this post, we’ll learn how Go and Python programs can communicate between each other using [gRPC](https://grpc.io/). This post assumes basic knowledge in both Go and Python. 14 | 15 | ### gRPC Overview 16 | 17 | gRPC is a remote procedure call (RPC) framework from Google. It uses [Protocol Buffers](https://developers.google.com/protocol-buffers) as a serialization format and uses [HTTP2](https://en.wikipedia.org/wiki/HTTP/2) as the transport medium. 18 | 19 | By using these two well established technologies, you gain access to a lot of knowledge and tooling that's already available. Many companies I consult with use gRPC to connect internal services. 20 | 21 | Another advantage of using protocol buffers is that you write the message definition once, and then generate bindings to every language from the same source. This means that various services can be written in different programming languages and all the applications agree on the format of messages. 22 | 23 | Protocol buffers are also an efficient binary format: you gain both faster serialization times and less bytes over the wire, and this alone will save you money. Benchmarking on my machine, serialization time compared to JSON is about 7.5 faster and the data generated is about 4 times smaller. 24 | 25 | ### Example: Outlier Detection 26 | 27 | [Outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection), also called anomaly detection, is a way of finding suspicious values in data. Modern systems collect a lot of metrics from their services, and it's hard to come up with simple thresholds for finding malfunctioning services, which means waking on call developers at 2am too many times.. 28 | 29 | We’ll start by implementing a Go service that collects metrics. Then, using gRPC, we’ll send these metrics to a Python service, which will perform outlier detection on them. 30 | 31 | ### Project Structure 32 | 33 | In this project, we're going with a simple approach of having Go as the main project and Python as a sub-project in the source tree. 34 | 35 | **Listing 1** 36 | ``` 37 | . 38 | ├── client.go 39 | ├── gen.go 40 | ├── go.mod 41 | ├── go.sum 42 | ├── outliers.proto 43 | ├── pb 44 | │ └── outliers.pb.go 45 | └── py 46 | ├── Makefile 47 | ├── outliers_pb2_grpc.py 48 | ├── outliers_pb2.py 49 | ├── requirements.txt 50 | └── server.py 51 | ``` 52 | 53 | Listing1 shows the directory structure of our project. The project is using [Go modules](https://blog.golang.org/using-go-modules) and the name for the module is defined in the `go.mod` file (see listing 2). We’re going to reference the module name ( `github.com/ardanlabs/python-go/grpc`) in several places. 54 | 55 | **Listing 2** 56 | ``` 57 | 01 module github.com/ardanlabs/python-go/grpc 58 | 02 59 | 03 go 1.14 60 | 04 61 | 05 require ( 62 | 06 github.com/golang/protobuf v1.4.2 63 | 07 google.golang.org/grpc v1.29.1 64 | 08 google.golang.org/protobuf v1.23.0 65 | 09 ) 66 | ``` 67 | 68 | Listing 2 shows what the `go.mod` file looks like for our project. You can see on line 01 where the name of the module is defined. 69 | 70 | ### Defining Messages & Services 71 | 72 | In gRPC, you start by writing a `.proto` file which defines the messages being sent and the RPC methods. 73 | 74 | **Listing 3** 75 | ``` 76 | 01 syntax = "proto3"; 77 | 02 import "google/protobuf/timestamp.proto"; 78 | 03 package pb; 79 | 04 80 | 05 option go_package = "github.com/ardanlabs/python-go/grpc/pb"; 81 | 06 82 | 07 message Metric { 83 | 08 google.protobuf.Timestamp time = 1; 84 | 09 string name = 2; 85 | 10 double value = 3; 86 | 11 } 87 | 12 88 | 13 message OutliersRequest { 89 | 14 repeated Metric metrics = 1; 90 | 15 } 91 | 16 92 | 17 message OutliersResponse { 93 | 18 repeated int32 indices = 1; 94 | 19 } 95 | 20 96 | 21 service Outliers { 97 | 22 rpc Detect(OutliersRequest) returns (OutliersResponse) {} 98 | 23 } 99 | ``` 100 | 101 | Listing 3 shows what the `outliers.proto` looks like. Important things to mention are on line 02 in listing 3, where we import the protocol buffers definition of `timestamp`, and then on line 05, where we define the full Go package name - github.com/ardanlabs/python-go/grpc/pb 102 | 103 | A metric is a measurement of resource usage that is used for monitoring and diagnostics your system. We define a `Metric` on line 07, which has a timestamp, a name (e.g. "CPU"), and a float value. For example we can say that at `2020-03-14T12:30:14` we measured `41.2% CPU` utilization. 104 | 105 | Every RPC method has an input type (or types) and an output type. Our method `Detect` (line 22) uses an `OutliersRequest` message type (line 13) as the input and an `OutliersResponse` message type (line 17) for the output. The `OutliersRequest` message type is a list/slice of Metric and the `OutliersResponse` message type is a list/slice of indices where the outlier values were found. For example, if we have the values of `[1, 2, 100, 1, 3, 200, 1]`, the result will be `[2, 5]` which is the index of the outliers 100 and 200. 106 | 107 | ### Python Service 108 | 109 | In this section, we’ll go over the Python service code. 110 | 111 | **Listing 4** 112 | ``` 113 | . 114 | ├── client.go 115 | ├── gen.go 116 | ├── go.mod 117 | ├── go.sum 118 | ├── outliers.proto 119 | ├── pb 120 | │ └── outliers.pb.go 121 | └── py 122 | ├── Makefile 123 | ├── outliers_pb2_grpc.py 124 | ├── outliers_pb2.py 125 | ├── requirements.txt 126 | └── server.py 127 | ``` 128 | 129 | Listing 4 shows how the code for the Python service is in the `py` directory off the root of the project. 130 | 131 | To generate the Python bindings you'll need to install the `protoc` compiler, which you can download[here](https://developers.google.com/protocol-buffers/docs/downloads). You can also install the compiler using your operating system package manager (e.g. `apt-get`, `brew` ...) 132 | 133 | Once you have the compiler installed, you'll need to install the Python `grpcio-tools` package. 134 | 135 | _Note: I highly recommend using a virtual environment for all your Python projects. Read [this](https://docs.python.org/3/tutorial/venv.html) to learn more._ 136 | 137 | **Listing 5** 138 | ``` 139 | $ cat requirements.txt 140 | 141 | OUTPUT: 142 | grpcio-tools==1.29.0 143 | numpy==1.18.4 144 | 145 | $ python -m pip install -r requirements.txt 146 | ``` 147 | 148 | Listing 5 shows how to inspect and install the external dependencies for our Python project. The `requirements.txt` specifies the external dependencies for the project, very much like how `go.mod` specifies dependencies for Go projects. 149 | 150 | As you can see from the output of the `cat` command, we need two external dependencies: `grpcio-tools` and [numpy](https://numpy.org/) . A good practice is to have this file in source control and always version your dependencies (e.g. `numpy==1.18.4`), similar to what you would do with your `go.mod` for your Go projects. 151 | 152 | Once you have the dependencies installed, you can generate the Python bindings. 153 | 154 | **Listing 6** 155 | ``` 156 | $ python -m grpc_tools.protoc \ 157 | -I.. --python_out=. --grpc_python_out=. \ 158 | ../outliers.proto 159 | ``` 160 | 161 | Listing 6 shows how to generate the Python binding for the gRPC support. Let's break this long command in listing 5 down: 162 | 163 | * `python -m grpc_tools.protoc` runs the `grpc_tools.protoc` module as a script. 164 | * `-I..` tells the tool where `.proto` files can be found. 165 | * `--python_out=.` tells the tool to generate the protocol buffers serialization code in the current directory. 166 | * `--grpc_python_out=.` tells the tool to generate the gRPC code in the current directory. 167 | * `../outliers.proto` is the name of the protocol buffers + gRPC definitions file. 168 | 169 | This Python command will run without any output, and at the end you'll see two new files: `outliers_pb2.py` which is the protocol buffers code and `outliers_pb2_grpc.py` which is the gRPC client and server code. 170 | 171 | _Note: I usually use a `Makefile` to automate tasks in Python projects and create a `make` rule to run this command. I add the generated files to source control so that the deployment machine won't have to install the `protoc` compiler._ 172 | 173 | To write the Python service, you need to inherit from the `OutliersServicer` defined in `outliers_pb2_grpc.py` and override the `Detect` method. We're going to use the numpy package and use a simple method of picking all the values that are more than two [standard deviations](https://en.wikipedia.org/wiki/Standard_deviation) from the [mean](https://en.wikipedia.org/wiki/Mean). 174 | 175 | **Listing 7** 176 | ``` 177 | 01 import logging 178 | 02 from concurrent.futures import ThreadPoolExecutor 179 | 03 180 | 04 import grpc 181 | 05 import numpy as np 182 | 06 183 | 07 from outliers_pb2 import OutliersResponse 184 | 08 from outliers_pb2_grpc import OutliersServicer, add_OutliersServicer_to_server 185 | 09 186 | 10 187 | 11 def find_outliers(data: np.ndarray): 188 | 12 """Return indices where values more than 2 standard deviations from mean""" 189 | 13 out = np.where(np.abs(data - data.mean()) > 2 * data.std()) 190 | 14 # np.where returns a tuple for each dimension, we want the 1st element 191 | 15 return out[0] 192 | 16 193 | 17 194 | 18 class OutliersServer(OutliersServicer): 195 | 19 def Detect(self, request, context): 196 | 20 logging.info('detect request size: %d', len(request.metrics)) 197 | 21 # Convert metrics to numpy array of values only 198 | 22 data = np.fromiter((m.value for m in request.metrics), dtype='float64') 199 | 23 indices = find_outliers(data) 200 | 24 logging.info('found %d outliers', len(indices)) 201 | 25 resp = OutliersResponse(indices=indices) 202 | 26 return resp 203 | 27 204 | 28 205 | 29 if __name__ == '__main__': 206 | 30 logging.basicConfig( 207 | 31 level=logging.INFO, 208 | 32 format='%(asctime)s - %(levelname)s - %(message)s', 209 | 33 ) 210 | 34 server = grpc.server(ThreadPoolExecutor()) 211 | 35 add_OutliersServicer_to_server(OutliersServer(), server) 212 | 36 port = 9999 213 | 37 server.add_insecure_port(f'[::]:{port}') 214 | 38 server.start() 215 | 39 logging.info('server ready on port %r', port) 216 | 40 server.wait_for_termination() 217 | ``` 218 | 219 | Listing 7 shows the code in the `server.py` file. This is all the code we needed to write the Python service. In line 19 we override `Detect` from the generated `OutlierServicer` and write the actual outlier detection code. In line 34 we create a gRPC server that uses a ThreadPoolExecutor to run requests in parallel and in line 35 we register our OutliersServer to handle requests in the server. 220 | 221 | **Listing 8** 222 | ``` 223 | $ python server.py 224 | 225 | OUTPUT: 226 | 2020-05-23 13:45:12,578 - INFO - server ready on port 9999 227 | ``` 228 | Listing 8 shows how to run the service. 229 | 230 | ### Go Client 231 | 232 | Now that we have our Python service running, we can write the Go client that will communicate with it. 233 | 234 | We'll start by generating Go bindings for gRPC. To automate this process, I usually have a file called `gen.go` with a `go:generate` command to generate the bindings. You will need to download the `github.com/golang/protobuf/protoc-gen-go` module which is the gRPC plugin for Go. 235 | 236 | **Listing 9** 237 | ``` 238 | 01 package main 239 | 02 240 | 03 //go:generate mkdir -p pb 241 | 04 //go:generate protoc --go_out=plugins=grpc:pb --go_opt=paths=source_relative outliers.proto 242 | ``` 243 | 244 | Listing 9 shows the `gen.go` file and how `go:generate` is used to execute the gRPC plugin to generate the bindings. 245 | 246 | Let's break down the command on line 04 which generates the bindings: 247 | 248 | * `protoc` is the protocol buffer compiler. 249 | * `--go-out=plugins=grpc:pb` tells `protoc` to use the gRPC plugin and place the files in the `pb` directory. 250 | * `--go_opt=source_relative` tells `protoc` to generate the code in the `pb` directory relative to the current directory. 251 | * `outliers.proto` is the name of the protocol buffers + gRPC definitions file. 252 | 253 | When you run `go generate` in the shell, you should see no output, but there will be a new file called `outliers.pb.go` in the `pb` directory. 254 | 255 | **Listing 10** 256 | ``` 257 | . 258 | ├── client.go 259 | ├── gen.go 260 | ├── go.mod 261 | ├── go.sum 262 | ├── outliers.proto 263 | ├── pb 264 | │ └── outliers.pb.go 265 | └── py 266 | ├── Makefile 267 | ├── outliers_pb2_grpc.py 268 | ├── outliers_pb2.py 269 | ├── requirements.txt 270 | └── server.py 271 | ``` 272 | 273 | Listing 10 shows the `pb` directory and the new file `outliers.pb.go` that was generated by the `go generate` call. I add the `pb` directory to source control so if the project is cloned to a new machine the project will work without requiring `protoc` to be installed on that machine. 274 | 275 | Now we can build and run the Go client. 276 | 277 | **Listing 11** 278 | ``` 279 | 01 package main 280 | 02 281 | 03 import ( 282 | 04 "context" 283 | 05 "log" 284 | 06 "math/rand" 285 | 07 "time" 286 | 08 287 | 09 "github.com/ardanlabs/python-go/grpc/pb" 288 | 10 "google.golang.org/grpc" 289 | 11 pbtime "google.golang.org/protobuf/types/known/timestamppb" 290 | 12 ) 291 | 13 292 | 14 func main() { 293 | 15 addr := "localhost:9999" 294 | 16 conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock()) 295 | 17 if err != nil { 296 | 18 log.Fatal(err) 297 | 19 } 298 | 20 defer conn.Close() 299 | 21 300 | 22 client := pb.NewOutliersClient(conn) 301 | 23 req := pb.OutliersRequest{ 302 | 24 Metrics: dummyData(), 303 | 25 } 304 | 26 305 | 27 resp, err := client.Detect(context.Background(), &req) 306 | 28 if err != nil { 307 | 29 log.Fatal(err) 308 | 30 } 309 | 31 log.Printf("outliers at: %v", resp.Indices) 310 | 32 } 311 | 33 312 | 34 func dummyData() []*pb.Metric { 313 | 35 const size = 1000 314 | 36 out := make([]*pb.Metric, size) 315 | 37 t := time.Date(2020, 5, 22, 14, 13, 11, 0, time.UTC) 316 | 38 for i := 0; i < size; i++ { 317 | 39 m := pb.Metric{ 318 | 40 Time: Timestamp(t), 319 | 41 Name: "CPU", 320 | 42 // Normally we're below 40% CPU utilization 321 | 43 Value: rand.Float64() * 40, 322 | 44 } 323 | 45 out[i] = &m 324 | 46 t.Add(time.Second) 325 | 47 } 326 | 48 // Create some outliers 327 | 49 out[7].Value = 97.3 328 | 50 out[113].Value = 92.1 329 | 51 out[835].Value = 93.2 330 | 52 return out 331 | 53 } 332 | 54 333 | 55 // Timestamp converts time.Time to protobuf *Timestamp 334 | 56 func Timestamp(t time.Time) *pbtime.Timestamp { 335 | 57 return &pbtime.Timestamp { 336 | 58 Seconds: t.Unix(), 337 | 59 Nanos: int32(t.Nanosecond()), 338 | 60 } 339 | 61 } 340 | ``` 341 | 342 | Listing 11 shows the code from `client.go`. The code fills an `OutliersRequest` value on line 23 with some dummy data (generated by the `dummyData` function on line 34) and then on line 27 calls the Python service. The call to the Python service returns an `OutliersResponse` value. 343 | 344 | Let's break down the code a little more: 345 | 346 | * On line 16, we connect to the Python server, using the `WithInsecure` option since the Python server we wrote doesn’t support HTTPS. 347 | * On line 22, we create a new `OutliersClient` with the connection we made on line 16. 348 | * On line 23, we create the gPRC request. 349 | * On line 27, we perform the actual gRPC call. Every gRPC call has a `context.Context` as the first parameter, allowing you to control timeouts and cancellations. 350 | * gRPC has it’s own implementation of a `Timestamp` struct. On line 56, we have a utility function to convert from Go’s `time.Time` value to gRPC `Timestamp` value. 351 | 352 | **Listing 12** 353 | ``` 354 | $ go run client.go 355 | 356 | OUTPUT: 357 | 2020/05/23 14:07:18 outliers at: [7 113 835] 358 | ``` 359 | 360 | Listing 12 shows you how to run the Go client. This assumes the Python server is running on the same machine. 361 | 362 | ### Conclusion 363 | 364 | gRPC makes it easy and safe to pass messages from one service to another. You can maintain one place where all data types and methods are defined, and there is great tooling and best practices for the gRPC framework. 365 | 366 | The whole code: `outliers.proto`, `py/server.py` and `client.go` is less than a 100 lines. You can view the project code [here](https://github.com/ardanlabs/python-go/tree/master/grpc). 367 | 368 | There is much more to gRPC like timeout, load balancing, TLS, and streaming. I highly recommend going over the [official site](https://grpc.io/) read the documentation and play with the provided examples. 369 | 370 | In the next post in this series, we’ll flip the roles and have Python call a Go service. 371 | -------------------------------------------------------------------------------- /grpc/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "math/rand" 7 | "time" 8 | 9 | "google.golang.org/grpc" 10 | pbtime "google.golang.org/protobuf/types/known/timestamppb" 11 | 12 | "github.com/ardanlabs/python-go/grpc/pb" 13 | ) 14 | 15 | func main() { 16 | addr := "localhost:9999" 17 | conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock()) 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | defer conn.Close() 22 | 23 | client := pb.NewOutliersClient(conn) 24 | req := &pb.OutliersRequest{ 25 | Metrics: dummyData(), 26 | } 27 | 28 | resp, err := client.Detect(context.Background(), req) 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | log.Printf("outliers at: %v", resp.Indices) 33 | } 34 | 35 | func dummyData() []*pb.Metric { 36 | const size = 1000 37 | out := make([]*pb.Metric, size) 38 | t := time.Date(2020, 5, 22, 14, 13, 11, 0, time.UTC) 39 | for i := 0; i < size; i++ { 40 | m := pb.Metric{ 41 | Time: Timestamp(t), 42 | Name: "CPU", 43 | // normally we're below 40% CPU utilization 44 | Value: rand.Float64() * 40, 45 | } 46 | out[i] = &m 47 | t.Add(time.Second) 48 | } 49 | // Create some outliers 50 | out[7].Value = 97.3 51 | out[113].Value = 92.1 52 | out[835].Value = 93.2 53 | return out 54 | } 55 | 56 | // Timestamp converts time.Time to protobuf *Timestamp 57 | func Timestamp(t time.Time) *pbtime.Timestamp { 58 | return &pbtime.Timestamp{ 59 | Seconds: t.Unix(), 60 | Nanos: int32(t.Nanosecond()), 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /grpc/client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | "google.golang.org/grpc" 9 | 10 | "github.com/ardanlabs/python-go/grpc/pb" 11 | ) 12 | 13 | func BenchmarkClient(b *testing.B) { 14 | require := require.New(b) 15 | 16 | addr := "localhost:9999" 17 | conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock()) 18 | require.NoError(err, "connect") 19 | defer conn.Close() 20 | 21 | client := pb.NewOutliersClient(conn) 22 | req := &pb.OutliersRequest{ 23 | Metrics: dummyData(), 24 | } 25 | 26 | b.ResetTimer() 27 | for i := 0; i < b.N; i++ { 28 | _, err := client.Detect(context.Background(), req) 29 | require.NoError(err, "detect") 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /grpc/gen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:generate mkdir -p pb 4 | //go:generate protoc --go_out=plugins=grpc:pb --go_opt=paths=source_relative outliers.proto 5 | -------------------------------------------------------------------------------- /grpc/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ardanlabs/python-go/grpc 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/golang/protobuf v1.4.2 7 | github.com/stretchr/testify v1.6.1 8 | google.golang.org/grpc v1.29.1 9 | google.golang.org/protobuf v1.24.0 10 | ) 11 | -------------------------------------------------------------------------------- /grpc/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 6 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 9 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 10 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 11 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 12 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 13 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 14 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 16 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 17 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 18 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 19 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 20 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 21 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 22 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 23 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 24 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 25 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 26 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 27 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 28 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 29 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 30 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 31 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 32 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 33 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 34 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 35 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 36 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 37 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 38 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 39 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 40 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 41 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 42 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 43 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 44 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 45 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= 46 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 47 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 48 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 49 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 50 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 51 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 52 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 53 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 54 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 55 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 56 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 57 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 58 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 59 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 60 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 61 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 62 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 63 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 64 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 65 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 66 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 67 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= 68 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 69 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 70 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 71 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 72 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 73 | google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= 74 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 75 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 76 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 77 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 78 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 79 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 80 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 81 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 82 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 83 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 84 | google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= 85 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 86 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 87 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 88 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 89 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 90 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 91 | -------------------------------------------------------------------------------- /grpc/outliers.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | import "google/protobuf/timestamp.proto"; 3 | package pb; 4 | 5 | option go_package = "github.com/ardanlabs/python-go/grpc/pb"; 6 | 7 | message Metric { 8 | google.protobuf.Timestamp time = 1; 9 | string name = 2; 10 | double value = 3; 11 | } 12 | 13 | message OutliersRequest { 14 | repeated Metric metrics = 1; 15 | } 16 | 17 | message OutliersResponse { 18 | repeated int32 indices = 1; 19 | } 20 | 21 | service Outliers { 22 | rpc Detect(OutliersRequest) returns (OutliersResponse) {} 23 | } 24 | -------------------------------------------------------------------------------- /grpc/pb/outliers.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.23.0 4 | // protoc v3.11.4 5 | // source: outliers.proto 6 | 7 | package pb 8 | 9 | import ( 10 | context "context" 11 | proto "github.com/golang/protobuf/proto" 12 | timestamp "github.com/golang/protobuf/ptypes/timestamp" 13 | grpc "google.golang.org/grpc" 14 | codes "google.golang.org/grpc/codes" 15 | status "google.golang.org/grpc/status" 16 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 17 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 18 | reflect "reflect" 19 | sync "sync" 20 | ) 21 | 22 | const ( 23 | // Verify that this generated code is sufficiently up-to-date. 24 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 25 | // Verify that runtime/protoimpl is sufficiently up-to-date. 26 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 27 | ) 28 | 29 | // This is a compile-time assertion that a sufficiently up-to-date version 30 | // of the legacy proto package is being used. 31 | const _ = proto.ProtoPackageIsVersion4 32 | 33 | type Metric struct { 34 | state protoimpl.MessageState 35 | sizeCache protoimpl.SizeCache 36 | unknownFields protoimpl.UnknownFields 37 | 38 | Time *timestamp.Timestamp `protobuf:"bytes,1,opt,name=time,proto3" json:"time,omitempty"` 39 | Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` 40 | Value float64 `protobuf:"fixed64,3,opt,name=value,proto3" json:"value,omitempty"` 41 | } 42 | 43 | func (x *Metric) Reset() { 44 | *x = Metric{} 45 | if protoimpl.UnsafeEnabled { 46 | mi := &file_outliers_proto_msgTypes[0] 47 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 48 | ms.StoreMessageInfo(mi) 49 | } 50 | } 51 | 52 | func (x *Metric) String() string { 53 | return protoimpl.X.MessageStringOf(x) 54 | } 55 | 56 | func (*Metric) ProtoMessage() {} 57 | 58 | func (x *Metric) ProtoReflect() protoreflect.Message { 59 | mi := &file_outliers_proto_msgTypes[0] 60 | if protoimpl.UnsafeEnabled && x != nil { 61 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 62 | if ms.LoadMessageInfo() == nil { 63 | ms.StoreMessageInfo(mi) 64 | } 65 | return ms 66 | } 67 | return mi.MessageOf(x) 68 | } 69 | 70 | // Deprecated: Use Metric.ProtoReflect.Descriptor instead. 71 | func (*Metric) Descriptor() ([]byte, []int) { 72 | return file_outliers_proto_rawDescGZIP(), []int{0} 73 | } 74 | 75 | func (x *Metric) GetTime() *timestamp.Timestamp { 76 | if x != nil { 77 | return x.Time 78 | } 79 | return nil 80 | } 81 | 82 | func (x *Metric) GetName() string { 83 | if x != nil { 84 | return x.Name 85 | } 86 | return "" 87 | } 88 | 89 | func (x *Metric) GetValue() float64 { 90 | if x != nil { 91 | return x.Value 92 | } 93 | return 0 94 | } 95 | 96 | type OutliersRequest struct { 97 | state protoimpl.MessageState 98 | sizeCache protoimpl.SizeCache 99 | unknownFields protoimpl.UnknownFields 100 | 101 | Metrics []*Metric `protobuf:"bytes,1,rep,name=metrics,proto3" json:"metrics,omitempty"` 102 | } 103 | 104 | func (x *OutliersRequest) Reset() { 105 | *x = OutliersRequest{} 106 | if protoimpl.UnsafeEnabled { 107 | mi := &file_outliers_proto_msgTypes[1] 108 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 109 | ms.StoreMessageInfo(mi) 110 | } 111 | } 112 | 113 | func (x *OutliersRequest) String() string { 114 | return protoimpl.X.MessageStringOf(x) 115 | } 116 | 117 | func (*OutliersRequest) ProtoMessage() {} 118 | 119 | func (x *OutliersRequest) ProtoReflect() protoreflect.Message { 120 | mi := &file_outliers_proto_msgTypes[1] 121 | if protoimpl.UnsafeEnabled && x != nil { 122 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 123 | if ms.LoadMessageInfo() == nil { 124 | ms.StoreMessageInfo(mi) 125 | } 126 | return ms 127 | } 128 | return mi.MessageOf(x) 129 | } 130 | 131 | // Deprecated: Use OutliersRequest.ProtoReflect.Descriptor instead. 132 | func (*OutliersRequest) Descriptor() ([]byte, []int) { 133 | return file_outliers_proto_rawDescGZIP(), []int{1} 134 | } 135 | 136 | func (x *OutliersRequest) GetMetrics() []*Metric { 137 | if x != nil { 138 | return x.Metrics 139 | } 140 | return nil 141 | } 142 | 143 | type OutliersResponse struct { 144 | state protoimpl.MessageState 145 | sizeCache protoimpl.SizeCache 146 | unknownFields protoimpl.UnknownFields 147 | 148 | Indices []int32 `protobuf:"varint,1,rep,packed,name=indices,proto3" json:"indices,omitempty"` 149 | } 150 | 151 | func (x *OutliersResponse) Reset() { 152 | *x = OutliersResponse{} 153 | if protoimpl.UnsafeEnabled { 154 | mi := &file_outliers_proto_msgTypes[2] 155 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 156 | ms.StoreMessageInfo(mi) 157 | } 158 | } 159 | 160 | func (x *OutliersResponse) String() string { 161 | return protoimpl.X.MessageStringOf(x) 162 | } 163 | 164 | func (*OutliersResponse) ProtoMessage() {} 165 | 166 | func (x *OutliersResponse) ProtoReflect() protoreflect.Message { 167 | mi := &file_outliers_proto_msgTypes[2] 168 | if protoimpl.UnsafeEnabled && x != nil { 169 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 170 | if ms.LoadMessageInfo() == nil { 171 | ms.StoreMessageInfo(mi) 172 | } 173 | return ms 174 | } 175 | return mi.MessageOf(x) 176 | } 177 | 178 | // Deprecated: Use OutliersResponse.ProtoReflect.Descriptor instead. 179 | func (*OutliersResponse) Descriptor() ([]byte, []int) { 180 | return file_outliers_proto_rawDescGZIP(), []int{2} 181 | } 182 | 183 | func (x *OutliersResponse) GetIndices() []int32 { 184 | if x != nil { 185 | return x.Indices 186 | } 187 | return nil 188 | } 189 | 190 | var File_outliers_proto protoreflect.FileDescriptor 191 | 192 | var file_outliers_proto_rawDesc = []byte{ 193 | 0x0a, 0x0e, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 194 | 0x12, 0x02, 0x70, 0x62, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 195 | 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 196 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x62, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 197 | 0x2e, 0x0a, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 198 | 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 199 | 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x12, 200 | 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 201 | 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 202 | 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x37, 0x0a, 0x0f, 0x4f, 0x75, 0x74, 203 | 0x6c, 0x69, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x07, 204 | 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 205 | 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 206 | 0x63, 0x73, 0x22, 0x2c, 0x0a, 0x10, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x73, 0x52, 0x65, 207 | 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 208 | 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x07, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 209 | 0x32, 0x41, 0x0a, 0x08, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x06, 210 | 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x12, 0x13, 0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x6c, 211 | 0x69, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x62, 212 | 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 213 | 0x65, 0x22, 0x00, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 214 | 0x6d, 0x2f, 0x61, 0x72, 0x64, 0x61, 0x6e, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x70, 0x79, 0x74, 0x68, 215 | 0x6f, 0x6e, 0x2d, 0x67, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 216 | 0x72, 0x6f, 0x74, 0x6f, 0x33, 217 | } 218 | 219 | var ( 220 | file_outliers_proto_rawDescOnce sync.Once 221 | file_outliers_proto_rawDescData = file_outliers_proto_rawDesc 222 | ) 223 | 224 | func file_outliers_proto_rawDescGZIP() []byte { 225 | file_outliers_proto_rawDescOnce.Do(func() { 226 | file_outliers_proto_rawDescData = protoimpl.X.CompressGZIP(file_outliers_proto_rawDescData) 227 | }) 228 | return file_outliers_proto_rawDescData 229 | } 230 | 231 | var file_outliers_proto_msgTypes = make([]protoimpl.MessageInfo, 3) 232 | var file_outliers_proto_goTypes = []interface{}{ 233 | (*Metric)(nil), // 0: pb.Metric 234 | (*OutliersRequest)(nil), // 1: pb.OutliersRequest 235 | (*OutliersResponse)(nil), // 2: pb.OutliersResponse 236 | (*timestamp.Timestamp)(nil), // 3: google.protobuf.Timestamp 237 | } 238 | var file_outliers_proto_depIdxs = []int32{ 239 | 3, // 0: pb.Metric.time:type_name -> google.protobuf.Timestamp 240 | 0, // 1: pb.OutliersRequest.metrics:type_name -> pb.Metric 241 | 1, // 2: pb.Outliers.Detect:input_type -> pb.OutliersRequest 242 | 2, // 3: pb.Outliers.Detect:output_type -> pb.OutliersResponse 243 | 3, // [3:4] is the sub-list for method output_type 244 | 2, // [2:3] is the sub-list for method input_type 245 | 2, // [2:2] is the sub-list for extension type_name 246 | 2, // [2:2] is the sub-list for extension extendee 247 | 0, // [0:2] is the sub-list for field type_name 248 | } 249 | 250 | func init() { file_outliers_proto_init() } 251 | func file_outliers_proto_init() { 252 | if File_outliers_proto != nil { 253 | return 254 | } 255 | if !protoimpl.UnsafeEnabled { 256 | file_outliers_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 257 | switch v := v.(*Metric); i { 258 | case 0: 259 | return &v.state 260 | case 1: 261 | return &v.sizeCache 262 | case 2: 263 | return &v.unknownFields 264 | default: 265 | return nil 266 | } 267 | } 268 | file_outliers_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 269 | switch v := v.(*OutliersRequest); i { 270 | case 0: 271 | return &v.state 272 | case 1: 273 | return &v.sizeCache 274 | case 2: 275 | return &v.unknownFields 276 | default: 277 | return nil 278 | } 279 | } 280 | file_outliers_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 281 | switch v := v.(*OutliersResponse); i { 282 | case 0: 283 | return &v.state 284 | case 1: 285 | return &v.sizeCache 286 | case 2: 287 | return &v.unknownFields 288 | default: 289 | return nil 290 | } 291 | } 292 | } 293 | type x struct{} 294 | out := protoimpl.TypeBuilder{ 295 | File: protoimpl.DescBuilder{ 296 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 297 | RawDescriptor: file_outliers_proto_rawDesc, 298 | NumEnums: 0, 299 | NumMessages: 3, 300 | NumExtensions: 0, 301 | NumServices: 1, 302 | }, 303 | GoTypes: file_outliers_proto_goTypes, 304 | DependencyIndexes: file_outliers_proto_depIdxs, 305 | MessageInfos: file_outliers_proto_msgTypes, 306 | }.Build() 307 | File_outliers_proto = out.File 308 | file_outliers_proto_rawDesc = nil 309 | file_outliers_proto_goTypes = nil 310 | file_outliers_proto_depIdxs = nil 311 | } 312 | 313 | // Reference imports to suppress errors if they are not otherwise used. 314 | var _ context.Context 315 | var _ grpc.ClientConnInterface 316 | 317 | // This is a compile-time assertion to ensure that this generated file 318 | // is compatible with the grpc package it is being compiled against. 319 | const _ = grpc.SupportPackageIsVersion6 320 | 321 | // OutliersClient is the client API for Outliers service. 322 | // 323 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. 324 | type OutliersClient interface { 325 | Detect(ctx context.Context, in *OutliersRequest, opts ...grpc.CallOption) (*OutliersResponse, error) 326 | } 327 | 328 | type outliersClient struct { 329 | cc grpc.ClientConnInterface 330 | } 331 | 332 | func NewOutliersClient(cc grpc.ClientConnInterface) OutliersClient { 333 | return &outliersClient{cc} 334 | } 335 | 336 | func (c *outliersClient) Detect(ctx context.Context, in *OutliersRequest, opts ...grpc.CallOption) (*OutliersResponse, error) { 337 | out := new(OutliersResponse) 338 | err := c.cc.Invoke(ctx, "/pb.Outliers/Detect", in, out, opts...) 339 | if err != nil { 340 | return nil, err 341 | } 342 | return out, nil 343 | } 344 | 345 | // OutliersServer is the server API for Outliers service. 346 | type OutliersServer interface { 347 | Detect(context.Context, *OutliersRequest) (*OutliersResponse, error) 348 | } 349 | 350 | // UnimplementedOutliersServer can be embedded to have forward compatible implementations. 351 | type UnimplementedOutliersServer struct { 352 | } 353 | 354 | func (*UnimplementedOutliersServer) Detect(context.Context, *OutliersRequest) (*OutliersResponse, error) { 355 | return nil, status.Errorf(codes.Unimplemented, "method Detect not implemented") 356 | } 357 | 358 | func RegisterOutliersServer(s *grpc.Server, srv OutliersServer) { 359 | s.RegisterService(&_Outliers_serviceDesc, srv) 360 | } 361 | 362 | func _Outliers_Detect_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 363 | in := new(OutliersRequest) 364 | if err := dec(in); err != nil { 365 | return nil, err 366 | } 367 | if interceptor == nil { 368 | return srv.(OutliersServer).Detect(ctx, in) 369 | } 370 | info := &grpc.UnaryServerInfo{ 371 | Server: srv, 372 | FullMethod: "/pb.Outliers/Detect", 373 | } 374 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 375 | return srv.(OutliersServer).Detect(ctx, req.(*OutliersRequest)) 376 | } 377 | return interceptor(ctx, in, info, handler) 378 | } 379 | 380 | var _Outliers_serviceDesc = grpc.ServiceDesc{ 381 | ServiceName: "pb.Outliers", 382 | HandlerType: (*OutliersServer)(nil), 383 | Methods: []grpc.MethodDesc{ 384 | { 385 | MethodName: "Detect", 386 | Handler: _Outliers_Detect_Handler, 387 | }, 388 | }, 389 | Streams: []grpc.StreamDesc{}, 390 | Metadata: "outliers.proto", 391 | } 392 | -------------------------------------------------------------------------------- /grpc/py/Makefile: -------------------------------------------------------------------------------- 1 | proto: 2 | python -m grpc_tools.protoc \ 3 | -I.. --python_out=. --grpc_python_out=. \ 4 | ../outliers.proto 5 | -------------------------------------------------------------------------------- /grpc/py/outliers_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: outliers.proto 4 | 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import message as _message 7 | from google.protobuf import reflection as _reflection 8 | from google.protobuf import symbol_database as _symbol_database 9 | # @@protoc_insertion_point(imports) 10 | 11 | _sym_db = _symbol_database.Default() 12 | 13 | 14 | from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 15 | 16 | 17 | DESCRIPTOR = _descriptor.FileDescriptor( 18 | name='outliers.proto', 19 | package='pb', 20 | syntax='proto3', 21 | serialized_options=b'Z&github.com/ardanlabs/python-go/grpc/pb', 22 | serialized_pb=b'\n\x0eoutliers.proto\x12\x02pb\x1a\x1fgoogle/protobuf/timestamp.proto\"O\n\x06Metric\x12(\n\x04time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\x01\".\n\x0fOutliersRequest\x12\x1b\n\x07metrics\x18\x01 \x03(\x0b\x32\n.pb.Metric\"#\n\x10OutliersResponse\x12\x0f\n\x07indices\x18\x01 \x03(\x05\x32\x41\n\x08Outliers\x12\x35\n\x06\x44\x65tect\x12\x13.pb.OutliersRequest\x1a\x14.pb.OutliersResponse\"\x00\x42(Z&github.com/ardanlabs/python-go/grpc/pbb\x06proto3' 23 | , 24 | dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) 25 | 26 | 27 | 28 | 29 | _METRIC = _descriptor.Descriptor( 30 | name='Metric', 31 | full_name='pb.Metric', 32 | filename=None, 33 | file=DESCRIPTOR, 34 | containing_type=None, 35 | fields=[ 36 | _descriptor.FieldDescriptor( 37 | name='time', full_name='pb.Metric.time', index=0, 38 | number=1, type=11, cpp_type=10, label=1, 39 | has_default_value=False, default_value=None, 40 | message_type=None, enum_type=None, containing_type=None, 41 | is_extension=False, extension_scope=None, 42 | serialized_options=None, file=DESCRIPTOR), 43 | _descriptor.FieldDescriptor( 44 | name='name', full_name='pb.Metric.name', index=1, 45 | number=2, type=9, cpp_type=9, label=1, 46 | has_default_value=False, default_value=b"".decode('utf-8'), 47 | message_type=None, enum_type=None, containing_type=None, 48 | is_extension=False, extension_scope=None, 49 | serialized_options=None, file=DESCRIPTOR), 50 | _descriptor.FieldDescriptor( 51 | name='value', full_name='pb.Metric.value', index=2, 52 | number=3, type=1, cpp_type=5, label=1, 53 | has_default_value=False, default_value=float(0), 54 | message_type=None, enum_type=None, containing_type=None, 55 | is_extension=False, extension_scope=None, 56 | serialized_options=None, file=DESCRIPTOR), 57 | ], 58 | extensions=[ 59 | ], 60 | nested_types=[], 61 | enum_types=[ 62 | ], 63 | serialized_options=None, 64 | is_extendable=False, 65 | syntax='proto3', 66 | extension_ranges=[], 67 | oneofs=[ 68 | ], 69 | serialized_start=55, 70 | serialized_end=134, 71 | ) 72 | 73 | 74 | _OUTLIERSREQUEST = _descriptor.Descriptor( 75 | name='OutliersRequest', 76 | full_name='pb.OutliersRequest', 77 | filename=None, 78 | file=DESCRIPTOR, 79 | containing_type=None, 80 | fields=[ 81 | _descriptor.FieldDescriptor( 82 | name='metrics', full_name='pb.OutliersRequest.metrics', index=0, 83 | number=1, type=11, cpp_type=10, label=3, 84 | has_default_value=False, default_value=[], 85 | message_type=None, enum_type=None, containing_type=None, 86 | is_extension=False, extension_scope=None, 87 | serialized_options=None, file=DESCRIPTOR), 88 | ], 89 | extensions=[ 90 | ], 91 | nested_types=[], 92 | enum_types=[ 93 | ], 94 | serialized_options=None, 95 | is_extendable=False, 96 | syntax='proto3', 97 | extension_ranges=[], 98 | oneofs=[ 99 | ], 100 | serialized_start=136, 101 | serialized_end=182, 102 | ) 103 | 104 | 105 | _OUTLIERSRESPONSE = _descriptor.Descriptor( 106 | name='OutliersResponse', 107 | full_name='pb.OutliersResponse', 108 | filename=None, 109 | file=DESCRIPTOR, 110 | containing_type=None, 111 | fields=[ 112 | _descriptor.FieldDescriptor( 113 | name='indices', full_name='pb.OutliersResponse.indices', index=0, 114 | number=1, type=5, cpp_type=1, label=3, 115 | has_default_value=False, default_value=[], 116 | message_type=None, enum_type=None, containing_type=None, 117 | is_extension=False, extension_scope=None, 118 | serialized_options=None, file=DESCRIPTOR), 119 | ], 120 | extensions=[ 121 | ], 122 | nested_types=[], 123 | enum_types=[ 124 | ], 125 | serialized_options=None, 126 | is_extendable=False, 127 | syntax='proto3', 128 | extension_ranges=[], 129 | oneofs=[ 130 | ], 131 | serialized_start=184, 132 | serialized_end=219, 133 | ) 134 | 135 | _METRIC.fields_by_name['time'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP 136 | _OUTLIERSREQUEST.fields_by_name['metrics'].message_type = _METRIC 137 | DESCRIPTOR.message_types_by_name['Metric'] = _METRIC 138 | DESCRIPTOR.message_types_by_name['OutliersRequest'] = _OUTLIERSREQUEST 139 | DESCRIPTOR.message_types_by_name['OutliersResponse'] = _OUTLIERSRESPONSE 140 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 141 | 142 | Metric = _reflection.GeneratedProtocolMessageType('Metric', (_message.Message,), { 143 | 'DESCRIPTOR' : _METRIC, 144 | '__module__' : 'outliers_pb2' 145 | # @@protoc_insertion_point(class_scope:pb.Metric) 146 | }) 147 | _sym_db.RegisterMessage(Metric) 148 | 149 | OutliersRequest = _reflection.GeneratedProtocolMessageType('OutliersRequest', (_message.Message,), { 150 | 'DESCRIPTOR' : _OUTLIERSREQUEST, 151 | '__module__' : 'outliers_pb2' 152 | # @@protoc_insertion_point(class_scope:pb.OutliersRequest) 153 | }) 154 | _sym_db.RegisterMessage(OutliersRequest) 155 | 156 | OutliersResponse = _reflection.GeneratedProtocolMessageType('OutliersResponse', (_message.Message,), { 157 | 'DESCRIPTOR' : _OUTLIERSRESPONSE, 158 | '__module__' : 'outliers_pb2' 159 | # @@protoc_insertion_point(class_scope:pb.OutliersResponse) 160 | }) 161 | _sym_db.RegisterMessage(OutliersResponse) 162 | 163 | 164 | DESCRIPTOR._options = None 165 | 166 | _OUTLIERS = _descriptor.ServiceDescriptor( 167 | name='Outliers', 168 | full_name='pb.Outliers', 169 | file=DESCRIPTOR, 170 | index=0, 171 | serialized_options=None, 172 | serialized_start=221, 173 | serialized_end=286, 174 | methods=[ 175 | _descriptor.MethodDescriptor( 176 | name='Detect', 177 | full_name='pb.Outliers.Detect', 178 | index=0, 179 | containing_service=None, 180 | input_type=_OUTLIERSREQUEST, 181 | output_type=_OUTLIERSRESPONSE, 182 | serialized_options=None, 183 | ), 184 | ]) 185 | _sym_db.RegisterServiceDescriptor(_OUTLIERS) 186 | 187 | DESCRIPTOR.services_by_name['Outliers'] = _OUTLIERS 188 | 189 | # @@protoc_insertion_point(module_scope) 190 | -------------------------------------------------------------------------------- /grpc/py/outliers_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | import outliers_pb2 as outliers__pb2 5 | 6 | 7 | class OutliersStub(object): 8 | """Missing associated documentation comment in .proto file""" 9 | 10 | def __init__(self, channel): 11 | """Constructor. 12 | 13 | Args: 14 | channel: A grpc.Channel. 15 | """ 16 | self.Detect = channel.unary_unary( 17 | '/pb.Outliers/Detect', 18 | request_serializer=outliers__pb2.OutliersRequest.SerializeToString, 19 | response_deserializer=outliers__pb2.OutliersResponse.FromString, 20 | ) 21 | 22 | 23 | class OutliersServicer(object): 24 | """Missing associated documentation comment in .proto file""" 25 | 26 | def Detect(self, request, context): 27 | """Missing associated documentation comment in .proto file""" 28 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 29 | context.set_details('Method not implemented!') 30 | raise NotImplementedError('Method not implemented!') 31 | 32 | 33 | def add_OutliersServicer_to_server(servicer, server): 34 | rpc_method_handlers = { 35 | 'Detect': grpc.unary_unary_rpc_method_handler( 36 | servicer.Detect, 37 | request_deserializer=outliers__pb2.OutliersRequest.FromString, 38 | response_serializer=outliers__pb2.OutliersResponse.SerializeToString, 39 | ), 40 | } 41 | generic_handler = grpc.method_handlers_generic_handler( 42 | 'pb.Outliers', rpc_method_handlers) 43 | server.add_generic_rpc_handlers((generic_handler,)) 44 | 45 | 46 | # This class is part of an EXPERIMENTAL API. 47 | class Outliers(object): 48 | """Missing associated documentation comment in .proto file""" 49 | 50 | @staticmethod 51 | def Detect(request, 52 | target, 53 | options=(), 54 | channel_credentials=None, 55 | call_credentials=None, 56 | compression=None, 57 | wait_for_ready=None, 58 | timeout=None, 59 | metadata=None): 60 | return grpc.experimental.unary_unary(request, target, '/pb.Outliers/Detect', 61 | outliers__pb2.OutliersRequest.SerializeToString, 62 | outliers__pb2.OutliersResponse.FromString, 63 | options, channel_credentials, 64 | call_credentials, compression, wait_for_ready, timeout, metadata) 65 | -------------------------------------------------------------------------------- /grpc/py/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio-tools==1.29.0 2 | numpy==1.18.4 3 | -------------------------------------------------------------------------------- /grpc/py/server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from concurrent.futures import ThreadPoolExecutor 3 | 4 | import grpc 5 | import numpy as np 6 | 7 | from outliers_pb2 import OutliersResponse 8 | from outliers_pb2_grpc import OutliersServicer, add_OutliersServicer_to_server 9 | 10 | 11 | def find_outliers(data: np.ndarray): 12 | """Return indices where values more than 2 standard deviations from mean""" 13 | out = np.where(np.abs(data - data.mean()) > 2 * data.std()) 14 | # np.where returns a tuple for each dimension, we want the 1st element 15 | return out[0] 16 | 17 | 18 | class OutliersServer(OutliersServicer): 19 | def Detect(self, request, context): 20 | logging.info('detect request size: %d', len(request.metrics)) 21 | # Convert metrics to numpy array of values only 22 | data = np.fromiter((m.value for m in request.metrics), dtype='float64') 23 | indices = find_outliers(data) 24 | logging.info('found %d outliers', len(indices)) 25 | resp = OutliersResponse(indices=indices) 26 | return resp 27 | 28 | 29 | if __name__ == '__main__': 30 | logging.basicConfig( 31 | level=logging.INFO, 32 | format='%(asctime)s - %(levelname)s - %(message)s', 33 | ) 34 | server = grpc.server(ThreadPoolExecutor()) 35 | add_OutliersServicer_to_server(OutliersServer(), server) 36 | port = 9999 37 | server.add_insecure_port(f'[::]:{port}') 38 | server.start() 39 | logging.info('server ready on port %r', port) 40 | server.wait_for_termination() 41 | -------------------------------------------------------------------------------- /lisp/README-1.md: -------------------------------------------------------------------------------- 1 | # Writing a Lisp Interpreter in Python & Go Part I 2 | 3 | ### Introduction 4 | 5 | There's an old joke linguistics professors tell (just setting your expectations here): 6 | 7 | > During the cold war, the US developed a system to translate from Russian to English and back. 8 | > When they finished writing the system, they decided to test it by giving it a sentence in English, translate it to Russian and back. 9 | > They gave it the sentence "The spirit is willing but the flesh is weak." And got back "The vodka is good but the meat is rotten." 10 | 11 | The point of this joke is to show that different languages have a different way of saying things. This is true for programming languages as well. 12 | 13 | Working with Python for 25 years and with Go for the past 10, I consider myself fluent in both languages. In Python we say you write "pythonic" code when you grok the language and in Go we say you write "idiomatic Go". 14 | 15 | I'd like to explore and compare both languages. I believe that programming languages are tools and that "If all you have is a hammer, every program looks like a nail." 16 | 17 | _Note: Except for C++ where every problem looks like your thumb :)_ 18 | 19 | We're going to compare both languages by ... implementing an interpreter to another language. During the development of the interpreter we'll discuss syntax, types, scope and many other aspects that related to programming language design and implementation. 20 | 21 | ### The Target Language 22 | 23 | I'd like to focus the discussion on language design and semantics, so I've picked a very small language that's easy to parse: Scheme. Scheme is a dialect of list, and even though it even simpler than Go - it has everything you need to write code. 24 | 25 | Here's an example: 26 | 27 | **Listing 1: Example Scheme Program - collatz.scm** 28 | 29 | ``` 30 | 01 (define collatz-step 31 | 02 (lambda (n) 32 | 03 (if (eq? (% n 2) 0) 33 | 04 (/ n 2) 34 | 05 (+ (* n 3) 1)))) 35 | 06 36 | 07 37 | 08 (println (collatz-step 7)) ; 22 38 | ``` 39 | 40 | Listing one shows the Scheme code to calculate a step in the [Collatz conjucture](https://en.wikipedia.org/wiki/Collatz_conjecture). On line 1 we use the `define` keyword to assign a value to the name `collatz`. On line 2 we define the value which is a `lambda` expression - an anonymous function. On line 3 we have an `if` statement and on line 4 we have the true branch of the `if` and on line 5 we have the false branch of the `if` statement. On line 08 we use the `println` function to print the result of calling `collatz` on the number 7 - which outputs `22'. 41 | 42 | Scheme programs are built like an expression tree and uses prefix operator, meaning we write `(+ 1 2)` vs `1 + 2`. There's also no `return`, the return value is the last expression in the function. 43 | 44 | ![](https://imgs.xkcd.com/comics/lisp_cycles.png) 45 | 46 | 47 | 48 | ### Python & Go Implementation 49 | 50 | https://imgs.xkcd.com/comics/lisp_cycles.png 51 | 52 | ``` 53 | 01 package main 54 | 02 55 | 03 import ( 56 | 04 "fmt" 57 | 05 ) 58 | 06 59 | 07 func collatzStep(n int) int { 60 | 08 if n%2 == 0 { 61 | 09 return n / 2 62 | 10 } 63 | 11 return n*3 + 1 64 | 12 } 65 | 13 66 | 14 func main() { 67 | 15 fmt.Println(collatzStep(7)) // 22 68 | 16 } 69 | ``` 70 | 71 | Listing 2 shows the implementation of `collatz` in Go. 72 | 73 | **Listing 3: colltaz in Python - collatz.py** 74 | 75 | ``` 76 | 01 def collatz_step(n): 77 | 02 if n % 2 == 0: 78 | 03 return n // 2 79 | 04 return n * 3 + 1 80 | 05 81 | 06 82 | 07 print(collatz(7)) # 22 83 | ``` 84 | 85 | Listing 3 shows the implementation of `collatz` in Python. 86 | 87 | ### Comparison 88 | 89 | Let's go over some similarities and differences between the languages. 90 | 91 | #### Source Code 92 | 93 | All three languages use files to store source code. This is an approach that most programming languages take, but isn't the only one. Who says that code structure should be tied to the file system? There are visual languages like [MIT Scratch](https://scratch.mit.edu/) and there are languages like [Smalltalk](https://en.wikipedia.org/wiki/Smalltalk) where the code is stored ... somewhere in the image. 94 | 95 | Another design decision is what can be an identifier (name) in the language. Go & Python both take a similar approach and Scheme takes a wider stance. `collatz-step` is not a valid identifier in Python or Go. Both Python & Go allow Unicode identifiers (e.g. `π = 3.14`), our Scheme implementation will take the same approach. 96 | 97 | #### Syntax 98 | 99 | Go takes after C, with it's infix notation (`1 + 2`) and using curly braces for scope. Python also uses infix notation but uses indendentation and `:` for scope. Scheme, uses prefix notation `(+ 1 2)` and uses braces for lists. 100 | 101 | #### Execution 102 | 103 | Python & Scheme are interpreted, the interpreter will build an abstract syntax tree (AST) and then evaluate it. Python compiles the AST to byte code and runs this byte code in a virtual machine. Go compiles it's programs to AST and then to machine code - there's no runtime involved. 104 | 105 | _Note: Go does compile parts of its runtime, like the garbage collector, into the executable. But it doesn't need a VM to interpret instructions to execute._ 106 | 107 | ### Conclusion 108 | 109 | By looking at the differences between programming languages, you can understand each language better and clearly see the design trade offs every language makes. 110 | 111 | Implementing your own language is a right of passage for developers, and also as Steve Yegge [said](http://steve-yegge.blogspot.com/2007/06/rich-programmer-food.html) 112 | 113 | > If you don't know how compilers work, then you don't know how computers work. 114 | 115 | This series of posts is inspired by Peter Norvig's [(How to Write a (Lisp) Interpreter (in Python))](https://norvig.com/lispy.html). In the next part we'll start implementing our Scheme interpreter both in Go & Python. 116 | 117 | 118 | -------------------------------------------------------------------------------- /lisp/collatz.clj: -------------------------------------------------------------------------------- 1 | (def collatz-step 2 | (fn [n] 3 | (if (= (mod n 2) 0) 4 | (/ n 2) 5 | (+ (* n 3) 1)))) 6 | 7 | (println (collatz-step 7)) ; 22 8 | -------------------------------------------------------------------------------- /lisp/collatz.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func collatzStep(n int) int { 8 | if n%2 == 0 { 9 | return n / 2 10 | } 11 | return n*3 + 1 12 | } 13 | 14 | func main() { 15 | fmt.Println(collatzStep(7)) // 22 16 | } 17 | -------------------------------------------------------------------------------- /lisp/collatz.py: -------------------------------------------------------------------------------- 1 | def collatz_step(n): 2 | if n % 2 == 0: 3 | return n // 2 4 | return n * 3 + 1 5 | 6 | 7 | print(collatz_step(7)) # 22 8 | -------------------------------------------------------------------------------- /py-bindings/outliers/README.md: -------------------------------------------------------------------------------- 1 | # example: using Python bindings for Go 2 | 3 | for more infos see the blogpost [here](https://poweruser.blog/embedding-python-in-go-338c0399f3d5) 4 | 5 | -------------------------------------------------------------------------------- /py-bindings/outliers/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dp0="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | destdir=$dp0 4 | #mkdir -p "$destdir" 5 | #go get -u github.com/christian-korneck/go-python3 #force update 6 | source "$dp0/set_env.sh" 7 | #echo $PKG_CONFIG_PATH 8 | go build -o "$destdir" 9 | 10 | -------------------------------------------------------------------------------- /py-bindings/outliers/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/christian-korneck/python-go/py-bindings/outliers 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/christian-korneck/go-python3 v0.0.0-20200921060006-847aa92a90a5 7 | github.com/stretchr/testify v1.6.1 // indirect 8 | ) 9 | -------------------------------------------------------------------------------- /py-bindings/outliers/go.sum: -------------------------------------------------------------------------------- 1 | github.com/christian-korneck/go-python3 v0.0.0-20200921060006-847aa92a90a5 h1:j1uhylB7hgzsXM2R4eWllYH+r/858S3VpJc1s90TAew= 2 | github.com/christian-korneck/go-python3 v0.0.0-20200921060006-847aa92a90a5/go.mod h1:P7tyhjAgcLLs7mJgR3TiyoNxEfm9M8Scc184dNLKIQc= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 6 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 7 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 8 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 9 | -------------------------------------------------------------------------------- /py-bindings/outliers/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/christian-korneck/go-python3" 10 | ) 11 | 12 | //detect wraps around the detect() function from the given Python module. Borrows the PyObject reference. 13 | func detect(module *python3.PyObject, data []float64) ([]int, error) { 14 | 15 | pylist := python3.PyList_New(len(data)) //retval: New reference, gets stolen later 16 | for i := 0; i < len(data); i++ { 17 | item := python3.PyFloat_FromDouble(data[i]) //retval: New reference, gets stolen later 18 | ret := python3.PyList_SetItem(pylist, i, item) 19 | if ret != 0 { 20 | if python3.PyErr_Occurred() != nil { 21 | python3.PyErr_Print() 22 | } 23 | item.DecRef() 24 | pylist.DecRef() 25 | return nil, fmt.Errorf("error setting list item") 26 | } 27 | } 28 | 29 | args := python3.PyTuple_New(1) //retval: New reference 30 | if args == nil { 31 | pylist.DecRef() 32 | return nil, fmt.Errorf("error creating args tuple") 33 | } 34 | defer args.DecRef() 35 | ret := python3.PyTuple_SetItem(args, 0, pylist) //steals ref to pylist 36 | pylist = nil 37 | if ret != 0 { 38 | if python3.PyErr_Occurred() != nil { 39 | python3.PyErr_Print() 40 | } 41 | pylist.DecRef() 42 | pylist = nil 43 | return nil, fmt.Errorf("error setting args tuple item") 44 | } 45 | 46 | oDict := python3.PyModule_GetDict(module) //retval: Borrowed 47 | if !(oDict != nil && python3.PyErr_Occurred() == nil) { 48 | python3.PyErr_Print() 49 | return nil, fmt.Errorf("could not get dict for module") 50 | } 51 | detect := python3.PyDict_GetItemString(oDict, "detect") 52 | if !(detect != nil && python3.PyCallable_Check(detect)) { //retval: Borrowed 53 | return nil, fmt.Errorf("could not find function 'detect'") 54 | } 55 | detectdataPy := detect.CallObject(args) 56 | if !(detectdataPy != nil && python3.PyErr_Occurred() == nil) { //retval: New reference 57 | python3.PyErr_Print() 58 | return nil, fmt.Errorf("error calling function detect") 59 | } 60 | defer detectdataPy.DecRef() 61 | outliers, err := goSliceFromPylist(detectdataPy, "int", false) 62 | if err != nil { 63 | return nil, fmt.Errorf("error converting pylist to go list: %s", err) 64 | } 65 | 66 | return outliers.([]int), nil 67 | 68 | } 69 | 70 | //goSliceFromPylist converts a []float64 pylist to a go list. Borrows the PyObject reference. 71 | func goSliceFromPylist(pylist *python3.PyObject, itemtype string, strictfail bool) (interface{}, error) { 72 | 73 | seq := pylist.GetIter() //ret val: New reference 74 | if !(seq != nil && python3.PyErr_Occurred() == nil) { 75 | python3.PyErr_Print() 76 | return nil, fmt.Errorf("error creating iterator for list") 77 | } 78 | defer seq.DecRef() 79 | tNext := seq.GetAttrString("__next__") //ret val: new ref 80 | if !(tNext != nil && python3.PyCallable_Check(tNext)) { 81 | return nil, fmt.Errorf("iterator has no __next__ function") 82 | } 83 | defer tNext.DecRef() 84 | 85 | var golist interface{} 86 | var compare *python3.PyObject 87 | switch itemtype { 88 | case "float64": 89 | golist = []float64{} 90 | compare = python3.PyFloat_FromDouble(0) 91 | case "int": 92 | golist = []int{} 93 | compare = python3.PyLong_FromGoInt(0) 94 | } 95 | if compare == nil { 96 | return nil, fmt.Errorf("error creating compare var") 97 | } 98 | defer compare.DecRef() 99 | 100 | pytype := compare.Type() //ret val: new ref 101 | if pytype == nil && python3.PyErr_Occurred() != nil { 102 | python3.PyErr_Print() 103 | return nil, fmt.Errorf("error getting type of compare var") 104 | } 105 | defer pytype.DecRef() 106 | 107 | errcnt := 0 108 | 109 | pylistLen := pylist.Length() 110 | if pylistLen == -1 { 111 | return nil, fmt.Errorf("error getting list length") 112 | } 113 | 114 | for i := 1; i <= pylistLen; i++ { 115 | item := tNext.CallObject(nil) //ret val: new ref 116 | if item == nil && python3.PyErr_Occurred() != nil { 117 | python3.PyErr_Print() 118 | return nil, fmt.Errorf("error getting next item in sequence") 119 | } 120 | itemType := item.Type() 121 | if itemType == nil && python3.PyErr_Occurred() != nil { 122 | python3.PyErr_Print() 123 | return nil, fmt.Errorf("error getting item type") 124 | } 125 | 126 | defer itemType.DecRef() 127 | 128 | if itemType != pytype { 129 | //item has wrong type, skip it 130 | if item != nil { 131 | item.DecRef() 132 | } 133 | errcnt++ 134 | continue 135 | } 136 | 137 | switch itemtype { 138 | case "float64": 139 | itemGo := python3.PyFloat_AsDouble(item) 140 | if itemGo != -1 && python3.PyErr_Occurred() == nil { 141 | golist = append(golist.([]float64), itemGo) 142 | } else { 143 | if item != nil { 144 | item.DecRef() 145 | } 146 | errcnt++ 147 | } 148 | case "int": 149 | itemGo := python3.PyLong_AsLong(item) 150 | if itemGo != -1 && python3.PyErr_Occurred() == nil { 151 | golist = append(golist.([]int), itemGo) 152 | } else { 153 | if item != nil { 154 | item.DecRef() 155 | } 156 | errcnt++ 157 | } 158 | } 159 | 160 | if item != nil { 161 | item.DecRef() 162 | item = nil 163 | } 164 | } 165 | if errcnt > 0 { 166 | if strictfail { 167 | return nil, fmt.Errorf("could not add %d values (wrong type?)", errcnt) 168 | } 169 | } 170 | 171 | return golist, nil 172 | } 173 | 174 | //genTestdata wraps around the gen_testdata() function from the given Python module. Borrows the PyObject reference. 175 | func genTestdata(module *python3.PyObject) ([]float64, error) { 176 | oDict := python3.PyModule_GetDict(module) //ret val: Borrowed 177 | if !(oDict != nil && python3.PyErr_Occurred() == nil) { 178 | python3.PyErr_Print() 179 | return nil, fmt.Errorf("could not get dict for module") 180 | } 181 | genTestdata := python3.PyDict_GetItemString(oDict, "gen_testdata") //retval: Borrowed 182 | if !(genTestdata != nil && python3.PyCallable_Check(genTestdata)) { 183 | return nil, fmt.Errorf("could not find function 'gen_testdata'") 184 | } 185 | testdataPy := genTestdata.CallObject(nil) //retval: New reference 186 | if !(testdataPy != nil && python3.PyErr_Occurred() == nil) { 187 | python3.PyErr_Print() 188 | return nil, fmt.Errorf("error calling function gen_testdata") 189 | } 190 | defer testdataPy.DecRef() 191 | testdataGo, err := goSliceFromPylist(testdataPy, "float64", false) 192 | if err != nil { 193 | return nil, fmt.Errorf("error converting pylist to go list: %s", err) 194 | } 195 | 196 | return testdataGo.([]float64), nil 197 | } 198 | 199 | func demo(module *python3.PyObject) { 200 | 201 | testdata, err := genTestdata(module) 202 | if err != nil { 203 | log.Fatalf("Error getting testdata: %s", err) 204 | } 205 | 206 | outliers, err := detect(module, testdata) 207 | if err != nil { 208 | log.Fatalf("Error detecting outliers: %s", err) 209 | } 210 | fmt.Println(outliers) 211 | 212 | } 213 | 214 | func main() { 215 | 216 | defer python3.Py_Finalize() 217 | python3.Py_Initialize() 218 | if !python3.Py_IsInitialized() { 219 | fmt.Println("Error initializing the python interpreter") 220 | os.Exit(1) 221 | } 222 | 223 | dir, err := filepath.Abs(filepath.Dir(os.Args[0])) 224 | if err != nil { 225 | log.Fatal(err) 226 | } 227 | 228 | // we could also use PySys_GetObject("path") + PySys_SetPath, 229 | //but this is easier (at the cost of less flexible error handling) 230 | ret := python3.PyRun_SimpleString("import sys\nsys.path.append(\"" + dir + "\")") 231 | if ret != 0 { 232 | log.Fatalf("error appending '%s' to python sys.path", dir) 233 | } 234 | 235 | oImport := python3.PyImport_ImportModule("pyoutliers") //ret val: new ref 236 | if !(oImport != nil && python3.PyErr_Occurred() == nil) { 237 | python3.PyErr_Print() 238 | log.Fatal("failed to import module 'pyoutliers'") 239 | } 240 | 241 | defer oImport.DecRef() 242 | 243 | oModule := python3.PyImport_AddModule("pyoutliers") //ret val: borrowed ref (from oImport) 244 | 245 | if !(oModule != nil && python3.PyErr_Occurred() == nil) { 246 | python3.PyErr_Print() 247 | log.Fatal("failed to add module 'pyoutliers'") 248 | } 249 | 250 | demo(oModule) 251 | 252 | } 253 | -------------------------------------------------------------------------------- /py-bindings/outliers/pkg-config/py385-Darwin-x86_64/README: -------------------------------------------------------------------------------- 1 | installer download: 2 | https://www.python.org/ftp/python/3.8.5/python-3.8.5-macosx10.9.pkg 3 | -------------------------------------------------------------------------------- /py-bindings/outliers/pkg-config/py385-Darwin-x86_64/python3.pc: -------------------------------------------------------------------------------- 1 | # See: man pkg-config 2 | prefix=/Library/Frameworks/Python.framework/Versions/3.8 3 | exec_prefix=${prefix} 4 | libdir=${exec_prefix}/lib 5 | includedir=${prefix}/include 6 | 7 | Name: Python 8 | Description: Embed Python into an application 9 | Requires: 10 | Version: 3.8 11 | Libs.private: -ldl -framework CoreFoundation 12 | Libs: -L${libdir} -lpython3.8 13 | Cflags: -I${includedir}/python3.8 14 | -------------------------------------------------------------------------------- /py-bindings/outliers/pyoutliers/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["detect"] 2 | from . import * 3 | gen_testdata = detect.gen_testdata 4 | detect = detect.detect 5 | 6 | -------------------------------------------------------------------------------- /py-bindings/outliers/pyoutliers/detect.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations # min Py version 3.7 2 | import numpy as np 3 | 4 | 5 | def detect(data: list[float]) -> list[int]: 6 | """Return indices where values more than 2 standard deviations from mean""" 7 | data = np.fromiter(data, dtype='float64') # data: np.ndarray[np.float64] 8 | indices = np.where(np.abs(data - data.mean()) > 2 * data.std())[0] 9 | return(indices.tolist()) # return: list[int] 10 | 11 | 12 | def gen_testdata() -> list[float]: 13 | """Return testdata""" 14 | data = np.random.rand(1000) 15 | indices = [7, 113, 835] 16 | for i in indices: 17 | data[i] += 97 18 | 19 | return(data.tolist()) 20 | -------------------------------------------------------------------------------- /py-bindings/outliers/set_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dp0="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | py_version=385 4 | export PKG_CONFIG_PATH=$dp0/pkg-config/py$py_version-$(uname -s)-$(uname -m) 5 | -------------------------------------------------------------------------------- /py-in-mem/.gitignore: -------------------------------------------------------------------------------- 1 | .clang_complete 2 | .exrc 3 | __pycache__/ 4 | -------------------------------------------------------------------------------- /py-in-mem/Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | RUN apt-get update && apt-get install -y curl gcc make pkg-config 3 | RUN curl -LO https://golang.org/dl/go1.14.7.linux-amd64.tar.gz 4 | RUN tar xz -C /opt -f go1.14.7.linux-amd64.tar.gz 5 | ENV PATH="/opt/go/bin:${PATH}" 6 | RUN python -m pip install --upgrade pip 7 | RUN python -m pip install numpy~=1.19 8 | WORKDIR /py-in-mem 9 | COPY . . 10 | RUN make test 11 | -------------------------------------------------------------------------------- /py-in-mem/Makefile: -------------------------------------------------------------------------------- 1 | NPY_INC = $(shell python -c 'import numpy; print(numpy.get_include())') 2 | 3 | test: 4 | PYTHONPATH=$(PWD) CGO_CFLAGS="-I $(NPY_INC)" \ 5 | go test -v 6 | 7 | bench: 8 | PYTHONPATH=$(PWD) CGO_CFLAGS="-I $(NPY_INC)" \ 9 | go test -run NONE -bench . 10 | -------------------------------------------------------------------------------- /py-in-mem/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Go ↔ Python: Part IV Using Python in Memory 3 | 4 | ### Introduction 5 | 6 | In [a previous post](https://www.ardanlabs.com/blog/2020/06/python-go-grpc.html) we used [gRPC](https://grpc.io/) to call Python code from Go. gRPC is a great framework, but there is a performance cost to it. Every function call needs to marshal the arguments using [protobuf](https://developers.google.com/protocol-buffers), make a network call over [HTTP/2](https://en.wikipedia.org/wiki/HTTP/2), and then un-marshal the result using `protobuf`. 7 | 8 | In this blog post, we'll get rid of the networking layer and to some extent, the marshalling. We'll do this by using [cgo](https://golang.org/cmd/cgo/) to interact with Python as a shared library. 9 | 10 | I'm not going to cover all of the code in detail in order to keep this blog size down. You can find all the code on [github](https://github.com/ardanlabs/python-go/tree/master/py-in-mem) and I did my best to provide proper documentation. Feel free to reach out and ask me questions if you don’t understand something. 11 | 12 | And finally, if you want to follow along, you’ll need to install the following (apart from Go): 13 | 14 | * Python 3.8 15 | * numpy 16 | * A C compiler (such as gcc) 17 | 18 | ### A Crash Course in Python Internals 19 | 20 | The version of Python most of us use is called `CPython`. It's written in C and is designed to be extended and embedded using C. In this section, we'll cover some topics that will help you understand the code I’m going to show. 21 | 22 | _Note: The Python C API is well [documented](https://docs.python.org/3/c-api/index.html), and there’s even [a book](https://realpython.com/products/cpython-internals-book/) in the works._ 23 | 24 | In CPython, every value is a `PyObject *` and most of Python's API functions will return a `PyObject *` or will receive a `PyObject *` as an argument. Also, errors are signaled by returning `NULL`, and you can use the `PyErr_Occurred` function to get the last exception raised. 25 | 26 | CPython uses a [reference counting](https://en.wikipedia.org/wiki/Reference_counting) garbage collector which means that every `PyObject *` has a counter for how many variables are referencing it. Once the reference counter reaches 0, Python frees the object's memory. As a programmer, you need to take care to decrement the reference counter using the `Py_DECREF` C macro once you're done with an object. 27 | 28 | ### From Go to Python and Back Again 29 | 30 | Our code tries to minimize memory allocations and avoid unnecessary serialization. In order to do this, we will share memory between Go and Python instead of allocating new memory and copying that data on each side of the function call and return. Sharing memory between two runtimes is tricky and you need to pay a lot of attention to who owns what piece of memory and when each runtime is allowed to release it. 31 | 32 | **Figure 1** 33 | ![](images/go-py-stack.png) 34 | 35 | 36 | Figure 1 shows the flow of data from the Go function to the Python function. 37 | 38 | Our input is a Go slice (`[]float64`) which has an underlying array in memory managed by Go. We will pass a pointer to the slice’s underlying array to C, which in turn will create a [numpy](https://numpy.org/) array that will use the same underlying array in memory. It’s this numpy array that will be passed as input to the Python outliers detection function called `detect`. 39 | 40 | **Figure 2** 41 | ![](images/py-go-stack.png) 42 | 43 | 44 | Figure 2 shows the flow of data from the Python function back to the Go function. 45 | 46 | When the Python `detect` function completes, it returns a new numpy array whose underlying memory is allocated and managed by Python. Like we did between Go and Python, we will share the memory back to Go by passing the Python pointer to the underlying numpy array (via C). 47 | 48 | In order to simplify memory management, on the Go side once we have access to the numpy array pointer, we create a new Go slice (`[]int`) and copy the content of the numpy array inside.. Then we tell Python it can free the memory it allocated for the numpy array. 49 | 50 | After the call to `detect` completes, the only memory we are left with is the input (`[]float64`) and the output (`[]int`) slices both being managed by Go. Any Python memory allocations should be released. 51 | 52 | ### Code Overview 53 | 54 | Our Go code is going to load and initialize a Python shared library so it can call the `detect` function that uses numpy to perform [outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection) on a series of floating point values. 55 | 56 | These are the steps that we will follow: 57 | 58 | * Convert the Go slice `[]float64` parameter to a C `double *` (`outliers.go`) 59 | * Create a numpy array from the C `double *` (`glue.c`) 60 | * Call the Python function with the numpy array (`glue.c`) 61 | * Get back a numpy array with indices of outliers (`glue.c`) 62 | * Extract C `long *` from the numpy array (`glue.c`) 63 | * Convert the C `long *` to a Go slice `[]int` and return it from the Go function 64 | (`outliers.go`) 65 | 66 | The Go code is in `outliers.go`, there's some C code in `glue.c`, and finally the outlier detection Python function is in `outliers.py`. I’m not going to show the C code, but if you're curious about it have a look at [glue.c](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/glue.c). 67 | 68 | **Listing 1: Example Usage** 69 | ``` 70 | 15 o, err := NewOutliers("outliers", "detect") 71 | 16 if err != nil { 72 | 17 return err 73 | 18 } 74 | 19 defer o.Close() 75 | 20 indices, err := o.Detect(data) 76 | 21 if err != nil { 77 | 22 return err 78 | 23 } 79 | 24 fmt.Printf("outliers at: %v\n", indices) 80 | 25 return nil 81 | ``` 82 | 83 | Listing 1 shows an example of how to use what we’re going to build in Go. On line 15, we create an `Outliers` object which uses the function `detect` from the `outliers` Python module. On line 19, we make sure to free the Python object. On line 20, we call the `Detect` method and get the indices of the outliers in the data. 84 | 85 | ### Code Highlights 86 | 87 | **Listing 2: outliers.go [initialize](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/outliers.go#L25)** 88 | ``` 89 | 19 var ( 90 | 20 initOnce sync.Once 91 | 21 initErr error 92 | 22 ) 93 | 23 94 | 24 // initialize Python & numpy, idempotent 95 | 25 func initialize() { 96 | 26 initOnce.Do(func() { 97 | 27 C.init_python() 98 | 28 initErr = pyLastError() 99 | 29 }) 100 | 30 } 101 | ``` 102 | 103 | Listing 2 shows how we initialize Python for use in our Go program. On line 20, we declare a variable of type [sync.Once](https://golang.org/pkg/sync/#Once) that will be used to make sure we initialize Python only once. On line 25, we create a function to initialize Python. On line 26, we call the `Do` method to call the initialization code and on line 28, we set the `initErr` variable to the last Python error. 104 | 105 | **Listing 3: outliers.go [Outliers](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/outliers.go#L32)** 106 | ``` 107 | 32 // Outliers does outlier detection 108 | 33 type Outliers struct { 109 | 34 fn *C.PyObject // Outlier detection Python function object 110 | 35 } 111 | ``` 112 | 113 | Listing 3 shows the definition of the `Outliers` struct. It has one field on line 34 which is a pointer to the Python function object that does the actual outlier detection. 114 | 115 | **Listing 4: outliers.go [NewOutliers](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/outliers.go#L38)** 116 | ``` 117 | 37 // NewOutliers returns an new Outliers using moduleName.funcName Python function 118 | 38 func NewOutliers(moduleName, funcName string) (*Outliers, error) { 119 | 39 initialize() 120 | 40 if initErr != nil { 121 | 41 return nil, initErr 122 | 42 } 123 | 43 124 | 44 fn, err := loadPyFunc(moduleName, funcName) 125 | 45 if err != nil { 126 | 46 return nil, err 127 | 47 } 128 | 48 129 | 49 return &Outliers{fn}, nil 130 | 50 } 131 | ``` 132 | 133 | Listing 4 shows the `NewOutliers` function that created an `Outliers` struct. On lines 39-42, we make sure Python is initialized and there's no error. On line 44, we get a pointer to the Python `detect` function. This is the same as doing an `import` statement in Python. On line 49, we save this Python pointer for later use in the `Outliers` struct. 134 | 135 | **Listing 5: outliers.go [Detect](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/outliers.go#L52)** 136 | ``` 137 | 52 // Detect returns slice of outliers indices 138 | 53 func (o *Outliers) Detect(data []float64) ([]int, error) { 139 | 54 if o.fn == nil { 140 | 55 return nil, fmt.Errorf("closed") 141 | 56 } 142 | 57 143 | 58 if len(data) == 0 { // Short path 144 | 59 return nil, nil 145 | 60 } 146 | 61 147 | 62 // Convert []float64 to C double* 148 | 63 carr := (*C.double)(&(data[0])) 149 | 64 res := C.detect(o.fn, carr, (C.long)(len(data))) 150 | 65 151 | 66 // Tell Go's GC to keep data alive until here 152 | 67 runtime.KeepAlive(data) 153 | 68 if res.err != 0 { 154 | 69 return nil, pyLastError() 155 | 70 } 156 | 71 157 | 72 indices, err := cArrToSlice(res.indices, res.size) 158 | 73 if err != nil { 159 | 74 return nil, err 160 | 75 } 161 | 76 162 | 77 // Free Python array object 163 | 78 C.py_decref(res.obj) 164 | 79 return indices, nil 165 | 80 } 166 | 167 | ``` 168 | 169 | Listing 5 shows the code for the `Outliers.Detect` method. On line 63, we convert Go’s `[]float64` slice to a C `double *` by taking the address of the first element in the underlying slice value. On line 64, we call the Python `detect` function via CGO and we get back a result. On line 67, we tell Go's garbage collector that it can't reclaim the memory for `data` before this point in the program. On lines 68-70, we check if there was an error calling `detect`. On lines 72, we convert the C `double *` to a Go `[]int` slice. On line 79, we decrement the Python return value reference count. 170 | 171 | **Listing 6: outliers.go [Outliers.Close](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/outliers.go#L82) method** 172 | ``` 173 | 82 // Close frees the underlying Python function 174 | 83 // You can't use the object after closing it 175 | 84 func (o *Outliers) Close() { 176 | 85 if o.fn == nil { 177 | 86 return 178 | 87 } 179 | 88 C.py_decref(o.fn) 180 | 89 o.fn = nil 181 | 90 } 182 | ``` 183 | 184 | Listing 6 shows the 'Outliers.Close` method. On line 88, we decrement the Python function object reference count and on line 89, we set the `fn` field to `nil` to signal the `Outliers` object is closed. 185 | 186 | ### Building 187 | 188 | The glue code is using header files from Python and numpy. In order to build, we need to tell [cgo](https://golang.org/cmd/cgo/) where to find these header files. 189 | 190 | **Listing 7: outliers.go [cgo directives](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/outliers.go#L12)** 191 | ``` 192 | 11 /* 193 | 12 #cgo pkg-config: python3 194 | 13 #cgo LDFLAGS: -lpython3.8 195 | 14 196 | 15 #include "glue.h" 197 | 16 */ 198 | 17 import "C" 199 | ``` 200 | 201 | Listing 7 shows the `cgo` directives. 202 | 203 | On line 12, we use [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) to find C compiler directives for Python. On line 13, we tell `cgo` to use the Python 3.8 shared library. On line 15, we import the C code definitions from `glue.h` and on line 17, we have the `import "C"` directive that *must* come right after the comment for using `cgo`. 204 | 205 | Telling cgo where to find the numpy headers is tricky since numpy doesn’t come with a `pkg-config` file, but has a Python function that will tell you where the headers are. For security reasons, `cgo` won’t run arbitrary commands. I opted to ask the user to set the `CGO_CFLAGS` environment variable before building or installing the package. 206 | 207 | **Listing 8: Build commands** 208 | ``` 209 | 01 $ export CGO_CFLAGS="-I $(python -c 'import numpy; print(numpy.get_include())'" 210 | 02 $ go build 211 | ``` 212 | 213 | Listing 8 shows how to build the package. On line 01, we set `CGO_CFLAGS` to a value printed from a short Python program that prints the location of the numpy header files. On line 02, we build the package. 214 | 215 | I like to use [make](https://www.gnu.org/software/make/) to automate such tasks. Have a look at the [Makefile](https://github.com/ardanlabs/python-go/blob/master/py-in-mem/Makefile) to learn more. 216 | 217 | ### Conclusion 218 | 219 | I'd like to start by thanking the awesome people at the (aptly named) `#darkarts` channel in [Gophers Slack](https://gophers.slack.com/) for their help and insights. 220 | 221 | The code we wrote here is tricky and error prone so you should have some tight performance goals before going down this path. Benchmarking on my machine shows this code is about 45 times faster than the equivalent [gRPC code](https://www.ardanlabs.com/blog/2020/06/python-go-grpc.html) code, the function call overhead (without the outliers calculation time) is about 237 times faster. Even though I'm programming in Go for 10 years and in Python close to 25 - I learned some new things. 222 | 223 | -------------------------------------------------------------------------------- /py-in-mem/bench.ipy: -------------------------------------------------------------------------------- 1 | # vim: ft=python 2 | import numpy as np 3 | 4 | 5 | data = np.random.rand(1000) 6 | indices = [7, 113, 835] 7 | 8 | for i in indices: 9 | data[i] += 97 10 | 11 | %run outliers.py 12 | %timeit detect(data) 13 | -------------------------------------------------------------------------------- /py-in-mem/bench.txt: -------------------------------------------------------------------------------- 1 | gRPC 2 | BenchmarkClient-4 715 1604483 ns/op 3 | 4 | in-mem 5 | BenchmarkOutliers-4 32835 35527 ns/op 6 | 7 | pure python 8 | 28.9 µs ± 111 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each) 9 | 10 | 11 | call: 45.162355 12 | call only: 237.7520748453297 13 | -------------------------------------------------------------------------------- /py-in-mem/doc.go: -------------------------------------------------------------------------------- 1 | /* Package outliers provides outlier detection by calling a Python function. 2 | 3 | You *must* have numpy installed and the Python function you're calling should 4 | be importable (in the PYTHONPATH). 5 | 6 | You need to set CGO_CFLAGS before building the code 7 | 8 | $ export CGO_CFLAGS="-I $(python -c 'import numpy; print(numpy.get_include())'" 9 | $ go build 10 | 11 | Example: 12 | 13 | import ( 14 | "fmt" 15 | 16 | "github.com/ardanlabs/python-go/outliers" 17 | ) 18 | 19 | func main() { 20 | // Create data 21 | const size = 1000 22 | data := make([]float64, size) 23 | for i := 0; i < size; i++ { 24 | data[i] = rand.Float64() 25 | } 26 | data[9] = 92.3 27 | data[238] = 103.2 28 | data[743] = 86.1 29 | 30 | // Use "detect" function from "outliers" module 31 | o, err := outliers.NewOutliers("outliers", "detect") 32 | if err != nil { 33 | fmt.Printf("can't load 'outliers.detect': %s", err) 34 | return 35 | } 36 | indices, err := o.Detect(data) 37 | if err != nil { 38 | fmt.Println("can't call outliers.detect: %s", err) 39 | return 40 | } 41 | fmt.Println(indices) // [9 238 743] 42 | o.Close() // Free the underlying Python function 43 | } 44 | */ 45 | package outliers 46 | -------------------------------------------------------------------------------- /py-in-mem/example.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | func genData() []float64 { 10 | return nil 11 | } 12 | 13 | func Example() error { 14 | data := genData() 15 | o, err := NewOutliers("outliers", "detect") 16 | if err != nil { 17 | return err 18 | } 19 | defer o.Close() 20 | indices, err := o.Detect(data) 21 | if err != nil { 22 | return err 23 | } 24 | fmt.Printf("outliers at: %v\n", indices) 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /py-in-mem/glue.c: -------------------------------------------------------------------------------- 1 | #include "glue.h" 2 | #define NPY_NO_DEPRECATED_API NPY_1_19_API_VERSION 3 | #include 4 | 5 | // Return void * since import_array is a macro returning NULL 6 | void *init_python() { 7 | Py_Initialize(); 8 | import_array(); 9 | } 10 | 11 | // Load function, same as "import module_name.func_name as obj" in Python 12 | // Returns the function object or NULL if not found 13 | PyObject *load_func(const char *module_name, char *func_name) { 14 | // Import the module 15 | PyObject *py_mod_name = PyUnicode_FromString(module_name); 16 | if (py_mod_name == NULL) { 17 | return NULL; 18 | } 19 | 20 | PyObject *module = PyImport_Import(py_mod_name); 21 | Py_DECREF(py_mod_name); 22 | if (module == NULL) { 23 | return NULL; 24 | } 25 | 26 | // Get function, same as "getattr(module, func_name)" in Python 27 | PyObject *func = PyObject_GetAttrString(module, func_name); 28 | Py_DECREF(module); 29 | return func; 30 | } 31 | 32 | // Call a function with array of values 33 | result_t detect(PyObject *func, double *values, long size) { 34 | result_t res = {NULL, 0}; 35 | 36 | // Create numpy array from values 37 | npy_intp dim[] = {size}; 38 | PyObject *arr = PyArray_SimpleNewFromData(1, dim, NPY_DOUBLE, values); 39 | if (arr == NULL) { 40 | res.err = 1; 41 | return res; 42 | } 43 | 44 | // Construct function arguments 45 | PyObject *args = PyTuple_New(1); 46 | PyTuple_SetItem(args, 0, arr); 47 | 48 | PyArrayObject *out = (PyArrayObject *)PyObject_CallObject(func, args); 49 | if (out == NULL) { 50 | res.err = 1; 51 | return res; 52 | } 53 | 54 | res.obj = (PyObject *)out; 55 | res.size = PyArray_SIZE(out); 56 | res.indices = (long *)PyArray_GETPTR1(out, 0); 57 | return res; 58 | } 59 | 60 | // Return last error as char *, NULL if there was no error 61 | const char *py_last_error() { 62 | PyObject *err = PyErr_Occurred(); 63 | if (err == NULL) { 64 | return NULL; 65 | } 66 | 67 | PyObject *str = PyObject_Str(err); 68 | const char *utf8 = PyUnicode_AsUTF8(str); 69 | Py_DECREF(str); 70 | return utf8; 71 | } 72 | 73 | // Decrement reference counter for object. We can't use Py_DECREF directly from 74 | // Go since it's a macro 75 | void py_decref(PyObject *obj) { Py_DECREF(obj); } 76 | -------------------------------------------------------------------------------- /py-in-mem/glue.h: -------------------------------------------------------------------------------- 1 | #ifndef GLUE_H 2 | #define GLUE_H 3 | 4 | #include 5 | 6 | // Result of calling detect 7 | typedef struct { 8 | PyObject *obj; // numpy array object, so we can free it 9 | long *indices; // indices of outliers 10 | long size; // number of outliers 11 | int err; // Flag if there was an error 12 | } result_t; 13 | 14 | void *init_python(); 15 | PyObject *load_func(const char *module_name, char *func_name); 16 | result_t detect(PyObject *func, double *values, long size); 17 | const char *py_last_error(); 18 | void py_decref(PyObject *obj); 19 | 20 | #endif // GLUE_H 21 | -------------------------------------------------------------------------------- /py-in-mem/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ardanlabs/python-go/outliers 2 | 3 | go 1.14 4 | 5 | require github.com/stretchr/testify v1.6.1 6 | -------------------------------------------------------------------------------- /py-in-mem/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 6 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 7 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 8 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 9 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 10 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 11 | -------------------------------------------------------------------------------- /py-in-mem/images/data-flow.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "excalidraw", 3 | "version": 2, 4 | "source": "https://excalidraw.com", 5 | "elements": [ 6 | { 7 | "id": "za5oYbz-EEci3gh1lWXTj", 8 | "type": "line", 9 | "x": 449, 10 | "y": 326, 11 | "width": 633, 12 | "height": 1, 13 | "angle": 0, 14 | "strokeColor": "#000000", 15 | "backgroundColor": "transparent", 16 | "fillStyle": "hachure", 17 | "strokeWidth": 1, 18 | "strokeStyle": "solid", 19 | "roughness": 0, 20 | "opacity": 100, 21 | "groupIds": [], 22 | "seed": 365967344, 23 | "version": 76, 24 | "versionNonce": 409173776, 25 | "isDeleted": false, 26 | "boundElementIds": null, 27 | "points": [ 28 | [ 29 | 0, 30 | 0 31 | ], 32 | [ 33 | 633, 34 | 1 35 | ] 36 | ], 37 | "lastCommittedPoint": null, 38 | "startBinding": null, 39 | "endBinding": null 40 | }, 41 | { 42 | "id": "KJ7XJ_Zheg1BELyndD98V", 43 | "type": "line", 44 | "x": 451.49999999999994, 45 | "y": 497.5, 46 | "width": 633, 47 | "height": 1, 48 | "angle": 0, 49 | "strokeColor": "#000000", 50 | "backgroundColor": "transparent", 51 | "fillStyle": "hachure", 52 | "strokeWidth": 1, 53 | "strokeStyle": "solid", 54 | "roughness": 0, 55 | "opacity": 100, 56 | "groupIds": [], 57 | "seed": 1647770608, 58 | "version": 145, 59 | "versionNonce": 1800110576, 60 | "isDeleted": false, 61 | "boundElementIds": null, 62 | "points": [ 63 | [ 64 | 0, 65 | 0 66 | ], 67 | [ 68 | 633, 69 | 1 70 | ] 71 | ], 72 | "lastCommittedPoint": null, 73 | "startBinding": null, 74 | "endBinding": null 75 | }, 76 | { 77 | "id": "6m6_d2fwvXrXjGmsBgnYI", 78 | "type": "text", 79 | "x": 1018, 80 | "y": 278, 81 | "width": 38, 82 | "height": 35, 83 | "angle": 0, 84 | "strokeColor": "#364fc7", 85 | "backgroundColor": "transparent", 86 | "fillStyle": "hachure", 87 | "strokeWidth": 1, 88 | "strokeStyle": "solid", 89 | "roughness": 0, 90 | "opacity": 100, 91 | "groupIds": [], 92 | "seed": 1654248720, 93 | "version": 70, 94 | "versionNonce": 636494832, 95 | "isDeleted": false, 96 | "boundElementIds": null, 97 | "text": "Go", 98 | "fontSize": 28, 99 | "fontFamily": 1, 100 | "textAlign": "left", 101 | "verticalAlign": "top", 102 | "baseline": 25 103 | }, 104 | { 105 | "id": "Tu_BwPlqxo0IfIrXLhcFq", 106 | "type": "text", 107 | "x": 1019, 108 | "y": 356.5, 109 | "width": 45, 110 | "height": 35, 111 | "angle": 0, 112 | "strokeColor": "#364fc7", 113 | "backgroundColor": "transparent", 114 | "fillStyle": "hachure", 115 | "strokeWidth": 1, 116 | "strokeStyle": "solid", 117 | "roughness": 0, 118 | "opacity": 100, 119 | "groupIds": [], 120 | "seed": 1371911440, 121 | "version": 97, 122 | "versionNonce": 46777840, 123 | "isDeleted": false, 124 | "boundElementIds": null, 125 | "text": "cgo", 126 | "fontSize": 28, 127 | "fontFamily": 1, 128 | "textAlign": "left", 129 | "verticalAlign": "top", 130 | "baseline": 25 131 | }, 132 | { 133 | "id": "JrJ0Ro_9pmsJAa2tFDs56", 134 | "type": "text", 135 | "x": 998.5, 136 | "y": 520.5, 137 | "width": 90, 138 | "height": 35, 139 | "angle": 0, 140 | "strokeColor": "#364fc7", 141 | "backgroundColor": "transparent", 142 | "fillStyle": "hachure", 143 | "strokeWidth": 1, 144 | "strokeStyle": "solid", 145 | "roughness": 0, 146 | "opacity": 100, 147 | "groupIds": [], 148 | "seed": 1745715472, 149 | "version": 177, 150 | "versionNonce": 904690160, 151 | "isDeleted": false, 152 | "boundElementIds": null, 153 | "text": "Python", 154 | "fontSize": 28, 155 | "fontFamily": 1, 156 | "textAlign": "left", 157 | "verticalAlign": "top", 158 | "baseline": 25 159 | }, 160 | { 161 | "id": "cQA6pnAwYSL3Mg-RLlWE9", 162 | "type": "text", 163 | "x": 606, 164 | "y": 509.5, 165 | "width": 199, 166 | "height": 72, 167 | "angle": 0, 168 | "strokeColor": "#000000", 169 | "backgroundColor": "transparent", 170 | "fillStyle": "hachure", 171 | "strokeWidth": 1, 172 | "strokeStyle": "solid", 173 | "roughness": 0, 174 | "opacity": 100, 175 | "groupIds": [], 176 | "seed": 716222448, 177 | "version": 301, 178 | "versionNonce": 2126637552, 179 | "isDeleted": false, 180 | "boundElementIds": null, 181 | "text": "\ndef detect(data):\n ...", 182 | "fontSize": 20, 183 | "fontFamily": 3, 184 | "textAlign": "left", 185 | "verticalAlign": "top", 186 | "baseline": 67 187 | }, 188 | { 189 | "id": "-r8iZfkfbOyKhJS4KydM2", 190 | "type": "text", 191 | "x": 458, 192 | "y": 245.5, 193 | "width": 544, 194 | "height": 57, 195 | "angle": 0, 196 | "strokeColor": "#000000", 197 | "backgroundColor": "transparent", 198 | "fillStyle": "hachure", 199 | "strokeWidth": 1, 200 | "strokeStyle": "solid", 201 | "roughness": 0, 202 | "opacity": 100, 203 | "groupIds": [], 204 | "seed": 595808240, 205 | "version": 110, 206 | "versionNonce": 919293200, 207 | "isDeleted": false, 208 | "boundElementIds": [ 209 | "zzpFZil2y7DsL0X9tR6E0", 210 | "aHIu-C8Az1Z-__0dMY_VP" 211 | ], 212 | "text": "func (o *Outliers) Detect(data []float64) ([]int, error) {\n ...\n}", 213 | "fontSize": 16, 214 | "fontFamily": 3, 215 | "textAlign": "left", 216 | "verticalAlign": "top", 217 | "baseline": 53 218 | }, 219 | { 220 | "id": "dgCQCR-mB007J1HTf1uxT", 221 | "type": "line", 222 | "x": 455.49999999999994, 223 | "y": 418.5, 224 | "width": 633, 225 | "height": 1, 226 | "angle": 0, 227 | "strokeColor": "#000000", 228 | "backgroundColor": "transparent", 229 | "fillStyle": "hachure", 230 | "strokeWidth": 1, 231 | "strokeStyle": "solid", 232 | "roughness": 0, 233 | "opacity": 100, 234 | "groupIds": [], 235 | "seed": 164966160, 236 | "version": 215, 237 | "versionNonce": 936660976, 238 | "isDeleted": false, 239 | "boundElementIds": null, 240 | "points": [ 241 | [ 242 | 0, 243 | 0 244 | ], 245 | [ 246 | 633, 247 | 1 248 | ] 249 | ], 250 | "lastCommittedPoint": null, 251 | "startBinding": null, 252 | "endBinding": null 253 | }, 254 | { 255 | "id": "WIpj-5hoN_vuBZVHgDCHX", 256 | "type": "text", 257 | "x": 1021.5, 258 | "y": 437.5, 259 | "width": 14, 260 | "height": 35, 261 | "angle": 0, 262 | "strokeColor": "#364fc7", 263 | "backgroundColor": "transparent", 264 | "fillStyle": "hachure", 265 | "strokeWidth": 1, 266 | "strokeStyle": "solid", 267 | "roughness": 0, 268 | "opacity": 100, 269 | "groupIds": [], 270 | "seed": 308962064, 271 | "version": 117, 272 | "versionNonce": 879012848, 273 | "isDeleted": false, 274 | "boundElementIds": null, 275 | "text": "c", 276 | "fontSize": 28, 277 | "fontFamily": 1, 278 | "textAlign": "left", 279 | "verticalAlign": "top", 280 | "baseline": 25 281 | }, 282 | { 283 | "id": "zzpFZil2y7DsL0X9tR6E0", 284 | "type": "arrow", 285 | "x": 515, 286 | "y": 317, 287 | "width": 1, 288 | "height": 202, 289 | "angle": 0, 290 | "strokeColor": "#000000", 291 | "backgroundColor": "transparent", 292 | "fillStyle": "hachure", 293 | "strokeWidth": 1, 294 | "strokeStyle": "solid", 295 | "roughness": 0, 296 | "opacity": 100, 297 | "groupIds": [], 298 | "seed": 940885776, 299 | "version": 88, 300 | "versionNonce": 1029695984, 301 | "isDeleted": false, 302 | "boundElementIds": null, 303 | "points": [ 304 | [ 305 | 0, 306 | 0 307 | ], 308 | [ 309 | 1, 310 | 202 311 | ] 312 | ], 313 | "lastCommittedPoint": null, 314 | "startBinding": { 315 | "elementId": "-r8iZfkfbOyKhJS4KydM2", 316 | "focus": 0.790813588612488, 317 | "gap": 14.5 318 | }, 319 | "endBinding": null 320 | }, 321 | { 322 | "id": "2WMqEFi5zRfXeo85D5slw", 323 | "type": "text", 324 | "x": 528, 325 | "y": 368, 326 | "width": 135, 327 | "height": 18, 328 | "angle": 0, 329 | "strokeColor": "#000000", 330 | "backgroundColor": "transparent", 331 | "fillStyle": "hachure", 332 | "strokeWidth": 1, 333 | "strokeStyle": "solid", 334 | "roughness": 0, 335 | "opacity": 100, 336 | "groupIds": [], 337 | "seed": 747782640, 338 | "version": 68, 339 | "versionNonce": 785133552, 340 | "isDeleted": false, 341 | "boundElementIds": null, 342 | "text": "[]float64 -> double*", 343 | "fontSize": 16, 344 | "fontFamily": 2, 345 | "textAlign": "left", 346 | "verticalAlign": "top", 347 | "baseline": 14 348 | }, 349 | { 350 | "id": "8CqcRzMGwmxzPVMOM8WQH", 351 | "type": "text", 352 | "x": 527.5, 353 | "y": 445, 354 | "width": 167, 355 | "height": 18, 356 | "angle": 0, 357 | "strokeColor": "#000000", 358 | "backgroundColor": "transparent", 359 | "fillStyle": "hachure", 360 | "strokeWidth": 1, 361 | "strokeStyle": "solid", 362 | "roughness": 0, 363 | "opacity": 100, 364 | "groupIds": [], 365 | "seed": 548458768, 366 | "version": 104, 367 | "versionNonce": 867335152, 368 | "isDeleted": false, 369 | "boundElementIds": null, 370 | "text": "double* -> numpy array", 371 | "fontSize": 16, 372 | "fontFamily": 2, 373 | "textAlign": "left", 374 | "verticalAlign": "top", 375 | "baseline": 14 376 | }, 377 | { 378 | "id": "aHIu-C8Az1Z-__0dMY_VP", 379 | "type": "arrow", 380 | "x": 780, 381 | "y": 521, 382 | "width": 2, 383 | "height": 216, 384 | "angle": 0, 385 | "strokeColor": "#000000", 386 | "backgroundColor": "transparent", 387 | "fillStyle": "hachure", 388 | "strokeWidth": 1, 389 | "strokeStyle": "solid", 390 | "roughness": 0, 391 | "opacity": 100, 392 | "groupIds": [], 393 | "seed": 1941404432, 394 | "version": 68, 395 | "versionNonce": 1487833584, 396 | "isDeleted": false, 397 | "boundElementIds": null, 398 | "points": [ 399 | [ 400 | 0, 401 | 0 402 | ], 403 | [ 404 | -2, 405 | -216 406 | ] 407 | ], 408 | "lastCommittedPoint": null, 409 | "startBinding": null, 410 | "endBinding": { 411 | "elementId": "-r8iZfkfbOyKhJS4KydM2", 412 | "focus": -0.17524528558553965, 413 | "gap": 2.5 414 | } 415 | }, 416 | { 417 | "id": "x1q0SX1BpQeZyu6aRnj1w", 418 | "type": "text", 419 | "x": 811, 420 | "y": 445, 421 | "width": 149, 422 | "height": 18, 423 | "angle": 0, 424 | "strokeColor": "#000000", 425 | "backgroundColor": "transparent", 426 | "fillStyle": "hachure", 427 | "strokeWidth": 1, 428 | "strokeStyle": "solid", 429 | "roughness": 0, 430 | "opacity": 100, 431 | "groupIds": [], 432 | "seed": 2066271728, 433 | "version": 47, 434 | "versionNonce": 1450549520, 435 | "isDeleted": false, 436 | "boundElementIds": null, 437 | "text": "numpy array -> long*", 438 | "fontSize": 16, 439 | "fontFamily": 2, 440 | "textAlign": "left", 441 | "verticalAlign": "top", 442 | "baseline": 14 443 | }, 444 | { 445 | "id": "1xZE18FeScBxNDeAND0Mv", 446 | "type": "text", 447 | "x": 810.5, 448 | "y": 370, 449 | "width": 86, 450 | "height": 18, 451 | "angle": 0, 452 | "strokeColor": "#000000", 453 | "backgroundColor": "transparent", 454 | "fillStyle": "hachure", 455 | "strokeWidth": 1, 456 | "strokeStyle": "solid", 457 | "roughness": 0, 458 | "opacity": 100, 459 | "groupIds": [], 460 | "seed": 1067956720, 461 | "version": 112, 462 | "versionNonce": 359638288, 463 | "isDeleted": false, 464 | "boundElementIds": null, 465 | "text": "long* -> []int", 466 | "fontSize": 16, 467 | "fontFamily": 2, 468 | "textAlign": "left", 469 | "verticalAlign": "top", 470 | "baseline": 14 471 | } 472 | ], 473 | "appState": { 474 | "viewBackgroundColor": "#ffffff", 475 | "gridSize": null 476 | } 477 | } -------------------------------------------------------------------------------- /py-in-mem/images/data-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/py-in-mem/images/data-flow.png -------------------------------------------------------------------------------- /py-in-mem/images/data-ownership.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/py-in-mem/images/data-ownership.png -------------------------------------------------------------------------------- /py-in-mem/images/go-py-stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/py-in-mem/images/go-py-stack.png -------------------------------------------------------------------------------- /py-in-mem/images/go-to-py.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "excalidraw", 3 | "version": 2, 4 | "source": "https://excalidraw.com", 5 | "elements": [ 6 | { 7 | "id": "sKuqZ-NKv1mVn6H7H5Uyb", 8 | "type": "rectangle", 9 | "x": 467, 10 | "y": 112, 11 | "width": 579, 12 | "height": 37, 13 | "angle": 0, 14 | "strokeColor": "#000000", 15 | "backgroundColor": "transparent", 16 | "fillStyle": "hachure", 17 | "strokeWidth": 1, 18 | "strokeStyle": "solid", 19 | "roughness": 1, 20 | "opacity": 100, 21 | "groupIds": [], 22 | "strokeSharpness": "sharp", 23 | "seed": 1217596970, 24 | "version": 128, 25 | "versionNonce": 1398908842, 26 | "isDeleted": false, 27 | "boundElementIds": [] 28 | }, 29 | { 30 | "id": "cMqwDUALNUK0rVoISJyfu", 31 | "type": "text", 32 | "x": 594.5, 33 | "y": 118, 34 | "width": 299, 35 | "height": 25, 36 | "angle": 0, 37 | "strokeColor": "#000000", 38 | "backgroundColor": "transparent", 39 | "fillStyle": "hachure", 40 | "strokeWidth": 1, 41 | "strokeStyle": "solid", 42 | "roughness": 1, 43 | "opacity": 100, 44 | "groupIds": [], 45 | "strokeSharpness": "sharp", 46 | "seed": 150527978, 47 | "version": 126, 48 | "versionNonce": 1383536438, 49 | "isDeleted": false, 50 | "boundElementIds": [ 51 | "WTPgJyDSDOFV5tfztrf_s", 52 | "rEMqpAGhNJoRP5jvI5Rkb" 53 | ], 54 | "text": "Underlying array (Go managed)", 55 | "fontSize": 20, 56 | "fontFamily": 1, 57 | "textAlign": "center", 58 | "verticalAlign": "top", 59 | "baseline": 18 60 | }, 61 | { 62 | "id": "mSPhSH2vS7sOGOZGORwH_", 63 | "type": "text", 64 | "x": 524, 65 | "y": 193, 66 | "width": 105, 67 | "height": 24, 68 | "angle": 0, 69 | "strokeColor": "#000000", 70 | "backgroundColor": "transparent", 71 | "fillStyle": "hachure", 72 | "strokeWidth": 1, 73 | "strokeStyle": "solid", 74 | "roughness": 1, 75 | "opacity": 100, 76 | "groupIds": [], 77 | "strokeSharpness": "sharp", 78 | "seed": 180871222, 79 | "version": 81, 80 | "versionNonce": 2036297526, 81 | "isDeleted": false, 82 | "boundElementIds": [ 83 | "WTPgJyDSDOFV5tfztrf_s" 84 | ], 85 | "text": "[]float64", 86 | "fontSize": 20, 87 | "fontFamily": 3, 88 | "textAlign": "left", 89 | "verticalAlign": "top", 90 | "baseline": 19 91 | }, 92 | { 93 | "id": "zKjJPH3h9-A0JSH-x33YI", 94 | "type": "text", 95 | "x": 737.5, 96 | "y": 192, 97 | "width": 94, 98 | "height": 24, 99 | "angle": 0, 100 | "strokeColor": "#000000", 101 | "backgroundColor": "transparent", 102 | "fillStyle": "hachure", 103 | "strokeWidth": 1, 104 | "strokeStyle": "solid", 105 | "roughness": 1, 106 | "opacity": 100, 107 | "groupIds": [], 108 | "strokeSharpness": "sharp", 109 | "seed": 823202678, 110 | "version": 123, 111 | "versionNonce": 1230572726, 112 | "isDeleted": false, 113 | "boundElementIds": [ 114 | "rEMqpAGhNJoRP5jvI5Rkb" 115 | ], 116 | "text": "double *", 117 | "fontSize": 20, 118 | "fontFamily": 3, 119 | "textAlign": "left", 120 | "verticalAlign": "top", 121 | "baseline": 19 122 | }, 123 | { 124 | "id": "O33o0PTBtE5XRjdSq_g6O", 125 | "type": "text", 126 | "x": 953, 127 | "y": 192, 128 | "width": 129, 129 | "height": 24, 130 | "angle": 0, 131 | "strokeColor": "#000000", 132 | "backgroundColor": "transparent", 133 | "fillStyle": "hachure", 134 | "strokeWidth": 1, 135 | "strokeStyle": "solid", 136 | "roughness": 1, 137 | "opacity": 100, 138 | "groupIds": [], 139 | "strokeSharpness": "sharp", 140 | "seed": 1590195242, 141 | "version": 407, 142 | "versionNonce": 1491890282, 143 | "isDeleted": false, 144 | "boundElementIds": [ 145 | "rEMqpAGhNJoRP5jvI5Rkb", 146 | "oVTMOMTNd6yhjFPoVJtAU" 147 | ], 148 | "text": "numpy array", 149 | "fontSize": 20, 150 | "fontFamily": 3, 151 | "textAlign": "left", 152 | "verticalAlign": "top", 153 | "baseline": 19 154 | }, 155 | { 156 | "id": "WTPgJyDSDOFV5tfztrf_s", 157 | "type": "arrow", 158 | "x": 596.01870666631, 159 | "y": 190, 160 | "width": 63.76829205651768, 161 | "height": 37.922631706829634, 162 | "angle": 0, 163 | "strokeColor": "#000000", 164 | "backgroundColor": "transparent", 165 | "fillStyle": "hachure", 166 | "strokeWidth": 1, 167 | "strokeStyle": "solid", 168 | "roughness": 1, 169 | "opacity": 100, 170 | "groupIds": [], 171 | "strokeSharpness": "round", 172 | "seed": 1007582186, 173 | "version": 207, 174 | "versionNonce": 1700272246, 175 | "isDeleted": false, 176 | "boundElementIds": null, 177 | "points": [ 178 | [ 179 | 0, 180 | 0 181 | ], 182 | [ 183 | 63.76829205651768, 184 | -37.922631706829634 185 | ] 186 | ], 187 | "lastCommittedPoint": null, 188 | "startBinding": { 189 | "elementId": "mSPhSH2vS7sOGOZGORwH_", 190 | "focus": -0.08143939393939394, 191 | "gap": 3 192 | }, 193 | "endBinding": { 194 | "elementId": "cMqwDUALNUK0rVoISJyfu", 195 | "focus": 0.4840461642905635, 196 | "gap": 11 197 | } 198 | }, 199 | { 200 | "id": "oVTMOMTNd6yhjFPoVJtAU", 201 | "type": "arrow", 202 | "x": 1011.2942442189437, 203 | "y": 190.54642398282886, 204 | "width": 20.754362773976254, 205 | "height": 48.847742438316345, 206 | "angle": 0, 207 | "strokeColor": "#000000", 208 | "backgroundColor": "transparent", 209 | "fillStyle": "hachure", 210 | "strokeWidth": 1, 211 | "strokeStyle": "solid", 212 | "roughness": 1, 213 | "opacity": 100, 214 | "groupIds": [], 215 | "strokeSharpness": "round", 216 | "seed": 1674346346, 217 | "version": 239, 218 | "versionNonce": 1032999414, 219 | "isDeleted": false, 220 | "boundElementIds": null, 221 | "points": [ 222 | [ 223 | 0, 224 | 0 225 | ], 226 | [ 227 | -20.754362773976254, 228 | -48.847742438316345 229 | ] 230 | ], 231 | "lastCommittedPoint": null, 232 | "startBinding": { 233 | "elementId": "O33o0PTBtE5XRjdSq_g6O", 234 | "focus": -0.00703489704059986, 235 | "gap": 1.4535760171711445 236 | }, 237 | "endBinding": null 238 | }, 239 | { 240 | "id": "rEMqpAGhNJoRP5jvI5Rkb", 241 | "type": "arrow", 242 | "x": 777.9909279577435, 243 | "y": 190.91165460840799, 244 | "width": 2.496043361042439, 245 | "height": 38.635582614136894, 246 | "angle": 0, 247 | "strokeColor": "#000000", 248 | "backgroundColor": "transparent", 249 | "fillStyle": "hachure", 250 | "strokeWidth": 1, 251 | "strokeStyle": "solid", 252 | "roughness": 1, 253 | "opacity": 100, 254 | "groupIds": [], 255 | "strokeSharpness": "round", 256 | "seed": 1593634358, 257 | "version": 242, 258 | "versionNonce": 943314742, 259 | "isDeleted": false, 260 | "boundElementIds": null, 261 | "points": [ 262 | [ 263 | 0, 264 | 0 265 | ], 266 | [ 267 | -2.496043361042439, 268 | -38.635582614136894 269 | ] 270 | ], 271 | "lastCommittedPoint": null, 272 | "startBinding": { 273 | "elementId": "zKjJPH3h9-A0JSH-x33YI", 274 | "focus": -0.11854467486557224, 275 | "gap": 1.0883453915920143 276 | }, 277 | "endBinding": { 278 | "elementId": "cMqwDUALNUK0rVoISJyfu", 279 | "focus": -0.3817913702699076, 280 | "gap": 9.276071994271092 281 | } 282 | }, 283 | { 284 | "id": "mt2VmY3KBqwd1vqqlZIhQ", 285 | "type": "line", 286 | "x": 902, 287 | "y": 174, 288 | "width": 1.73073165342214, 289 | "height": 117.54764233902097, 290 | "angle": 0, 291 | "strokeColor": "#000000", 292 | "backgroundColor": "transparent", 293 | "fillStyle": "hachure", 294 | "strokeWidth": 1, 295 | "strokeStyle": "solid", 296 | "roughness": 1, 297 | "opacity": 100, 298 | "groupIds": [], 299 | "strokeSharpness": "round", 300 | "seed": 1418348790, 301 | "version": 162, 302 | "versionNonce": 1560724918, 303 | "isDeleted": false, 304 | "boundElementIds": null, 305 | "points": [ 306 | [ 307 | 0, 308 | 0 309 | ], 310 | [ 311 | 1.73073165342214, 312 | 117.54764233902097 313 | ] 314 | ], 315 | "lastCommittedPoint": null, 316 | "startBinding": null, 317 | "endBinding": null 318 | }, 319 | { 320 | "id": "RO5eeKZFU2iQLbxJKm2HV", 321 | "type": "text", 322 | "x": 536, 323 | "y": 285.5, 324 | "width": 85, 325 | "height": 35, 326 | "angle": 0, 327 | "strokeColor": "#364fc7", 328 | "backgroundColor": "transparent", 329 | "fillStyle": "hachure", 330 | "strokeWidth": 1, 331 | "strokeStyle": "solid", 332 | "roughness": 1, 333 | "opacity": 100, 334 | "groupIds": [], 335 | "strokeSharpness": "sharp", 336 | "seed": 1938801462, 337 | "version": 166, 338 | "versionNonce": 13987958, 339 | "isDeleted": false, 340 | "boundElementIds": null, 341 | "text": "[1] Go", 342 | "fontSize": 28, 343 | "fontFamily": 1, 344 | "textAlign": "left", 345 | "verticalAlign": "top", 346 | "baseline": 25 347 | }, 348 | { 349 | "id": "CrCGIrgqKWo1k1eIcGlm9", 350 | "type": "text", 351 | "x": 929, 352 | "y": 286.5, 353 | "width": 194, 354 | "height": 35, 355 | "angle": 0, 356 | "strokeColor": "#364fc7", 357 | "backgroundColor": "transparent", 358 | "fillStyle": "hachure", 359 | "strokeWidth": 1, 360 | "strokeStyle": "solid", 361 | "roughness": 1, 362 | "opacity": 100, 363 | "groupIds": [], 364 | "strokeSharpness": "sharp", 365 | "seed": 803851818, 366 | "version": 206, 367 | "versionNonce": 147863914, 368 | "isDeleted": false, 369 | "boundElementIds": null, 370 | "text": "[3] C/Python", 371 | "fontSize": 28, 372 | "fontFamily": 1, 373 | "textAlign": "left", 374 | "verticalAlign": "top", 375 | "baseline": 25 376 | }, 377 | { 378 | "id": "-dZf3xwNY7Vt0y_sEUoU4", 379 | "type": "text", 380 | "x": 740, 381 | "y": 286.5, 382 | "width": 136, 383 | "height": 35, 384 | "angle": 0, 385 | "strokeColor": "#364fc7", 386 | "backgroundColor": "transparent", 387 | "fillStyle": "hachure", 388 | "strokeWidth": 1, 389 | "strokeStyle": "solid", 390 | "roughness": 1, 391 | "opacity": 100, 392 | "groupIds": [], 393 | "strokeSharpness": "sharp", 394 | "seed": 974011626, 395 | "version": 334, 396 | "versionNonce": 713839338, 397 | "isDeleted": false, 398 | "boundElementIds": null, 399 | "text": "[2] Go/C", 400 | "fontSize": 28, 401 | "fontFamily": 1, 402 | "textAlign": "left", 403 | "verticalAlign": "top", 404 | "baseline": 25 405 | }, 406 | { 407 | "id": "67oDsCuzjn2Fz7u0Eg0bv", 408 | "type": "line", 409 | "x": 678.134634173289, 410 | "y": 180.22617883048952, 411 | "width": 1.73073165342214, 412 | "height": 117.54764233902097, 413 | "angle": 0, 414 | "strokeColor": "#000000", 415 | "backgroundColor": "transparent", 416 | "fillStyle": "hachure", 417 | "strokeWidth": 1, 418 | "strokeStyle": "solid", 419 | "roughness": 1, 420 | "opacity": 100, 421 | "groupIds": [], 422 | "strokeSharpness": "round", 423 | "seed": 1317927594, 424 | "version": 136, 425 | "versionNonce": 878667318, 426 | "isDeleted": false, 427 | "boundElementIds": null, 428 | "points": [ 429 | [ 430 | 0, 431 | 0 432 | ], 433 | [ 434 | 1.73073165342214, 435 | 117.54764233902097 436 | ] 437 | ], 438 | "lastCommittedPoint": null, 439 | "startBinding": null, 440 | "endBinding": null 441 | }, 442 | { 443 | "id": "rdMYy-aL-6MQ9SbYNxKng", 444 | "type": "text", 445 | "x": 485.5, 446 | "y": 226, 447 | "width": 178, 448 | "height": 19, 449 | "angle": 0, 450 | "strokeColor": "#000000", 451 | "backgroundColor": "transparent", 452 | "fillStyle": "hachure", 453 | "strokeWidth": 1, 454 | "strokeStyle": "solid", 455 | "roughness": 1, 456 | "opacity": 100, 457 | "groupIds": [], 458 | "strokeSharpness": "sharp", 459 | "seed": 681080054, 460 | "version": 175, 461 | "versionNonce": 1720676970, 462 | "isDeleted": false, 463 | "boundElementIds": [ 464 | "WTPgJyDSDOFV5tfztrf_s" 465 | ], 466 | "text": "outliers.go:53 data", 467 | "fontSize": 16, 468 | "fontFamily": 3, 469 | "textAlign": "left", 470 | "verticalAlign": "top", 471 | "baseline": 15 472 | }, 473 | { 474 | "id": "MQf0DY_lb6iw-jOcvlE5a", 475 | "type": "text", 476 | "x": 696, 477 | "y": 226.5, 478 | "width": 197, 479 | "height": 38, 480 | "angle": 0, 481 | "strokeColor": "#000000", 482 | "backgroundColor": "transparent", 483 | "fillStyle": "hachure", 484 | "strokeWidth": 1, 485 | "strokeStyle": "solid", 486 | "roughness": 1, 487 | "opacity": 100, 488 | "groupIds": [], 489 | "strokeSharpness": "sharp", 490 | "seed": 324713322, 491 | "version": 292, 492 | "versionNonce": 1189398518, 493 | "isDeleted": false, 494 | "boundElementIds": [ 495 | "WTPgJyDSDOFV5tfztrf_s" 496 | ], 497 | "text": "outliers.go:63 carr\nglue.c:33 values", 498 | "fontSize": 16, 499 | "fontFamily": 3, 500 | "textAlign": "left", 501 | "verticalAlign": "top", 502 | "baseline": 34 503 | }, 504 | { 505 | "id": "o3BECU3Uh5p3o2CYORTsY", 506 | "type": "text", 507 | "x": 922, 508 | "y": 229, 509 | "width": 169, 510 | "height": 38, 511 | "angle": 0, 512 | "strokeColor": "#000000", 513 | "backgroundColor": "transparent", 514 | "fillStyle": "hachure", 515 | "strokeWidth": 1, 516 | "strokeStyle": "solid", 517 | "roughness": 1, 518 | "opacity": 100, 519 | "groupIds": [], 520 | "strokeSharpness": "sharp", 521 | "seed": 319512310, 522 | "version": 384, 523 | "versionNonce": 494804214, 524 | "isDeleted": false, 525 | "boundElementIds": [ 526 | "WTPgJyDSDOFV5tfztrf_s" 527 | ], 528 | "text": "glue.c:38 arr\noutliers.py:5 data", 529 | "fontSize": 16, 530 | "fontFamily": 3, 531 | "textAlign": "left", 532 | "verticalAlign": "top", 533 | "baseline": 34 534 | } 535 | ], 536 | "appState": { 537 | "viewBackgroundColor": "#ffffff", 538 | "gridSize": null 539 | } 540 | } -------------------------------------------------------------------------------- /py-in-mem/images/go-to-py.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/py-in-mem/images/go-to-py.png -------------------------------------------------------------------------------- /py-in-mem/images/py-go-stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/py-in-mem/images/py-go-stack.png -------------------------------------------------------------------------------- /py-in-mem/images/py-to-go.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/py-in-mem/images/py-to-go.png -------------------------------------------------------------------------------- /py-in-mem/outliers.go: -------------------------------------------------------------------------------- 1 | // outliers provides outlier detection via Python 2 | package outliers 3 | 4 | import ( 5 | "fmt" 6 | "runtime" 7 | "sync" 8 | "unsafe" 9 | ) 10 | 11 | /* 12 | #cgo pkg-config: python3 13 | #cgo LDFLAGS: -lpython3.8 14 | 15 | #include "glue.h" 16 | */ 17 | import "C" 18 | 19 | var ( 20 | initOnce sync.Once 21 | initErr error 22 | ) 23 | 24 | // initialize Python & numpy, idempotent 25 | func initialize() { 26 | initOnce.Do(func() { 27 | C.init_python() 28 | initErr = pyLastError() 29 | }) 30 | } 31 | 32 | // Outliers does outlier detection 33 | type Outliers struct { 34 | fn *C.PyObject // Outlier detection Python function object 35 | } 36 | 37 | // NewOutliers returns an new Outliers using moduleName.funcName Python function 38 | func NewOutliers(moduleName, funcName string) (*Outliers, error) { 39 | initialize() 40 | if initErr != nil { 41 | return nil, initErr 42 | } 43 | 44 | fn, err := loadPyFunc(moduleName, funcName) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | return &Outliers{fn}, nil 50 | } 51 | 52 | // Detect returns slice of outliers indices 53 | func (o *Outliers) Detect(data []float64) ([]int, error) { 54 | if o.fn == nil { 55 | return nil, fmt.Errorf("closed") 56 | } 57 | 58 | if len(data) == 0 { // Short path 59 | return nil, nil 60 | } 61 | 62 | // Convert []float64 to C double* 63 | carr := (*C.double)(&(data[0])) 64 | res := C.detect(o.fn, carr, (C.long)(len(data))) 65 | 66 | // Tell Go's GC to keep data alive until here 67 | runtime.KeepAlive(data) 68 | if res.err != 0 { 69 | return nil, pyLastError() 70 | } 71 | 72 | // Create a Go slice from C long* 73 | indices, err := cArrToSlice(res.indices, res.size) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | // Free Python array object 79 | C.py_decref(res.obj) 80 | return indices, nil 81 | } 82 | 83 | // Close frees the underlying Python function 84 | // You can't use the object after closing it 85 | func (o *Outliers) Close() { 86 | if o.fn == nil { 87 | return 88 | } 89 | C.py_decref(o.fn) 90 | o.fn = nil 91 | } 92 | 93 | // loadPyFunc loads a Python function by module and function name 94 | func loadPyFunc(moduleName, funcName string) (*C.PyObject, error) { 95 | // Convert names to C char* 96 | cMod := C.CString(moduleName) 97 | cFunc := C.CString(funcName) 98 | 99 | // Free memory allocated by C.CString 100 | defer func() { 101 | C.free(unsafe.Pointer(cMod)) 102 | C.free(unsafe.Pointer(cFunc)) 103 | }() 104 | 105 | fn := C.load_func(cMod, cFunc) 106 | if fn == nil { 107 | return nil, pyLastError() 108 | } 109 | 110 | return fn, nil 111 | } 112 | 113 | // Python last error 114 | func pyLastError() error { 115 | cp := C.py_last_error() 116 | if cp == nil { 117 | return nil 118 | } 119 | 120 | err := C.GoString(cp) 121 | // We don't need to free cp, see 122 | // https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_AsUTF8AndSize 123 | // which says: "The caller is not responsible for deallocating the buffer." 124 | return fmt.Errorf("%s", err) 125 | } 126 | 127 | // Create a new []int from a *C.long 128 | func cArrToSlice(cArr *C.long, size C.long) ([]int, error) { 129 | const maxSize = 1 << 20 130 | if size > maxSize { 131 | return nil, fmt.Errorf("C array to large (%d > %d)", size, maxSize) 132 | } 133 | 134 | // Ugly hack to convert C.long* to []int - make the compiler think there's 135 | // a Go array at the C array memory location 136 | ptr := unsafe.Pointer(cArr) 137 | arr := (*[maxSize]int)(ptr) 138 | 139 | // Create a slice with copy of data managed by Go 140 | s := make([]int, size) 141 | copy(s, arr[:size]) 142 | 143 | return s, nil 144 | } 145 | -------------------------------------------------------------------------------- /py-in-mem/outliers.py: -------------------------------------------------------------------------------- 1 | """Detect outliers""" 2 | import numpy as np 3 | 4 | 5 | def detect(data): 6 | """Return indices where values more than 2 standard deviations from mean""" 7 | out = np.where(np.abs(data - data.mean()) > 2 * data.std()) 8 | # np.where returns a tuple for each dimension, we want the 1st element 9 | return out[0] 10 | -------------------------------------------------------------------------------- /py-in-mem/outliers_test.go: -------------------------------------------------------------------------------- 1 | package outliers 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func genData() ([]float64, []int) { 11 | const size = 1000 12 | data := make([]float64, size) 13 | for i := 0; i < size; i++ { 14 | data[i] = rand.Float64() 15 | } 16 | 17 | indices := []int{7, 113, 835} 18 | for _, i := range indices { 19 | data[i] += 97 20 | } 21 | 22 | return data, indices 23 | } 24 | 25 | func TestDetect(t *testing.T) { 26 | require := require.New(t) 27 | 28 | o, err := NewOutliers("outliers", "detect") 29 | require.NoError(err, "new") 30 | defer o.Close() 31 | 32 | data, indices := genData() 33 | 34 | out, err := o.Detect(data) 35 | require.NoError(err, "detect") 36 | require.Equal(indices, out, "outliers") 37 | } 38 | 39 | func TestNotFound(t *testing.T) { 40 | require := require.New(t) 41 | 42 | _, err := NewOutliers("outliers", "no-such-function") 43 | require.Error(err, "attribute") 44 | 45 | _, err = NewOutliers("no_such_module", "detect") 46 | require.Error(err, "module") 47 | } 48 | 49 | func TestNil(t *testing.T) { 50 | require := require.New(t) 51 | 52 | o, err := NewOutliers("outliers", "detect") 53 | require.NoError(err, "attribute") 54 | indices, err := o.Detect(nil) 55 | require.NoError(err, "attribute") 56 | require.Equal(0, len(indices), "len") 57 | } 58 | 59 | func BenchmarkOutliers(b *testing.B) { 60 | require := require.New(b) 61 | o, err := NewOutliers("outliers", "detect") 62 | require.NoError(err, "new") 63 | defer o.Close() 64 | 65 | data, _ := genData() 66 | 67 | b.ResetTimer() 68 | for i := 0; i < b.N; i++ { 69 | _, err := o.Detect(data) 70 | require.NoError(err) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pyext/.gitignore: -------------------------------------------------------------------------------- 1 | *.h 2 | *.so 3 | build/ 4 | checksig.egg-info/ 5 | dist/ 6 | -------------------------------------------------------------------------------- /pyext/Dockerfile.test-a: -------------------------------------------------------------------------------- 1 | # Test docker file (used by CI) 2 | FROM python:3.8-slim 3 | RUN apt-get update && apt-get install -y bzip2 curl gcc 4 | RUN curl -LO https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz 5 | RUN tar xzf go1.14.4.linux-amd64.tar.gz 6 | RUN ln -s /go/bin/go /usr/local/bin 7 | RUN python -m pip install --upgrade pip 8 | COPY ./testdata/logs /tmp/logs 9 | WORKDIR /code 10 | COPY . . 11 | RUN go build -buildmode=c-shared -o _checksig.so 12 | RUN python py_session.py 13 | -------------------------------------------------------------------------------- /pyext/Dockerfile.test-b: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | RUN apt-get update && apt-get install -y bzip2 curl gcc 3 | RUN curl -LO https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz 4 | RUN tar xzf go1.14.4.linux-amd64.tar.gz 5 | RUN ln -s /go/bin/go /usr/local/bin 6 | RUN python -m pip install --upgrade pip 7 | COPY ./testdata/logs /tmp/logs 8 | WORKDIR /code 9 | COPY . . 10 | COPY example.py /tmp 11 | RUN python setup.py sdist 12 | RUN python setup.py bdist_wheel 13 | WORKDIR /tmp 14 | # Check wheel 15 | RUN python -m pip install /code/dist/checksig-0.1.0-cp38-cp38-linux_x86_64.whl 16 | RUN python example.py 17 | # Check sdist 18 | RUN python -m pip uninstall -y checksig 19 | RUN python -m pip install /code/dist/checksig-0.1.0.tar.gz 20 | RUN python example.py 21 | -------------------------------------------------------------------------------- /pyext/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include *.go go.mod go.sum 3 | -------------------------------------------------------------------------------- /pyext/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | $(error please pick a target) 3 | 4 | 5 | _checksig.so: *.go 6 | go build -buildmode=c-shared -o $@ 7 | 8 | so: _checksig.so 9 | 10 | clean: 11 | -rm *.h *.so 12 | 13 | test: test-a test-b 14 | 15 | test-a: 16 | docker build -f Dockerfile.test-a . 17 | 18 | test-b: 19 | docker build -f Dockerfile.test-b . 20 | -------------------------------------------------------------------------------- /pyext/README-A.md: -------------------------------------------------------------------------------- 1 | # Go ↔ Python: Part II Extending Python With Go 2 | 3 | ### Introduction 4 | 5 | In [the previous post](https://www.ardanlabs.com/blog/2020/06/python-go-grpc.html) we saw how a Go service can call a Python service using gRPC. Using gRPC to connect a Go and Python program together can be a great choice, but there’s a complexity price that goes with it. You need to manage one more service, deployment becomes more complex, and you need monitoring plus alerting for each service. Compared to a monolithic application, there is an order of magnitude more complexity. 6 | 7 | In this post, we're going to reduce the complexity of using gRPC by writing a shared library in Go that a Python program can consume directly. With this approach, there’s no networking involved and depending on the data types, no marshalling as well. Out of the several approaches of calling functions from a shared library in Python, we decided to use Python's [ctypes](https://docs.python.org/3/library/ctypes.html) module. 8 | 9 | _Note: ctypes uses [libffi](https://github.com/libffi/libffi) under the hood. If you want to read some really scary C code - head over to the repo and start reading. :)_ 10 | 11 | I'll also show my workflow that is one of the big factors in my productivity. We'll first write "pure" Go code, then write code to export it to a shared library. Then we'll switch to the Python world and use Python's interactive prompt to Play around with the code. Once we're happy, we'll use what we've learned in the interactive prompt to write a Python module. 12 | 13 | ### Example: Checking the Digital Signature of Several Files in Parallel 14 | 15 | Imagine you have a directory with data files, and you need to validate the integrity of these files. The directory contains a `sha1sum.txt` file with a [sha1](https://en.wikipedia.org/wiki/SHA-1) digital signature for every file. Go, with its concurrency primitives and ability to use all the cores of your machine, is much better suited to this task than Python. 16 | 17 | **Listing 1: sha1sum.txt** 18 | ``` 19 | 6659cb84ab403dc85962fc77b9156924bbbaab2c httpd-00.log 20 | 5693325790ee53629d6ed3264760c4463a3615ee httpd-01.log 21 | fce486edf5251951c7b92a3d5098ea6400bfd63f httpd-02.log 22 | b5b04eb809e9c737dbb5de76576019e9db1958fd httpd-03.log 23 | ff0e3f644371d0fbce954dace6f678f9f77c3e08 httpd-04.log 24 | c154b2aa27122c07da77b85165036906fb6cbc3c httpd-05.log 25 | 28fccd72fb6fe88e1665a15df397c1d207de94ef httpd-06.log 26 | 86ed10cd87ac6f9fb62f6c29e82365c614089ae8 httpd-07.log 27 | feaf526473cb2887781f4904bd26f021a91ee9eb httpd-08.log 28 | 330d03af58919dd12b32804d9742b55c7ed16038 httpd-09.log 29 | ``` 30 | 31 | Listing 1 shows an example of a digital signature file. It provides hash codes for all the different log files contained in the directory. This file can be used to verify that a log file was downloaded correctly or has not been tampered with. We’ll write Go code to calculate the hash code of each log file and then match it against the hash code listed in the digital signature file. 32 | 33 | To speed this process up, we'll calculate the digital signature of each file in a separate goroutine, spreading the work across all of the CPUs on our machine. 34 | 35 | ### Architecture Overview & Work Plan 36 | 37 | On the Python side of the code, we’re going to write a function named `check_signatures` and on the Go side, we’re going to write a function (that does the actual work) named `CheckSignatures`. In between these two functions, we’ll use the `ctypes` module (on the Python side) and write a `verify` function (on the Go side) to provide marshaling support. 38 | 39 | 40 | **Figure 1** 41 | ![](func-call.png) 42 | 43 | 44 | Figure 1 shows the flow of data from the Python function to the Go function and back. 45 | 46 | Here are the steps we’re going to follow for the rest of the post: 47 | 48 | * Write Go code (`CheckSignature`), 49 | * Exporting to the shared library (`verify`) 50 | * Use ctypes in the Python interactive prompt to call the Go code 51 | * Write and package the Python code (`check_signatures`) 52 | * We’ll do this part in the next blog post - this one is already long enough 53 | 54 | ### Go Code - The “CheckSignatures” Function 55 | 56 | I'm not going to break down all of the Go source code here, if you're curious to see all of it, look at this source code [file](https://github.com/ardanlabs/python-go/blob/master/pyext/checksig.go). 57 | 58 | The important part of the code to see now is the definition of the `CheckSignatures` function. 59 | 60 | **Listing 2: CheckSignatures function definition** 61 | ``` 62 | // CheckSignatures calculates sha1 signatures for files in rootDir and compare 63 | // them with signatures found at "sha1sum.txt" in the same directory. It'll 64 | // return an error if one of the signatures don't match 65 | func CheckSignatures(rootDir string) error { 66 | ``` 67 | 68 | Listing 2 shows the definition of the `CheckSignatures` function. This function will spin a goroutine per file to check if the calculated sha1 signature of any given file matches the one in “sha1sum.txt”. If there is a mismatch in one or more files, the function will return an error. 69 | 70 | ### Exporting Go Code to a Shared Library 71 | 72 | With the Go code written and tested, we can move on to exporting it to a shared library. 73 | 74 | Here are the steps we’ll follow in order to compile the Go source code into a shared library so Python can call it: 75 | 76 | * import the `C` package (aka [cgo](https://golang.org/cmd/cgo/)) 77 | * Use the `//export` directives on every function we need to expose 78 | * Have an empty `main` function 79 | * Build the source code with the special `-buildmode=c-shared` flag 80 | 81 | _Note: Apart from the Go toolchain, we’ll also need a C compiler (such as `gcc` on your machine). There’s a good free C compiler for each of the major platforms: `gcc` for Linux, `clang` on OSX (via [XCode](https://developer.apple.com/xcode/resources/)) and [Visual Studio](https://visualstudio.microsoft.com/vs/community/) for Windows_ 82 | 83 | 84 | **Listing 3: export.go** 85 | ``` 86 | 01 package main 87 | 02 88 | 03 import "C" 89 | 04 90 | 05 //export verify 91 | 06 func verify(root *C.char) *C.char { 92 | 07 rootDir := C.GoString(root) 93 | 08 if err := CheckSignatures(rootDir); err != nil { 94 | 09 return C.CString(err.Error()) 95 | 10 } 96 | 11 97 | 12 return nil 98 | 13 } 99 | 14 100 | 15 func main() {} 101 | ``` 102 | 103 | Listing 3 shows the `export.go` file from the project. We import “C” on line 03 and then on line 05, the `verify` function is marked to be exported in the shared library. It’s important that the comment is provided exactly as is. You can see on line 06, the `verify` function accepts a C based string pointer using the C package `char` type. For Go code to work with C strings, the C package provides a `GoString` function (which is used on line 07) and a `CString` function (which is used in line 09). Finally, an empty `main` function is declared at the end. 104 | 105 | To build the shared library, you need to run the `go build` command with a special flag. 106 | 107 | **Listing 4: Building the Shared Library** 108 | ``` 109 | $ go build -buildmode=c-shared -o _checksig.so 110 | ``` 111 | 112 | Listing 4 shows the command to generate the C based shared library which will be named `_checksig.so`. 113 | 114 | _Note: The reason for using `_` is to avoid name collision with the `checksig.py` Python module that we'll show later. If the shared library was named `checksig.so` then executing `import checksig` in Python will load the shared library instead of the Python file._ 115 | 116 | ### Preparing the Test Data 117 | 118 | Before we can try calling `verify` from Python, we need some data. You'll find a directory called [logs](https://github.com/ardanlabs/python-go/tree/master/pyext/testdata/logs) in the code repository. This directory contains some log files and a `sha1sum.txt` file. 119 | 120 | _Note: **The signature for `http08.log` is intentionally wrong.**_ 121 | 122 | On my machine, this directory is located at `/tmp/logs`. 123 | 124 | ### A Python Session 125 | 126 | I love the interactive shell in Python, it lets me play around with code in small chunks. After I have a working version, I write the Python code in a file. 127 | 128 | **Listing 5: A Python Session** 129 | ``` 130 | $ python 131 | Python 3.8.3 (default, May 17 2020, 18:15:42) 132 | [GCC 10.1.0] on linux 133 | Type "help", "copyright", "credits" or "license" for more information. 134 | 135 | 01 >>> import ctypes 136 | 02 >>> so = ctypes.cdll.LoadLibrary('./_checksig.so') 137 | 03 >>> verify = so.verify 138 | 04 >>> verify.argtypes = [ctypes.c_char_p] 139 | 05 >>> verify.restype = ctypes.c_void_p 140 | 06 >>> free = so.free 141 | 07 >>> free.argtypes = [ctypes.c_void_p] 142 | 08 >>> ptr = verify('/tmp/logs'.encode('utf-8')) 143 | 09 >>> out = ctypes.string_at(ptr) 144 | 10 >>> free(ptr) 145 | 11 >>> print(out.decode('utf-8')) 146 | 12 "/tmp/logs/httpd-08.log" - mismatch 147 | ``` 148 | 149 | Listing 6 shows an interactive Python session that walks you through testing the use of the exported to a shared library we wrote in Go. line 01 we import the `ctypes` module to start. Then on line 06, we load the shared library into memory. On lines 03-05 we load the `verify` function from the shared library and set the input and output types. Lines 06-07 load the `free` function so we can free the memory allocated by Go (see more below). 150 | 151 | Line 08 is the actual function call to `verify`. We need to convert the directory name to Python's `bytes` before passing it to the function. The return value, which is a C string, is stored in `ptr`. On line 09, we convert the C string to a Python `bytes` and on line 10 we free the memory allocated by Go. Finally, on line 11, we convert `out` from `bytes` to `str` before printing it. 152 | 153 | _Note: Line 02 assumes the shared library, `_checksig.so` is in the current directory. If you started the Python session elsewhere, change the path to the `_checksig.so` in line 02._ 154 | 155 | With very little effort we're able to call Go code from Python. 156 | 157 | ### Intermezzo: Sharing Memory Between Go & Python 158 | 159 | Both Python & Go have a garbage collector that will automatically free unused memory. However having a garbage collector doesn't mean you can't leak memory. 160 | 161 | _Note: You should read Bill's [Garbage Collection In Go](https://www.ardanlabs.com/blog/2018/12/garbage-collection-in-go-part1-semantics.html) blog posts. They will give you a good understanding on garbage collectors in general and on the Go garbage collector specifically._ 162 | 163 | You need to be extra careful when sharing memory between Go and Python (or C). Sometimes it's not clear when a memory allocation happens. In `export.go` on line 13, we have the following code: 164 | 165 | **Listing 6: Converting Go Error to C String** 166 | ``` 167 | str := C.CString(err.Error()) 168 | ``` 169 | 170 | The [documentation](https://golang.org/cmd/cgo/) for `C.String` says: 171 | 172 | **Listing 7** 173 | ``` 174 | > // Go string to C string 175 | > // The C string is allocated in the C heap using malloc. 176 | > // **It is the caller's responsibility to arrange for it to be 177 | > // freed**, such as by calling C.free (be sure to include stdlib.h 178 | > // if C.free is needed). 179 | > func C.CString(string) *C.char 180 | ``` 181 | 182 | To avoid a memory leak in our interactive prompt, we loaded the `free` function and used it to free the memory allocated by Go. 183 | 184 | ### Conclusion 185 | 186 | With very little code, you can use Go from Python. Unlike the previous installment, there's no RPC step - meaning you don't need to marshal and unmarshal parameters on every function call and there's no network involved as well. Calling from Python to C this way is much faster than a gRPC call. On the other hand, you need to be more careful with memory management and the packaging process is more complex. 187 | 188 | _Note: A simple [benchmark[(https://github.com/ardanlabs/python-go/tree/master/pyext/bench) on my machine clocks gRPC function call at 128µs vs a shared library call at 3.61µs - about 35 times faster._ 189 | 190 | I hope that you've found this style of writing code: Pure Go first, then exporting and then trying it out in an interactive session appealing. I urge you to try this workflow yourself next time you write some code. 191 | 192 | In the next installment, we’ll finish the last step of my workflow and package the Python code as a module. 193 | 194 | 195 | -------------------------------------------------------------------------------- /pyext/README-B.md: -------------------------------------------------------------------------------- 1 | # Go ↔ Python: Part III Packaging Python Code 2 | 3 | ### Introduction 4 | 5 | In [the previous post](https://www.ardanlabs.com/blog/2020/07/extending-python-with-go.html) we compiled Go code to a shared library and used it from the Python interactive shell. In this post we're going to finish the development process by writing a Python module that hides the low level details of working with a shared library and then package this code as a Python package. 6 | 7 | ### Recap - Architecture & Workflow Overview 8 | 9 | **Figure 1** 10 | ![](func-call.png) 11 | 12 | 13 | Figure 1 shows the flow of data from Python to Go and back. 14 | 15 | The workflow we're following is: 16 | 17 | * Write Go code (`CheckSignature`), 18 | * Export to the shared library (`verify`) 19 | * Use ctypes in the Python interactive prompt to call the Go code 20 | * Write and package the Python code (`check_signatures`) 21 | 22 | In the previous blog post we've done the first three parts and in this one we're going to implement the Python module and package it. We'll do this is the following steps: 23 | 24 | * Write the Python module (`checksig.py`) 25 | * Write the project definition file (`setup.py`) 26 | * Build the extension 27 | 28 | ### Python Module 29 | 30 | Let's start with writing a Python module. This module will have a Pythonic API and will hide the low level details of working with the shared library built from our Go code. 31 | 32 | **Listing 1: checksig.py** 33 | ``` 34 | 01 """Parallel check of files digital signature""" 35 | 02 36 | 03 import ctypes 37 | 04 from distutils.sysconfig import get_config_var 38 | 05 from pathlib import Path 39 | 06 40 | 07 # Location of shared library 41 | 08 here = Path(__file__).absolute().parent 42 | 09 ext_suffix = get_config_var('EXT_SUFFIX') 43 | 10 so_file = here / ('_checksig' + ext_suffix) 44 | 11 45 | 12 # Load functions from shared library set their signatures 46 | 13 so = ctypes.cdll.LoadLibrary(so_file) 47 | 14 verify = so.verify 48 | 15 verify.argtypes = [ctypes.c_char_p] 49 | 16 verify.restype = ctypes.c_void_p 50 | 17 free = so.free 51 | 18 free.argtypes = [ctypes.c_void_p] 52 | 19 53 | 20 54 | 21 def check_signatures(root_dir): 55 | 22 """Check (in parallel) digital signature of all files in root_dir. 56 | 23 We assume there's a sha1sum.txt file under root_dir 57 | 24 """ 58 | 25 res = verify(root_dir.encode('utf-8')) 59 | 26 if res is not None: 60 | 27 msg = ctypes.string_at(res).decode('utf-8') 61 | 28 free(res) 62 | 29 raise ValueError(msg) 63 | ``` 64 | 65 | Listing 1 has the code of our Python module. The code on lines 12-18 is very much what we did in the interactive prompt. 66 | 67 | Lines 7-10 deal with the shared library file name - we'll get to why we need that when we'll talk about packaging below. On lines 21-29, we define the API of our module - a single function called `check_signatures`. ctypes will convert C's `NULL` to Python's `None`, hence the `if` statement in line 26. On line 29, we signal an error by raising a `ValueError` exception. 68 | 69 | _Note: Python's naming conventions differ from Go. Most Python code is following the standard defined in [PEP-8](https://www.python.org/dev/peps/pep-0008/)._ 70 | 71 | ### Installing and Building Packages 72 | 73 | Before we move on to the last part of building a Python module, we need to take a detour and see how installing Python packages work. You can skip this part and head over to code below but I believe this section will help you understand **why** we write the code below. 74 | 75 | Here’s a simplified workflow of what’s Python’s [pip](https://pip.pypa.io/en/stable/) (the cousin of “go install” in the Go world) is doing when it’s installing a package: 76 | 77 | **Figure 2** 78 | ![](func-call.png) 79 | 80 | 81 | Figure 2 shows a simplified flow chart of installing a Python package. If there’s a pre-built binary package (wheel) matching the current OS/architecture it’ll use it. Otherwise, it’ll download the sources and will build the package. 82 | 83 | We roughly split Python packages to “pure” and “non-pure”. Pure packages are written only in Python while non-pure have code written in other languages and compiled to a shared library. Since non-pure packages contain binary code, they need to be built specially for an OS/architecture combination (say Linux/amd64). Our package is considered “non pure” since it contains code written in Go. 84 | 85 | We start by writing a file called `setup.py`, this file defines the project and contains instructions on how to build it. 86 | 87 | **Listing 2: setup.py** 88 | ``` 89 | 01 """Setup for checksig package""" 90 | 02 from distutils.errors import CompileError 91 | 03 from subprocess import call 92 | 04 93 | 05 from setuptools import Extension, setup 94 | 06 from setuptools.command.build_ext import build_ext 95 | 07 96 | 08 97 | 09 class build_go_ext(build_ext): 98 | 10 """Custom command to build extension from Go source files""" 99 | 11 def build_extension(self, ext): 100 | 12 ext_path = self.get_ext_fullpath(ext.name) 101 | 13 cmd = ['go', 'build', '-buildmode=c-shared', '-o', ext_path] 102 | 14 cmd += ext.sources 103 | 15 out = call(cmd) 104 | 16 if out != 0: 105 | 17 raise CompileError('Go build failed') 106 | 18 107 | 19 108 | 20 setup( 109 | 21 name='checksig', 110 | 22 version='0.1.0', 111 | 23 py_modules=['checksig'], 112 | 24 ext_modules=[ 113 | 25 Extension('_checksig', ['checksig.go', 'export.go']) 114 | 26 ], 115 | 27 cmdclass={'build_ext': build_go_ext}, 116 | 28 zip_safe=False, 117 | 29 ) 118 | ``` 119 | 120 | Listing 2 shows the `setup.py` file for our project. On line 09, we define a command to build an extension that uses the Go compiler. Python has built-in support for extensions written in C, C++ and [SWIG](http://www.swig.org/), but not for Go. 121 | 122 | Lines 12-14 define the command to run and line 15 runs this command as an external command (Python's `subprocess` is like Go’s `os/exec`). 123 | 124 | On line 20, we call the setup command, specifying the package name on line 21 and a version on line 22. On line 23, we define the Python module name and on lines 24-26 we define the extension module (the Go code). On line 27, we override the built-in 'build_ext` command with our `build_ext` command that builds Go code. On line 28, we specify the package is not zip safe since it contains a shared library. 125 | 126 | Another file we need to create is `MANIFEST.in`. It's a file that defines all the extra files that need to be packaged in the source distribution. 127 | 128 | **Listing 3: MANIFEST.in** 129 | ``` 130 | 01 include README.md 131 | 02 include *.go go.mod go.sum 132 | ``` 133 | 134 | Listing 3 shows the extra files that should be packaged in source distribution (`sdist`). 135 | 136 | Now we can build the Python packages. 137 | 138 | **Listing 4: Building the Python Packages** 139 | ``` 140 | $ python setup.py bdist_wheel 141 | $ python setup.py sdist 142 | ``` 143 | 144 | Listing 4 shows the command to build binary (`wheel`) package and the source (`sdist`) package files. 145 | 146 | The packages are built in a subdirectory called `dist`. 147 | 148 | **Listing 5: Content of `dist` Directory** 149 | ``` 150 | $ ls dist 151 | checksig-0.1.0-cp38-cp38-linux_x86_64.whl 152 | checksig-0.1.0.tar.gz 153 | ``` 154 | 155 | In Listing 5, we use the `ls` command to show the content of the `dist` directory. 156 | 157 | The wheel binary package (with `.whl` extension) has the platform information in its name: `cp38` means CPython version 3.8, `linux_x86_64` is the operation system and the architecture - same as Go's `GOOS` and `GOARCH`. Since the wheel file name changes depending on the architecture it’s built on, we had to write some logic in Listing 1 on lines 08-10. 158 | 159 | Now you can use Python's package manager, [pip](https://packaging.python.org/tutorials/installing-packages/) to install these packages. If you want to publish your package, you can upload it to the Python Package Index [PyPI](https://pypi.org/) using tools such as [twine](https://github.com/pypa/twine). 160 | 161 | See `example.py and `Dockerfile.test-b` in the [source repository](https://github.com/ardanlabs/python-go/tree/master/pyext) for a full build, install & use flow. 162 | 163 | ### Conclusion 164 | 165 | With little effort, you can extend Python using Go and expose a Python module that has a Pythonic API. Packaging is what makes your code deployable and valuable, don't skip this step. 166 | 167 | If you'd like to return Python types from Go (say a `list` or a `dict`), you can use Python's [extensive C API](https://docs.python.org/3/extending/index.html) with cgo. You can also have a look at the [go-python](https://github.com/sbinet/go-python) that can ease a lot of the pain of writing Python extensions in Go. 168 | 169 | In the next installment we're going to flip the roles again, we'll call Python from Go - in the same memory space and with almost zero serialization. 170 | -------------------------------------------------------------------------------- /pyext/README.md: -------------------------------------------------------------------------------- 1 | # Go ↔ Python: Extending Python with Go 2 | 3 | - [Part A](README-A.md) - Building the extension 4 | - [Part B](README-B.md) - Packaging 5 | -------------------------------------------------------------------------------- /pyext/bench/.gitignore: -------------------------------------------------------------------------------- 1 | bench.h 2 | bench_pb2_grpc.py 3 | bench_pb2.py 4 | bench.so 5 | pb 6 | server 7 | -------------------------------------------------------------------------------- /pyext/bench/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | $(error please pick a target) 3 | 4 | bench: 5 | ipython bench.ipy 6 | 7 | grpc: grpc-go grpc-python 8 | 9 | grpc-go: 10 | mkdir -p pb 11 | protoc \ 12 | --go_out=plugins=grpc:pb \ 13 | --go_opt=paths=source_relative \ 14 | bench.proto 15 | 16 | grpc-python: 17 | python -m grpc_tools.protoc \ 18 | -I. \ 19 | --python_out=. --grpc_python_out=. \ 20 | bench.proto 21 | 22 | so: 23 | go build -buildmode=c-shared -o bench.so so.go 24 | 25 | clean: 26 | rm -fr server bench.so bench.h 27 | rm -fr bench_pb2*.py 28 | -------------------------------------------------------------------------------- /pyext/bench/bench.ipy: -------------------------------------------------------------------------------- 1 | import ctypes 2 | from subprocess import Popen 3 | from time import sleep 4 | 5 | print('=== gRPC ===') 6 | !make grpc 7 | !go build -o server server.go 8 | srv = Popen(['./server']) 9 | sleep(1) # Let the server start 10 | %run client.py 11 | # Connect once 12 | chan = connect('localhost', 8888) 13 | %timeit call(chan) 14 | srv.kill() 15 | 16 | print('=== ctypes ===') 17 | !make so 18 | so = ctypes.cdll.LoadLibrary('./bench.so') 19 | fn = so.bench 20 | %timeit fn() 21 | -------------------------------------------------------------------------------- /pyext/bench/bench.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | option go_package = "github.com/ardanlabs/python-go/pyext/bench/pb"; 5 | 6 | message Empty { 7 | } 8 | 9 | service Bench { 10 | rpc Bench(Empty) returns (Empty) {} 11 | } 12 | -------------------------------------------------------------------------------- /pyext/bench/client.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | 3 | import bench_pb2 as pb 4 | import bench_pb2_grpc as gpb 5 | 6 | 7 | def call(chan): 8 | msg = pb.Empty() 9 | stub = gpb.BenchStub(chan) 10 | return stub.Bench(msg) 11 | 12 | 13 | def connect(host, port): 14 | return grpc.insecure_channel(f'{host}:{port}') 15 | -------------------------------------------------------------------------------- /pyext/bench/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ardanlabs/python-go/pyext/bench 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/golang/protobuf v1.4.2 7 | google.golang.org/grpc v1.30.0 8 | google.golang.org/protobuf v1.25.0 9 | ) 10 | -------------------------------------------------------------------------------- /pyext/bench/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 6 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 7 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 8 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 9 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 10 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 11 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 12 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 13 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 14 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 15 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 16 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 17 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 18 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 19 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 20 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 21 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 22 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 23 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 24 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 25 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 26 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 27 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 28 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 29 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 30 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 31 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 32 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 33 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 34 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 35 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 36 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 37 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= 38 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 39 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 40 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 41 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 43 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 44 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 45 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 46 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 47 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 48 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 49 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 50 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 51 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 52 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 53 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 54 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 55 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 56 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 57 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= 58 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 59 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 60 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 61 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 62 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 63 | google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= 64 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 65 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 66 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 67 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 68 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 69 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 70 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 71 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 72 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 73 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 74 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 75 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 76 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 77 | -------------------------------------------------------------------------------- /pyext/bench/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net" 7 | 8 | "google.golang.org/grpc" 9 | 10 | "github.com/ardanlabs/python-go/pyext/bench/pb" 11 | ) 12 | 13 | type BenchServer struct { 14 | pb.UnimplementedBenchServer 15 | } 16 | 17 | func (b *BenchServer) Bench(context.Context, *pb.Empty) (*pb.Empty, error) { 18 | return &pb.Empty{}, nil 19 | } 20 | 21 | func main() { 22 | addr := "localhost:8888" 23 | lis, err := net.Listen("tcp", addr) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | 28 | log.Printf("server listening on %s", addr) 29 | 30 | srv := grpc.NewServer() 31 | pb.RegisterBenchServer(srv, &BenchServer{}) 32 | srv.Serve(lis) 33 | } 34 | -------------------------------------------------------------------------------- /pyext/bench/so.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "C" 4 | 5 | //export bench 6 | func bench() {} 7 | 8 | func main() {} 9 | -------------------------------------------------------------------------------- /pyext/checksig.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "crypto/sha1" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path" 10 | "strings" 11 | 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | // CheckSignatures calculates sha1 signatures for files in rootDir and compare 16 | // them with signatures found at "sha1sum.txt" in the same directory. It'll 17 | // return an error if one of the signatures don't match 18 | func CheckSignatures(rootDir string) error { 19 | file, err := os.Open(path.Join(rootDir, "sha1sum.txt")) 20 | if err != nil { 21 | return err 22 | } 23 | defer file.Close() 24 | 25 | sigs, err := parseSigFile(file) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | var g errgroup.Group 31 | for name, signature := range sigs { 32 | fileName := path.Join(rootDir, name) 33 | expected := signature 34 | g.Go(func() error { 35 | sig, err := fileSig(fileName) 36 | if err != nil { 37 | return err 38 | } 39 | if sig != expected { 40 | return fmt.Errorf("%q - mismatch", fileName) 41 | } 42 | return nil 43 | }) 44 | } 45 | 46 | return g.Wait() 47 | } 48 | 49 | // fileSig returns the fileName sha1 digital signature of the specified file. 50 | func fileSig(fileName string) (string, error) { 51 | file, err := os.Open(fileName) 52 | if err != nil { 53 | return "", err 54 | } 55 | defer file.Close() 56 | 57 | hash := sha1.New() 58 | if _, err = io.Copy(hash, file); err != nil { 59 | return "", err 60 | } 61 | 62 | return fmt.Sprintf("%x", hash.Sum(nil)), nil 63 | } 64 | 65 | // parseSigFile parses the signature file and returns a map of path->signature. 66 | func parseSigFile(r io.Reader) (map[string]string, error) { 67 | sigs := make(map[string]string) 68 | scanner := bufio.NewScanner(r) 69 | lnum := 0 70 | 71 | for scanner.Scan() { 72 | lnum++ 73 | 74 | // Line example: 6c6427da7893932731901035edbb9214 nasa-00.log 75 | fields := strings.Fields(scanner.Text()) 76 | if len(fields) != 2 { 77 | return nil, fmt.Errorf("%d: bad line: %q", lnum, scanner.Text()) 78 | } 79 | sigs[fields[1]] = fields[0] 80 | } 81 | 82 | if err := scanner.Err(); err != nil { 83 | return nil, err 84 | } 85 | 86 | return sigs, nil 87 | } 88 | -------------------------------------------------------------------------------- /pyext/checksig.py: -------------------------------------------------------------------------------- 1 | """Parallel check of files digital signature""" 2 | 3 | import ctypes 4 | from distutils.sysconfig import get_config_var 5 | from pathlib import Path 6 | 7 | # Location of shared library 8 | here = Path(__file__).absolute().parent 9 | ext_suffix = get_config_var('EXT_SUFFIX') 10 | so_file = here / ('_checksig' + ext_suffix) 11 | 12 | # Load functions from shared library set their signatures 13 | so = ctypes.cdll.LoadLibrary(so_file) 14 | verify = so.verify 15 | verify.argtypes = [ctypes.c_char_p] 16 | verify.restype = ctypes.c_void_p 17 | free = so.free 18 | free.argtypes = [ctypes.c_void_p] 19 | 20 | 21 | def check_signatures(root_dir): 22 | """Check (in parallel) digital signature of all files in root_dir. 23 | We assume there's a sha1sum.txt file under root_dir 24 | """ 25 | res = verify(root_dir.encode('utf-8')) 26 | if res is not None: 27 | msg = ctypes.string_at(res).decode('utf-8') 28 | free(res) 29 | raise ValueError(msg) 30 | -------------------------------------------------------------------------------- /pyext/checksig_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestLogs(t *testing.T) { 8 | logsDir := "testdata/logs" 9 | if err := CheckSignatures(logsDir); err == nil { 10 | t.Fatalf("no error no %q", logsDir) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /pyext/example.py: -------------------------------------------------------------------------------- 1 | from checksig import check_signatures 2 | 3 | root_dir = '/tmp/logs' 4 | 5 | print(f'checking {root_dir!r}') 6 | try: 7 | check_signatures(root_dir) 8 | print('OK') 9 | except ValueError as err: 10 | print(f'error: {err}') 11 | -------------------------------------------------------------------------------- /pyext/export.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "C" 4 | 5 | //export verify 6 | func verify(root *C.char) *C.char { 7 | rootDir := C.GoString(root) 8 | if err := CheckSignatures(rootDir); err != nil { 9 | return C.CString(err.Error()) 10 | } 11 | 12 | return nil 13 | } 14 | 15 | func main() {} 16 | -------------------------------------------------------------------------------- /pyext/func-call.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/pyext/func-call.png -------------------------------------------------------------------------------- /pyext/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ardanlabs/python-go/pyext 2 | 3 | go 1.14 4 | 5 | require golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a 6 | -------------------------------------------------------------------------------- /pyext/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= 2 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /pyext/image.md: -------------------------------------------------------------------------------- 1 | The image was generated using https://excalidraw.com/ 2 | You can load func-call.json to it and edit the drawing. 3 | -------------------------------------------------------------------------------- /pyext/pip-install.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "excalidraw", 3 | "version": 2, 4 | "source": "https://excalidraw.com", 5 | "elements": [ 6 | { 7 | "id": "ozG8DxbGN10nI2EZWyxAU", 8 | "type": "text", 9 | "x": 315, 10 | "y": 119, 11 | "width": 387, 12 | "height": 24, 13 | "angle": 0, 14 | "strokeColor": "#000000", 15 | "backgroundColor": "transparent", 16 | "fillStyle": "hachure", 17 | "strokeWidth": 1, 18 | "strokeStyle": "solid", 19 | "roughness": 1, 20 | "opacity": 100, 21 | "groupIds": [], 22 | "seed": 862599620, 23 | "version": 105, 24 | "versionNonce": 494293700, 25 | "isDeleted": false, 26 | "text": "$ python -m pip install checksig ", 27 | "fontSize": 20, 28 | "fontFamily": 3, 29 | "textAlign": "left", 30 | "verticalAlign": "top", 31 | "baseline": 19 32 | }, 33 | { 34 | "id": "w0HAkfFikFWqV9tWgIuiu", 35 | "type": "diamond", 36 | "x": 355, 37 | "y": 169, 38 | "width": 297.00000000000006, 39 | "height": 257, 40 | "angle": 0, 41 | "strokeColor": "#000000", 42 | "backgroundColor": "transparent", 43 | "fillStyle": "hachure", 44 | "strokeWidth": 1, 45 | "strokeStyle": "solid", 46 | "roughness": 0, 47 | "opacity": 100, 48 | "groupIds": [], 49 | "seed": 1019284292, 50 | "version": 180, 51 | "versionNonce": 1412251460, 52 | "isDeleted": false 53 | }, 54 | { 55 | "id": "MpQOiGLe-8OnfM3PMJFFt", 56 | "type": "text", 57 | "x": 391.5, 58 | "y": 267, 59 | "width": 230, 60 | "height": 75, 61 | "angle": 0, 62 | "strokeColor": "#000000", 63 | "backgroundColor": "transparent", 64 | "fillStyle": "hachure", 65 | "strokeWidth": 1, 66 | "strokeStyle": "solid", 67 | "roughness": 1, 68 | "opacity": 100, 69 | "groupIds": [], 70 | "seed": 1972304708, 71 | "version": 111, 72 | "versionNonce": 687574340, 73 | "isDeleted": false, 74 | "text": "Is there a compiled \nbinary (wheel) for this \nplatform on PyPI?", 75 | "fontSize": 20, 76 | "fontFamily": 1, 77 | "textAlign": "center", 78 | "verticalAlign": "middle", 79 | "baseline": 68 80 | }, 81 | { 82 | "id": "rjzMXlAuvYqBXiWoX8ruo", 83 | "type": "arrow", 84 | "x": 504, 85 | "y": 425, 86 | "width": 1, 87 | "height": 101, 88 | "angle": 0, 89 | "strokeColor": "#000000", 90 | "backgroundColor": "transparent", 91 | "fillStyle": "hachure", 92 | "strokeWidth": 1, 93 | "strokeStyle": "solid", 94 | "roughness": 0, 95 | "opacity": 100, 96 | "groupIds": [], 97 | "seed": 1582085828, 98 | "version": 69, 99 | "versionNonce": 1985791428, 100 | "isDeleted": false, 101 | "points": [ 102 | [ 103 | 0, 104 | 0 105 | ], 106 | [ 107 | -1, 108 | 101 109 | ] 110 | ], 111 | "lastCommittedPoint": null 112 | }, 113 | { 114 | "id": "VtXSRQPo3a1DeBI9jEJTj", 115 | "type": "text", 116 | "x": 671, 117 | "y": 264, 118 | "width": 24, 119 | "height": 25, 120 | "angle": 0, 121 | "strokeColor": "#364fc7", 122 | "backgroundColor": "transparent", 123 | "fillStyle": "hachure", 124 | "strokeWidth": 1, 125 | "strokeStyle": "solid", 126 | "roughness": 0, 127 | "opacity": 100, 128 | "groupIds": [], 129 | "seed": 1057908860, 130 | "version": 80, 131 | "versionNonce": 1714071876, 132 | "isDeleted": false, 133 | "text": "No", 134 | "fontSize": 20, 135 | "fontFamily": 1, 136 | "textAlign": "left", 137 | "verticalAlign": "top", 138 | "baseline": 18 139 | }, 140 | { 141 | "id": "nuVwqEnX52ISc0VfKYpBf", 142 | "type": "rectangle", 143 | "x": 394, 144 | "y": 536, 145 | "width": 282.99999999999994, 146 | "height": 57, 147 | "angle": 0, 148 | "strokeColor": "#000000", 149 | "backgroundColor": "transparent", 150 | "fillStyle": "hachure", 151 | "strokeWidth": 1, 152 | "strokeStyle": "solid", 153 | "roughness": 0, 154 | "opacity": 100, 155 | "groupIds": [], 156 | "seed": 406334972, 157 | "version": 103, 158 | "versionNonce": 1128378308, 159 | "isDeleted": false 160 | }, 161 | { 162 | "id": "RTHu8yYm9muVx2IF6Xn6G", 163 | "type": "text", 164 | "x": 402.63999999999976, 165 | "y": 542, 166 | "width": 267, 167 | "height": 50, 168 | "angle": 0, 169 | "strokeColor": "#000000", 170 | "backgroundColor": "transparent", 171 | "fillStyle": "hachure", 172 | "strokeWidth": 1, 173 | "strokeStyle": "solid", 174 | "roughness": 0, 175 | "opacity": 100, 176 | "groupIds": [], 177 | "seed": 665074812, 178 | "version": 178, 179 | "versionNonce": 2120313156, 180 | "isDeleted": false, 181 | "text": "- Download binary package\n- Install it", 182 | "fontSize": 20, 183 | "fontFamily": 1, 184 | "textAlign": "left", 185 | "verticalAlign": "top", 186 | "baseline": 43 187 | }, 188 | { 189 | "id": "1XGg__b38Bv9pTKc1RUbd", 190 | "type": "arrow", 191 | "x": 652, 192 | "y": 297, 193 | "width": 96, 194 | "height": 2, 195 | "angle": 0, 196 | "strokeColor": "#000000", 197 | "backgroundColor": "transparent", 198 | "fillStyle": "hachure", 199 | "strokeWidth": 1, 200 | "strokeStyle": "solid", 201 | "roughness": 0, 202 | "opacity": 100, 203 | "groupIds": [], 204 | "seed": 1807479420, 205 | "version": 35, 206 | "versionNonce": 2091698940, 207 | "isDeleted": false, 208 | "points": [ 209 | [ 210 | 0, 211 | 0 212 | ], 213 | [ 214 | 96, 215 | 2 216 | ] 217 | ], 218 | "lastCommittedPoint": null 219 | }, 220 | { 221 | "id": "_x01I9DB1F37bkuKUrCcH", 222 | "type": "text", 223 | "x": 456, 224 | "y": 435.5, 225 | "width": 30, 226 | "height": 25, 227 | "angle": 0, 228 | "strokeColor": "#364fc7", 229 | "backgroundColor": "transparent", 230 | "fillStyle": "hachure", 231 | "strokeWidth": 1, 232 | "strokeStyle": "solid", 233 | "roughness": 0, 234 | "opacity": 100, 235 | "groupIds": [], 236 | "seed": 1570027844, 237 | "version": 22, 238 | "versionNonce": 1464994684, 239 | "isDeleted": false, 240 | "text": "Yes", 241 | "fontSize": 20, 242 | "fontFamily": 1, 243 | "textAlign": "left", 244 | "verticalAlign": "top", 245 | "baseline": 18 246 | }, 247 | { 248 | "id": "fZyJBelOhtr7dC8cNpHf0", 249 | "type": "rectangle", 250 | "x": 754, 251 | "y": 277, 252 | "width": 325.9999999999999, 253 | "height": 82.99999999999999, 254 | "angle": 0, 255 | "strokeColor": "#000000", 256 | "backgroundColor": "transparent", 257 | "fillStyle": "hachure", 258 | "strokeWidth": 1, 259 | "strokeStyle": "solid", 260 | "roughness": 0, 261 | "opacity": 100, 262 | "groupIds": [], 263 | "seed": 1411865156, 264 | "version": 155, 265 | "versionNonce": 87938116, 266 | "isDeleted": false 267 | }, 268 | { 269 | "id": "F4-npIVYKsLxvJc_zhgEA", 270 | "type": "text", 271 | "x": 766, 272 | "y": 282.5, 273 | "width": 303, 274 | "height": 75, 275 | "angle": 0, 276 | "strokeColor": "#000000", 277 | "backgroundColor": "transparent", 278 | "fillStyle": "hachure", 279 | "strokeWidth": 1, 280 | "strokeStyle": "solid", 281 | "roughness": 0, 282 | "opacity": 100, 283 | "groupIds": [], 284 | "seed": 2079558724, 285 | "version": 185, 286 | "versionNonce": 3683068, 287 | "isDeleted": false, 288 | "text": "- Download source distribution\n- Build it (using Go)\n- Install it", 289 | "fontSize": 20, 290 | "fontFamily": 1, 291 | "textAlign": "left", 292 | "verticalAlign": "top", 293 | "baseline": 68 294 | }, 295 | { 296 | "id": "MVWy0HEHsDFlWZ_ifxzkc", 297 | "type": "text", 298 | "x": 761, 299 | "y": 442, 300 | "width": 457, 301 | "height": 48, 302 | "angle": 0, 303 | "strokeColor": "#000000", 304 | "backgroundColor": "transparent", 305 | "fillStyle": "hachure", 306 | "strokeWidth": 1, 307 | "strokeStyle": "solid", 308 | "roughness": 0, 309 | "opacity": 100, 310 | "groupIds": [], 311 | "seed": 2106392316, 312 | "version": 170, 313 | "versionNonce": 1341426940, 314 | "isDeleted": false, 315 | "text": ">>> from checksig import check_signaure\n>>> check_signature('/tmp/logs')", 316 | "fontSize": 20, 317 | "fontFamily": 3, 318 | "textAlign": "left", 319 | "verticalAlign": "top", 320 | "baseline": 43 321 | }, 322 | { 323 | "id": "bWJLTcAg1Jp8TQYmEEvzb", 324 | "type": "line", 325 | "x": 679, 326 | "y": 562, 327 | "width": 245, 328 | "height": 1, 329 | "angle": 0, 330 | "strokeColor": "#000000", 331 | "backgroundColor": "transparent", 332 | "fillStyle": "hachure", 333 | "strokeWidth": 1, 334 | "strokeStyle": "solid", 335 | "roughness": 0, 336 | "opacity": 100, 337 | "groupIds": [], 338 | "seed": 1092866244, 339 | "version": 62, 340 | "versionNonce": 1581550716, 341 | "isDeleted": false, 342 | "points": [ 343 | [ 344 | 0, 345 | 0 346 | ], 347 | [ 348 | 245, 349 | -1 350 | ] 351 | ], 352 | "lastCommittedPoint": null 353 | }, 354 | { 355 | "id": "ekuoX-z0VOzfAAXs1fSp7", 356 | "type": "arrow", 357 | "x": 923, 358 | "y": 561, 359 | "width": 2, 360 | "height": 59, 361 | "angle": 0, 362 | "strokeColor": "#000000", 363 | "backgroundColor": "transparent", 364 | "fillStyle": "hachure", 365 | "strokeWidth": 1, 366 | "strokeStyle": "solid", 367 | "roughness": 0, 368 | "opacity": 100, 369 | "groupIds": [], 370 | "seed": 2138409284, 371 | "version": 35, 372 | "versionNonce": 268417220, 373 | "isDeleted": false, 374 | "points": [ 375 | [ 376 | 0, 377 | 0 378 | ], 379 | [ 380 | 2, 381 | -59 382 | ] 383 | ], 384 | "lastCommittedPoint": null 385 | }, 386 | { 387 | "id": "xnG8Q__QxKzc_7GXKjeEn", 388 | "type": "arrow", 389 | "x": 908, 390 | "y": 361, 391 | "width": 3, 392 | "height": 71, 393 | "angle": 0, 394 | "strokeColor": "#000000", 395 | "backgroundColor": "transparent", 396 | "fillStyle": "hachure", 397 | "strokeWidth": 1, 398 | "strokeStyle": "solid", 399 | "roughness": 0, 400 | "opacity": 100, 401 | "groupIds": [], 402 | "seed": 998778236, 403 | "version": 36, 404 | "versionNonce": 44650436, 405 | "isDeleted": false, 406 | "points": [ 407 | [ 408 | 0, 409 | 0 410 | ], 411 | [ 412 | 3, 413 | 71 414 | ] 415 | ], 416 | "lastCommittedPoint": null 417 | } 418 | ], 419 | "appState": { 420 | "viewBackgroundColor": "#ffffff", 421 | "gridSize": null 422 | } 423 | } -------------------------------------------------------------------------------- /pyext/pip-install.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christian-korneck/python-go/b9bf06c9e65d60d1026dceb47dcdf817d11fc68e/pyext/pip-install.png -------------------------------------------------------------------------------- /pyext/py_session.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | so = ctypes.cdll.LoadLibrary('./_checksig.so') 3 | verify = so.verify 4 | verify.argtypes = [ctypes.c_char_p] 5 | verify.restype = ctypes.c_void_p 6 | free = so.free 7 | free.argtypes = [ctypes.c_void_p] 8 | ptr = verify('/tmp/logs'.encode('utf-8')) 9 | out = ctypes.string_at(ptr) 10 | free(ptr) 11 | print(out.decode('utf-8')) 12 | # "/tmp/logs/httpd-08.log" - mismatch 13 | -------------------------------------------------------------------------------- /pyext/setup.py: -------------------------------------------------------------------------------- 1 | """Setup for checksig package""" 2 | from distutils.errors import CompileError 3 | from subprocess import call 4 | 5 | from setuptools import Extension, setup 6 | from setuptools.command.build_ext import build_ext 7 | 8 | 9 | class build_go_ext(build_ext): 10 | """Custom command to build extension from Go source files""" 11 | def build_extension(self, ext): 12 | ext_path = self.get_ext_fullpath(ext.name) 13 | cmd = ['go', 'build', '-buildmode=c-shared', '-o', ext_path] 14 | cmd += ext.sources 15 | out = call(cmd) 16 | if out != 0: 17 | raise CompileError('Go build failed') 18 | 19 | 20 | setup( 21 | name='checksig', 22 | version='0.1.0', 23 | py_modules=['checksig'], 24 | ext_modules=[ 25 | Extension('_checksig', ['checksig.go', 'export.go']) 26 | ], 27 | cmdclass={'build_ext': build_go_ext}, 28 | zip_safe=False, 29 | ) 30 | -------------------------------------------------------------------------------- /pyext/test_checksig.py: -------------------------------------------------------------------------------- 1 | from checksig import check_signatures 2 | 3 | from unittest import TestCase 4 | 5 | 6 | class TestCheckSignatures(TestCase): 7 | def test_logs(self): 8 | logs_dir = 'testdata/logs' 9 | with self.assertRaises(ValueError): 10 | check_signatures(logs_dir) 11 | -------------------------------------------------------------------------------- /pyext/testdata/logs/sha1sum.txt: -------------------------------------------------------------------------------- 1 | 6659cb84ab403dc85962fc77b9156924bbbaab2c httpd-00.log 2 | 5693325790ee53629d6ed3264760c4463a3615ee httpd-01.log 3 | fce486edf5251951c7b92a3d5098ea6400bfd63f httpd-02.log 4 | b5b04eb809e9c737dbb5de76576019e9db1958fd httpd-03.log 5 | ff0e3f644371d0fbce954dace6f678f9f77c3e08 httpd-04.log 6 | c154b2aa27122c07da77b85165036906fb6cbc3c httpd-05.log 7 | 28fccd72fb6fe88e1665a15df397c1d207de94ef httpd-06.log 8 | 86ed10cd87ac6f9fb62f6c29e82365c614089ae8 httpd-07.log 9 | feaf526473cb2887781f4904bd26f021a91ee9eb httpd-08.log 10 | 330d03af58919dd12b32804d9742b55c7ed16038 httpd-09.log 11 | --------------------------------------------------------------------------------