├── tests ├── .gitignore ├── test-conf │ ├── .htpasswd │ ├── cpp_credentials_file.json │ ├── public-key.client-ecdsa.pem │ ├── public-key.client-rsa.pem │ ├── public-key.client-mismatch-rsa.pem │ ├── private-key.client-ecdsa.pem │ ├── client.conf │ ├── client-ssl.conf │ ├── client-key.pem │ ├── broker-key.pem │ ├── private-key.client-mismatch-rsa.pem │ ├── private-key.client-rsa.pem │ ├── client-cert.pem │ ├── broker-cert.pem │ └── cacert.pem ├── run-unit-tests.sh ├── debug_logger_test.py ├── interrupted_test.py ├── reader_test.py ├── asyncio_test.py ├── custom_logger_test.py ├── oauth2_test.py └── table_view_test.py ├── .gitmodules ├── NOTICE ├── .gitignore ├── pkg ├── manylinux │ ├── pulsar-client-cpp-3.7.0.patch │ └── Dockerfile ├── test-wheel.sh ├── manylinux_musl │ └── Dockerfile ├── build-wheel-inside-docker.sh └── mac │ └── build-mac-wheels.sh ├── examples ├── company.avsc ├── consumer.py ├── producer.py ├── rpc_server.py └── rpc_client.py ├── pulsar ├── __about__.py ├── functions │ ├── __init__.py │ ├── function.py │ ├── serde.py │ └── context.py ├── schema │ ├── __init__.py │ ├── schema.py │ └── schema_avro.py ├── exceptions.py ├── tableview.py └── asyncio.py ├── dependencies.yaml ├── .clang-format ├── CONTRIBUTING.md ├── src ├── schema.cc ├── pulsar.cc ├── utils.cc ├── reader.cc ├── utils.h ├── authentication.cc ├── table_view.cc ├── exceptions.h ├── producer.cc ├── client.cc ├── consumer.cc ├── message.cc ├── enums.cc └── exceptions.cc ├── SECURITY.md ├── .asf.yaml ├── .github └── workflows │ ├── codeql.yml │ ├── ci-build-release-wheels.yaml │ └── ci-pr-validation.yaml ├── setup.py ├── cmake_modules └── FindClangTools.cmake ├── CMakeLists.txt └── README.md /tests/.gitignore: -------------------------------------------------------------------------------- 1 | .test-token.txt -------------------------------------------------------------------------------- /tests/test-conf/.htpasswd: -------------------------------------------------------------------------------- 1 | admin:$apr1$FG4AO6aX$KGYPuMoLUou3i6vUkPUUf. 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pybind11"] 2 | path = pybind11 3 | url = https://github.com/pybind/pybind11.git 4 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Pulsar 2 | Copyright 2017-2022 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /tests/test-conf/cpp_credentials_file.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", 3 | "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" 4 | } 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | MANIFEST 2 | build 3 | dist 4 | *.egg-info 5 | .idea 6 | CMakeCache.txt 7 | CMakeFiles 8 | Makefile 9 | _pulsar.so 10 | cmake_install.cmake 11 | __pycache__ 12 | .build 13 | .pulsar-mac-wheels-cache 14 | .DS_Store 15 | wheelhouse 16 | .pulsar-mac-build 17 | vcpkg_installed/ 18 | *.pyd 19 | *.lib 20 | 21 | 22 | lib_pulsar.so 23 | tests/test.log 24 | .tests-container-id.txt 25 | 26 | -------------------------------------------------------------------------------- /tests/test-conf/public-key.client-ecdsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIHKMIGjBgcqhkjOPQIBMIGXAgEBMBwGByqGSM49AQECEQD////9//////////// 3 | ////MDsEEP////3///////////////wEEOh1ecEQefQ92CSZPCzuXtMDFQAADg1N 4 | aW5naHVhUXUMwDpEc9A2eQQhBBYf91KLiZstDChgfKUsW4bPWsg5W6/rE8AtopLd 5 | 7XqDAhEA/////gAAAAB1ow0bkDihFQIBAQMiAATrKj6RPHGPNKcZKIOccN4tgENM 6 | n1dzKjLrMZTkJ4oAaQ== 7 | -----END PUBLIC KEY----- 8 | -------------------------------------------------------------------------------- /tests/test-conf/public-key.client-rsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtKWwgqdnTYrOCv+j1MkT 3 | WfSH0wCsHZZca9wAW3qP4uuhlBvnb10JcFf5ZjzP9BSXK+tHmI8uoN368vEv6yhU 4 | RHM4yuXqzCxzuAwkQSo39rzX8PGC7qdjCN7LDJ3MnqiBIrUsSaEP1wrNsB1kI+o9 5 | ER1e5O/uEPAotP933hHQ0J2hMEekHqL7sBlJ98h6NmsicEaUkardk0TOXrlkjC+c 6 | Md8ZbGScPqI9M38bmn3OLxFTn1vthpvnXLvCmG4M+6xtYtD+npcVPZw1i1R90fMs 7 | 7ppZnRbv8Hc/DFdOKVQIgam6CDdnNKgW7c7IBMrP0AEm37HTu0LSOjP2OHXlvvlQ 8 | GQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /tests/test-conf/public-key.client-mismatch-rsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtKWwgqdnTYrOCv+j1MkT 3 | WfSH0wCsHZZca9wAW3qP4uuhlBvnb10JcFf5ZjzP9BSXK+tHmI8uoN368vEv6yhU 4 | RHM4yuXqzCxzuAwkQSo39rzX8PGC7qdjCN7LDJ3MnqiBIrUsSaEP1wrNsB1kI+o9 5 | ER1e5O/uEPAotP933hHQ0J2hMEekHqL7sBlJ98h6NmsicEaUkardk0TOXrlkjC+c 6 | Md8ZbGScPqI9M38bmn3OLxFTn1vthpvnXLvCmG4M+6xtYtD+npcVPZw1i1R90fMs 7 | 7ppZnRbv8Hc/DFdOKVQIgam6CDdnNKgW7c7IBMrP0AEm37HTu0LSOjP2OHXlvvlQ 8 | GQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /pkg/manylinux/pulsar-client-cpp-3.7.0.patch: -------------------------------------------------------------------------------- 1 | --- lib/CMakeLists.txt 2 | +++ lib/CMakeLists.txt 3 | @@ -93,10 +93,6 @@ 4 | if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 5 | target_link_options(pulsarShared PRIVATE -Wl,-Bsymbolic) 6 | endif () 7 | - check_cxx_symbol_exists(__GLIBCXX__ iostream GLIBCXX) 8 | - if (GLIBCXX) 9 | - target_link_libraries(pulsarShared PUBLIC -static-libgcc -static-libstdc++) 10 | - endif () 11 | endif() 12 | 13 | check_cxx_symbol_exists(getauxval sys/auxv.h HAVE_AUXV_GETAUXVAL) 14 | -------------------------------------------------------------------------------- /tests/test-conf/private-key.client-ecdsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | MIGXAgEBMBwGByqGSM49AQECEQD////9////////////////MDsEEP////3///// 3 | //////////wEEOh1ecEQefQ92CSZPCzuXtMDFQAADg1NaW5naHVhUXUMwDpEc9A2 4 | eQQhBBYf91KLiZstDChgfKUsW4bPWsg5W6/rE8AtopLd7XqDAhEA/////gAAAAB1 5 | ow0bkDihFQIBAQ== 6 | -----END EC PARAMETERS----- 7 | -----BEGIN EC PRIVATE KEY----- 8 | MIHYAgEBBBDeu9hc8kOvL3pl+LYSjLq9oIGaMIGXAgEBMBwGByqGSM49AQECEQD/ 9 | ///9////////////////MDsEEP////3///////////////wEEOh1ecEQefQ92CSZ 10 | PCzuXtMDFQAADg1NaW5naHVhUXUMwDpEc9A2eQQhBBYf91KLiZstDChgfKUsW4bP 11 | Wsg5W6/rE8AtopLd7XqDAhEA/////gAAAAB1ow0bkDihFQIBAaEkAyIABOsqPpE8 12 | cY80pxkog5xw3i2AQ0yfV3MqMusxlOQnigBp 13 | -----END EC PRIVATE KEY----- 14 | -------------------------------------------------------------------------------- /examples/company.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "doc": "this is doc", 3 | "namespace": "example.avro", 4 | "type": "record", 5 | "name": "Company", 6 | "fields": [ 7 | {"name": "name", "type": ["null", "string"]}, 8 | {"name": "address", "type": ["null", "string"]}, 9 | {"name": "employees", "type": ["null", {"type": "array", "items": { 10 | "type": "record", 11 | "name": "Employee", 12 | "fields": [ 13 | {"name": "name", "type": ["null", "string"]}, 14 | {"name": "age", "type": ["null", "int"]} 15 | ] 16 | }}]}, 17 | {"name": "labels", "type": ["null", {"type": "map", "values": "string"}]}, 18 | {"name": "companyType", "type": ["null", {"type": "enum", "name": "CompanyType", "symbols": 19 | ["companyType1", "companyType2", "companyType3"]}]} 20 | ] 21 | } -------------------------------------------------------------------------------- /pulsar/__about__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | __version__='3.9.0a1' 20 | -------------------------------------------------------------------------------- /pulsar/functions/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # -*- encoding: utf-8 -*- 21 | -------------------------------------------------------------------------------- /dependencies.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | pulsar-cpp: 4.0.0 21 | pybind11: 3.0.1 22 | # The OpenSSL dependency is only used when building Python from source 23 | openssl: 1.1.1q 24 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | 19 | BasedOnStyle: Google 20 | IndentWidth: 4 21 | ColumnLimit: 110 22 | SortIncludes: false 23 | BreakBeforeBraces: Custom 24 | BraceWrapping: 25 | AfterEnum: true 26 | -------------------------------------------------------------------------------- /pkg/test-wheel.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | set -e -x 22 | 23 | cd / 24 | 25 | pip3 install /pulsar-client-python/wheelhouse/pulsar_client-*.whl 26 | 27 | # Load the wheel to ensure there are no linking problems 28 | python3 -c 'import pulsar' 29 | -------------------------------------------------------------------------------- /tests/test-conf/client.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # Pulsar Client configuration 21 | webServiceUrl=http://localhost:8765/ 22 | brokerServiceUrl=pulsar://localhost:8885/ 23 | #authPlugin= 24 | #authParams= 25 | #useTls= 26 | #tlsAllowInsecureConnection 27 | #tlsTrustCertsFilePath 28 | -------------------------------------------------------------------------------- /pulsar/schema/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from .definition import Record, Field, Null, Boolean, Integer, Long, \ 21 | Float, Double, Bytes, String, Array, Map, CustomEnum 22 | 23 | from .schema import Schema, BytesSchema, StringSchema, JsonSchema 24 | from .schema_avro import AvroSchema 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 21 | 22 | ## Contributing to Apache Pulsar 23 | 24 | We would love for you to contribute to Apache Pulsar and make it even better! 25 | Please check the [Contributing to Apache Pulsar](https://pulsar.apache.org/contributing/) 26 | page before starting to work on the project. 27 | -------------------------------------------------------------------------------- /pkg/manylinux_musl/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | ARG ARCH 21 | FROM quay.io/pypa/musllinux_1_2_${ARCH} 22 | 23 | ARG PYTHON_VERSION 24 | ARG PYTHON_SPEC 25 | 26 | ENV PYTHON_VERSION=${PYTHON_VERSION} 27 | ENV PYTHON_SPEC=${PYTHON_SPEC} 28 | 29 | ARG ARCH 30 | ENV ARCH=${ARCH} 31 | 32 | ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" 33 | 34 | RUN pip install setuptools 35 | ENV CPP_BINARY_TYPE="apk" 36 | -------------------------------------------------------------------------------- /tests/run-unit-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | set -e -x 22 | 23 | ROOT_DIR=$(git rev-parse --show-toplevel) 24 | cd $ROOT_DIR/tests 25 | 26 | python3 custom_logger_test.py 27 | python3 debug_logger_test.py 28 | python3 interrupted_test.py 29 | python3 pulsar_test.py 30 | python3 schema_test.py 31 | python3 table_view_test.py 32 | python3 reader_test.py 33 | python3 asyncio_test.py 34 | -------------------------------------------------------------------------------- /tests/test-conf/client-ssl.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # Pulsar Client configuration 21 | webServiceUrl=https://localhost:8443/ 22 | brokerServiceUrl=pulsar+ssl://localhost:6651/ 23 | tlsAllowInsecureConnection=false 24 | tlsTrustCertsFilePath=test-conf/cacert.pem 25 | authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls 26 | authParams=tlsCertFile:test-conf/client-cert.pem,tlsKeyFile:test-conf/client-key.pem 27 | -------------------------------------------------------------------------------- /pkg/manylinux/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | ARG ARCH 21 | FROM quay.io/pypa/manylinux_2_28_${ARCH} 22 | 23 | ARG PYTHON_VERSION 24 | ARG PYTHON_SPEC 25 | ARG ARCH 26 | 27 | ENV CPP_BINARY_TYPE=rpm 28 | ENV PYTHON_VERSION=${PYTHON_VERSION} 29 | ENV PYTHON_SPEC=${PYTHON_SPEC} 30 | 31 | ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" 32 | ENV ARCH=${ARCH} 33 | 34 | RUN pip3 install setuptools 35 | # Dependencies for vcpkg on arm64 architecture 36 | RUN yum install -y curl zip unzip tar perl-IPC-Cmd 37 | -------------------------------------------------------------------------------- /src/schema.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | #include 21 | 22 | namespace py = pybind11; 23 | 24 | void export_schema(py::module_& m) { 25 | using namespace py; 26 | 27 | class_(m, "SchemaInfo") 28 | .def(init()) 29 | .def("schema_type", &SchemaInfo::getSchemaType) 30 | .def("name", &SchemaInfo::getName, return_value_policy::copy) 31 | .def("schema", &SchemaInfo::getSchema, return_value_policy::copy); 32 | } 33 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Security Vulnerability Process 4 | 5 | The Pulsar community follows the ASF [security vulnerability handling process](https://apache.org/security/#vulnerability-handling). 6 | 7 | To report a new vulnerability you have discovered, please follow the [ASF security vulnerability reporting process](https://apache.org/security/#reporting-a-vulnerability). To report a vulnerability for Pulsar, contact the [Apache Security Team](https://www.apache.org/security/). When reporting a vulnerability to [security@apache.org](mailto:security@apache.org), you can copy your email to [private@pulsar.apache.org](mailto:private@pulsar.apache.org) to send your report to the Apache Pulsar Project Management Committee. This is a private mailing list. 8 | 9 | It is the responsibility of the security vulnerability handling project team (Apache Pulsar PMC in most cases) to make public security vulnerability announcements. You can follow announcements on the [users@pulsar.apache.org](mailto:users@pulsar.apache.org) mailing list. For instructions on how to subscribe, please see https://pulsar.apache.org/contact/. 10 | 11 | ## Security Policy details and supported versions of Apache Pulsar 12 | 13 | The security policy and supported versions are outlined on the Pulsar website under [Security > Security Policy and Supported Versions](https://pulsar.apache.org/docs/security-policy-and-supported-versions/). 14 | 15 | ## Security Advisories 16 | 17 | Please visit the [Security Advisories](https://github.com/apache/pulsar/wiki/Security-advisories) page. -------------------------------------------------------------------------------- /examples/consumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | 22 | import pulsar 23 | 24 | client = pulsar.Client('pulsar://localhost:6650') 25 | consumer = client.subscribe('my-topic', "my-subscription", 26 | properties={ 27 | "consumer-name": "test-consumer-name", 28 | "consumer-id": "test-consumer-id" 29 | }) 30 | 31 | while True: 32 | try: 33 | msg = consumer.receive() 34 | print("Received message '{0}' id='{1}'".format(msg.data().decode('utf-8'), msg.message_id())) 35 | consumer.acknowledge(msg) 36 | except pulsar.Interrupted: 37 | print("Stop receiving messages") 38 | break 39 | 40 | client.close() 41 | -------------------------------------------------------------------------------- /examples/producer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from pulsar import BatchingType 22 | import pulsar 23 | 24 | 25 | client = pulsar.Client('pulsar://localhost:6650') 26 | 27 | producer = client.create_producer( 28 | 'my-topic', 29 | block_if_queue_full=True, 30 | batching_enabled=True, 31 | batching_max_publish_delay_ms=10, 32 | properties={ 33 | "producer-name": "test-producer-name", 34 | "producer-id": "test-producer-id" 35 | }, 36 | batching_type=BatchingType.KeyBased 37 | ) 38 | 39 | for i in range(10): 40 | try: 41 | producer.send('hello'.encode('utf-8'), None) 42 | except Exception as e: 43 | print("Failed to send message: %s", e) 44 | 45 | producer.flush() 46 | producer.close() 47 | -------------------------------------------------------------------------------- /src/pulsar.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | #include 21 | namespace py = pybind11; 22 | 23 | using Module = py::module_; 24 | 25 | void export_client(Module& m); 26 | void export_message(Module& m); 27 | void export_producer(Module& m); 28 | void export_consumer(Module& m); 29 | void export_reader(Module& m); 30 | void export_config(Module& m); 31 | void export_enums(Module& m); 32 | void export_authentication(Module& m); 33 | void export_schema(Module& m); 34 | void export_exceptions(Module& m); 35 | void export_table_view(Module& m); 36 | 37 | PYBIND11_MODULE(_pulsar, m) { 38 | export_exceptions(m); 39 | export_client(m); 40 | export_message(m); 41 | export_producer(m); 42 | export_consumer(m); 43 | export_reader(m); 44 | export_config(m); 45 | export_enums(m); 46 | export_authentication(m); 47 | export_schema(m); 48 | export_table_view(m); 49 | } 50 | -------------------------------------------------------------------------------- /tests/test-conf/client-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEoQIBAAKCAQEAq2H1ErHhrhkBPllKxsoADJbodjqDINmvOuERIBLg5NBwj0t7 3 | r+GJ75vFqcLtriSNu0Ju7FkRP/VjWWEYn3C2dojiynkVzPucXly7odfw2BHUFzQe 4 | gX4LDgW+XfrWRq/hldigXcUv2amPaWRJlfdCFmqEKy6vkXM9ttREVpphQ0kVIq6Q 5 | XQQpkE6yQTRzPqJIBRy8jhsLwdXfVjJA6ZGie94xK2fxjtbFwIdXcCn5r9tXoC6M 6 | MAqnRzkzTNctMqpIKb3ESMVYUgfEmbHMZtqsKE3BvB9EP6NjYb3/SGF2BLJ9HG6c 7 | 7oK792AceqCYvi1wQy9kv9IPICX3x31wBbguvwIDAQABAoIBAQCcwbSPrPRncaeZ 8 | h8LFoO36le16dnqKCZIloMcxNxNNNvo9lyVC8mBgMXLSm+Eab4TTyyf6Nl14ytJc 9 | ZltHOqkqMnp+B9LQ8zNLfDaDCijY+TWtI5bjio5B/S7qdwyXCzii/slv+3SQ+m6a 10 | T4ifCtH//t11QfaEa4v/NphrPjnIeAgB681bk8nKdRop84ar+51lgbHoAza+wv+8 11 | e+aK3Od8r4yD19ZoPiMg0o4t2cEi8kupVgjsuZVtcvF9Q6QLYV17BFYEHqYjcr18 12 | N1EJ96f2FLO6cwEM+cG4n8gHjfDGRcDlhT9Cum1kDpg4J88auVUXnrDyi5Dcv1Pz 13 | 6EC+ZmXBAoGBAOHUSUDMkbEePKDaM3Z+4jLqZWc3UZhxQLnqg5l7phdQ6iSogQX9 14 | 1LpZCJ+lOMTHBCnaTCoQpuSHgYgraVkD4KG6nzC423oDesd/xNvlfW3TRsmwZWbL 15 | khdcdBSoVy3Kbv1v8kxw0NlcR68qo1XYfmFCAITcFHdxDz/jGStydlR9AoGBAMJH 16 | gyPenL595X8t47R93rkGOIx5cVf5YrDIZCByp4K44Tf9OqZHbky7jSPSSbur10mI 17 | pypRq5EcZ/cudU4w4gGaMauczt5Dgvlqd3T+GTZY3jO8bxi66gvzYTbigAxaJWcY 18 | Uafiv5W9ldRKsY3pyCL8ubg38Ed2cSaS2wGd/SDrAn8NO2MPaO0gc6UZx688QjL+ 19 | yL0oTxV42Snxusv7MkOJGjSd8UGeGEFeqdjXgdbRsNeNnDzaOh+NRGNSlziU/qUq 20 | 1MR/FlXF0G5hQhtGxyuSQ87iAnPukf79X21tyG9TP4lBUE3iLLoQAlgw606muQiu 21 | qi9dmYeZeAZst+HBqfNFAoGADg6qmH/VC5uEbY1eeoLZCL5AfTmUT+9FitEVHZvu 22 | LvE9qpVyFvH4Mykm7z6aAzBN5Y4zukYqiddqVmJQLpYu5DrJ+UbhWQe9hFqFxjtU 23 | i7Amc8vgpgNwR+kWUahV547mQe1qiyFHB4iuPKwi6MfPqWhr775sbl9NlKLvodBS 24 | rn0CgYBDrLH6ehNV/RnJIVZYQD6YcocYdYFy4u76mCYKmEP57XmstZHZXQgiRwbK 25 | Oy2Yg/qieKtSMjstgHFK6ZNYIR37l9J9Lh9aeal61+wW2dsGEy29Rhg01FpvKReq 26 | wCHz3tneUyaOhq9m0gKMOpWYcO+FBX1/2K5Gwj8FgEpu9r2b3w== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tests/test-conf/broker-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEAmypvJAIj9//mdWHKB6jAq+mN61EuZPeem9S0vjr69G7Gko84 3 | TQjNiRU+LMSZbctYgPzgTdZ99oKrDZTy4kXJ0xWVVwpshtx4ZDs0SwF8Xd5P1CEa 4 | XSegpXB6LgJQ4Rm0uQXfmQ2LzGLcEHP6cos4f9NWVGFQu5L/CXEJx70EQzyMnIsy 5 | 0QUEisaJ2HhWTdov9Ow0Nya1h+Q/JslBYLoxEBm++AykCoUZWeIAXbfAvdEu/KY0 6 | i4UqzAX2++QA5nSV/wJvQ385p8KDjls4QMlCyLwmcjY1ZMJUIhGH6GWPPelBp20Z 7 | iJogm5pS59LLs+Auj8FWVLxtFDBzxdeO0FpezQIDAQABAoIBAG9pk63mP49l1kM4 8 | eQjw2Y9WvslVXBuxVNiNbU4eKW1zUO+RGJrvlC027JLWg1g7pwvPBvu85GspPcsd 9 | xRxFgfonyDhcSrq2+Vb2z8B/i54W73jgX/69YnMIBSKeFRbcD1C+7+MEv/l8jojd 10 | zdmLL4FQ7O7fhUl57dgIqz4Y8UOYyyBsPpz3pzJLFEb5rE/ajqmFzyl+dO+8140B 11 | niQ0+7+tAK0njX8OC0WN844GkO24WPCfWhUFrYGkfLq498eRUCWM2YP2tAJ+Uxnh 12 | v3K9icDwOX6PJXYlbvNEUCE+t60NoDYHcMpfzUdFEhBYpKadfKE/RFFcu0vAZ+aR 13 | y24oAuECgYEAyPLYXWIs88pPHQhSf2DAMRref5eeV+XA6Dy/P+z8z0bA7I6X9dl6 14 | AK6rRKGJl9HI7c/Gky6P10fymopYopNkClXm7SBTLKx0vfjil0U6Mx5ZsfDspE3q 15 | 0o9MJKVgobCxVZlLErU55XzktKwjlv2UvDX7VuxRndqN9qdf+YSMb9kCgYEAxayx 16 | sOrJcPZVfy3Ohy5CeStF+E2dtfcKB7M7xZxZqykVy+6J1XjXHmp1L7Wpi0ju57Hi 17 | l2ZqKasHDwtlLOnfSTbvC47hsa1ydnoFTjJBObR1wS43oVkyV0AHid4w81ddOWPC 18 | H0ZmhvNe7pUxm5crpxsY6hAAraJ4Hej23MOxghUCgYEAip26UvCeQa2U1VogTm3X 19 | Jgh641kbiVabs5fz9Yzs966+9m+Gs7jJSB81Vap415mHGUTyniTIZKDk4WX9rmgt 20 | 4lNPcNOTjIWKImHFLMQ8WXbeOLkRBGYbThQ7WiwadG8GZR3Rg54vyfZVbawxAL78 21 | ErjKIDP0OQfCVhsvQVgF6EECgYEAlQ2P+xA/Dv+gHkLjDUmTdBxuKToVZqU9merL 22 | cklfz9EuD1Tx99ajltq9PFll25IGGw0mB/WAraS5sN1tz/0VkfZrL7LwefKIcc+2 23 | em0og6OQezcnWXGRpPqx9IJnNMY2lFSlhsGmA7I1bf9vpZvKnbmwAqZIbKUqn5sP 24 | sg2ZprUCgYEApAVD+9wXfZE/YDHVZX1k6p38ORqjq/04AJkL/LmUW5DL5to1+1KQ 25 | Q438HzMtYIq7aZyzWmlF6DmyN5mxKKK3yY79p0rvdV74AoT+ucDzM3ge0Md7liCs 26 | 0GwNnDSiPzdau738UoIKc1VbF7dMDL3LzqnfrBUCr7nXRbR3BHHuqws= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tests/test-conf/private-key.client-mismatch-rsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | MIIBwgIBATBNBgcqhkjOPQEBAkIB//////////////////////////////////// 3 | //////////////////////////////////////////////////8wgZ4EQgH///// 4 | //////////////////////////////////////////////////////////////// 5 | /////////////////ARBUZU+uWGOHJofkpohoLaFQO6i2nJbmbMV87i0iZGO8Qnh 6 | Vhk5Uex+k3sWUsC9O7G/BzVz34g9LDTx70Uf1GtQPwADFQDQnogAKRy4U5bMZxc5 7 | MoSqoNpkugSBhQQAxoWOBrcEBOnNnj7LZiOVtEKcZIE5BT+1Ifgor2BrTT26oUte 8 | d+/nWSj+HcEnov+o3jNIs8GFakKb+X5+McLlvWYBGDkpaniaO8AEXIpftCx9G9mY 9 | 9URJV5tEaBevvRcnPmYsl+5ymV70JkDFULkBP60HYTU8cIaicsJAiL6Udp/RZlAC 10 | QgH///////////////////////////////////////////pRhoeDvy+Wa3/MAUj3 11 | CaXQO7XJuImcR667b7cekThkCQIBAQ== 12 | -----END EC PARAMETERS----- 13 | -----BEGIN EC PRIVATE KEY----- 14 | MIICnQIBAQRCAeNLEp1HefZ1nMl5vvgFMsJCd5ieCWqPT7TXbQkn27A8WkyAGTYC 15 | GtolyPokOgSjbJh+ofBt/MgvE/nMrqzmkZVtoIIBxjCCAcICAQEwTQYHKoZIzj0B 16 | AQJCAf////////////////////////////////////////////////////////// 17 | ////////////////////////////MIGeBEIB//////////////////////////// 18 | //////////////////////////////////////////////////////////wEQVGV 19 | PrlhjhyaH5KaIaC2hUDuotpyW5mzFfO4tImRjvEJ4VYZOVHsfpN7FlLAvTuxvwc1 20 | c9+IPSw08e9FH9RrUD8AAxUA0J6IACkcuFOWzGcXOTKEqqDaZLoEgYUEAMaFjga3 21 | BATpzZ4+y2YjlbRCnGSBOQU/tSH4KK9ga009uqFLXnfv51ko/h3BJ6L/qN4zSLPB 22 | hWpCm/l+fjHC5b1mARg5KWp4mjvABFyKX7QsfRvZmPVESVebRGgXr70XJz5mLJfu 23 | cple9CZAxVC5AT+tB2E1PHCGonLCQIi+lHaf0WZQAkIB//////////////////// 24 | ///////////////////////6UYaHg78vlmt/zAFI9wml0Du1ybiJnEeuu2+3HpE4 25 | ZAkCAQGhgYkDgYYABAFhUHeaHfIWre/pPmv2a2l891co79dFpg6ixPRg+Y5qe0C7 26 | src//LT/ZR5rgj8ne+YcaIlwyQRl5OYEd25n799IcgHIBTGyaLB6Td5mW/oWT/Fz 27 | soufOnUJ7O/kDHjIQ15sczk3rDhe8/mB9zPjKlKTuAl5jBEt6E3yiB44Dtng02xD 28 | uQ== 29 | -----END EC PRIVATE KEY----- 30 | -------------------------------------------------------------------------------- /tests/test-conf/private-key.client-rsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAtKWwgqdnTYrOCv+j1MkTWfSH0wCsHZZca9wAW3qP4uuhlBvn 3 | b10JcFf5ZjzP9BSXK+tHmI8uoN368vEv6yhURHM4yuXqzCxzuAwkQSo39rzX8PGC 4 | 7qdjCN7LDJ3MnqiBIrUsSaEP1wrNsB1kI+o9ER1e5O/uEPAotP933hHQ0J2hMEek 5 | HqL7sBlJ98h6NmsicEaUkardk0TOXrlkjC+cMd8ZbGScPqI9M38bmn3OLxFTn1vt 6 | hpvnXLvCmG4M+6xtYtD+npcVPZw1i1R90fMs7ppZnRbv8Hc/DFdOKVQIgam6CDdn 7 | NKgW7c7IBMrP0AEm37HTu0LSOjP2OHXlvvlQGQIDAQABAoIBAAaJFAi2C7u3cNrf 8 | AstY9vVDLoLIvHFZlkBktjKZDYmVIsRb+hSCViwVUrWLL67R6+Iv4eg4DeTOAx00 9 | 8pncXKgZTw2wIb1/QjR/Y/RjlaC8lkdmRWli7udMQCZVsyhuSjW6Pj7vr8YE4woj 10 | FhNijxEGcf9wWrmMJrzdnTWQiXByo+eTvUQ9BPgPGrRjsMZmTkLyAVJff2DfxO5b 11 | IWFDYDJcyYAMCIMQu7vys/I50ou6ilb1CO6QM6Z7KpPeOoVFPwtzbh8cf9xM8UNS 12 | j6J/JmdWhgI34GS3NA68xTQ6PV7zjnhCc+iccm3JKyzGXwaApAZ+Eoce/9j4WKmu 13 | 5B4ziR0CgYEA3l/9OHbl1zmyV+rRxWOIj/i2rTvHzwBnbnPJyuemL5VMFdpGodQ3 14 | vwHvyQmcECRVRxmXojQ4QuPPHs3qp6wEEFPCWxChLSTxlUc85SOFHWU2O99jV7zI 15 | 7+JOpDK/Mstsx9nHgXduJF+glTFtA3LH8Oqylzu2aFPsprwKuZf94Q8CgYEAz/Zx 16 | akEG+PEMtP5YS28cX5XfjsIX/V26Fs6/sH16QjUIEddE5T4fCuokxCjSiwUcWhml 17 | pHEJ5S5xp3VYRfISW3jRW3qstIH1tpZipB6+S0zTuJmLJbA3IiWEg2rtMt7X1uJv 18 | A/bYOqe0hOPTuXuZdtVZ0nMTKk7GG8O6VkBI7FcCgYEAkDfCmscJgs7JahlBWHmX 19 | zH9pwem+SPKjIc/4NB6N+dgikx2Pp05hpP/VihUwYIufvs/LNogVYNQrtHepUnrN 20 | 2+TmbHbZgNSv1Ldxt82UfB7y0FutKu6lhmXHyNecho3Fi8sih0V0aiSWmYuHfrAH 21 | GaiskEZKo1iiZvQXJIx9O2MCgYATBf0r9hTYMtyxtc6H3/sdd01C9thQ8gDy0yjP 22 | 0Tqc0dMSJroDqmIWkoKYew9/bhFA4LW5TCnWkCAPbHmNtG4fdfbYwmkH/hdnA2y0 23 | jKdlpfp8GXeUFAGHGx17FA3sqFvgKUh0eWEgRHUL7vdQMVFBgJS93o7zQM94fLgP 24 | 6cOB8wKBgFcGV4GjI2Ww9cillaC554MvoSjf8B/+04kXzDOh8iYIIzO9EUil1jjK 25 | Jvxp4hnLzTKWbux3MEWqurLkYas6GpKBjw+iNOCar6YdqWGVqM3RUx7PTUaZwkKx 26 | UdP63IfY7iZCIT/QbyHQvIUe2MaiVnH+ulxdkK6Y5e7gxcbckIH4 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tests/debug_logger_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from unittest import TestCase, main 22 | import pulsar 23 | 24 | class DebugLoggerTest(TestCase): 25 | 26 | def test_configure_log_level(self): 27 | client = pulsar.Client( 28 | service_url="pulsar://localhost:6650", 29 | logger=pulsar.ConsoleLogger(pulsar.LoggerLevel.Debug) 30 | ) 31 | 32 | producer = client.create_producer( 33 | topic='test_log_level' 34 | ) 35 | 36 | producer.send(b'hello') 37 | 38 | def test_configure_log_to_file(self): 39 | client = pulsar.Client( 40 | service_url="pulsar://localhost:6650", 41 | logger=pulsar.FileLogger(pulsar.LoggerLevel.Debug, 'test.log') 42 | ) 43 | 44 | producer = client.create_producer( 45 | topic='test_log_to_file' 46 | ) 47 | 48 | producer.send(b'hello') 49 | 50 | if __name__ == "__main__": 51 | main() 52 | -------------------------------------------------------------------------------- /tests/interrupted_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from unittest import TestCase, main 22 | import pulsar 23 | import signal 24 | import time 25 | import threading 26 | 27 | class InterruptedTest(TestCase): 28 | 29 | service_url = 'pulsar://localhost:6650' 30 | 31 | def test_sigint(self): 32 | def thread_function(): 33 | time.sleep(1) 34 | signal.raise_signal(signal.SIGINT) 35 | 36 | client = pulsar.Client(self.service_url) 37 | consumer = client.subscribe('test-sigint', "my-sub") 38 | thread = threading.Thread(target=thread_function) 39 | thread.start() 40 | 41 | start = time.time() 42 | with self.assertRaises(pulsar.Interrupted): 43 | consumer.receive() 44 | finish = time.time() 45 | print(f"time: {finish - start}") 46 | self.assertGreater(finish - start, 1) 47 | self.assertLess(finish - start, 1.5) 48 | client.close() 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /pulsar/exceptions.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | from _pulsar import PulsarException, UnknownError, InvalidConfiguration, Timeout, LookupError, ConnectError, \ 21 | ReadError, AuthenticationError, AuthorizationError, ErrorGettingAuthenticationData, BrokerMetadataError, \ 22 | BrokerPersistenceError, ChecksumError, ConsumerBusy, NotConnected, AlreadyClosed, InvalidMessage, \ 23 | ConsumerNotInitialized, ProducerNotInitialized, ProducerBusy, TooManyLookupRequestException, InvalidTopicName, \ 24 | InvalidUrl, ServiceUnitNotReady, OperationNotSupported, ProducerBlockedQuotaExceededError, \ 25 | ProducerBlockedQuotaExceededException, ProducerQueueIsFull, MessageTooBig, TopicNotFound, SubscriptionNotFound, \ 26 | ConsumerNotFound, UnsupportedVersionError, TopicTerminated, CryptoError, IncompatibleSchema, ConsumerAssignError, \ 27 | CumulativeAcknowledgementNotAllowedError, TransactionCoordinatorNotFoundError, InvalidTxnStatusError, \ 28 | NotAllowedError, TransactionConflict, TransactionNotFound, ProducerFenced, MemoryBufferIsFull, Interrupted 29 | -------------------------------------------------------------------------------- /.asf.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | github: 21 | description: "Apache Pulsar Python client library" 22 | homepage: https://pulsar.apache.org/ 23 | labels: 24 | - pulsar 25 | - pubsub 26 | - messaging 27 | - streaming 28 | - queuing 29 | - event-streaming 30 | features: 31 | wiki: false 32 | issues: true 33 | projects: true 34 | enabled_merge_buttons: 35 | squash: true 36 | merge: false 37 | rebase: false 38 | protected_branches: 39 | main: 40 | required_status_checks: 41 | # Contexts are the names of checks that must pass. 42 | # See ./github/workflows/README.md for more documentation on this list. 43 | contexts: 44 | - Check Completion 45 | required_pull_request_reviews: 46 | required_approving_review_count: 1 47 | 48 | notifications: 49 | commits: commits@pulsar.apache.org 50 | issues: commits@pulsar.apache.org 51 | pullrequests: commits@pulsar.apache.org 52 | discussions: dev@pulsar.apache.org 53 | jira_options: link label 54 | -------------------------------------------------------------------------------- /src/utils.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include "utils.h" 21 | 22 | void waitForAsyncResult(std::function func) { 23 | auto promise = std::make_shared>(); 24 | 25 | { 26 | // Always call the Pulsar C++ client methods without holding 27 | // the GIL. This avoids deadlocks due the sequence of acquiring 28 | // mutexes by different threads. eg: 29 | // Thread-1: GIL -> producer.lock 30 | // Thread-2: producer.lock -> GIL (In a callback) 31 | py::gil_scoped_release release; 32 | func([promise](Result result) { promise->set_value(result); }); 33 | } 34 | internal::waitForResult(*promise); 35 | } 36 | 37 | namespace internal { 38 | 39 | void waitForResult(std::promise& promise) { 40 | auto future = promise.get_future(); 41 | while (true) { 42 | { 43 | py::gil_scoped_release release; 44 | auto status = future.wait_for(std::chrono::milliseconds(100)); 45 | if (status == std::future_status::ready) { 46 | CHECK_RESULT(future.get()); 47 | return; 48 | } 49 | } 50 | py::gil_scoped_acquire acquire; 51 | if (PyErr_CheckSignals() != 0) { 52 | raiseException(ResultInterrupted); 53 | } 54 | } 55 | } 56 | 57 | } // namespace internal 58 | -------------------------------------------------------------------------------- /pulsar/functions/function.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # -*- encoding: utf-8 -*- 21 | 22 | # Licensed to the Apache Software Foundation (ASF) under one 23 | # or more contributor license agreements. See the NOTICE file 24 | # distributed with this work for additional information 25 | # regarding copyright ownership. The ASF licenses this file 26 | # to you under the Apache License, Version 2.0 (the 27 | # "License"); you may not use this file except in compliance 28 | # with the License. You may obtain a copy of the License at 29 | # 30 | # http://www.apache.org/licenses/LICENSE-2.0 31 | # 32 | # Unless required by applicable law or agreed to in writing, 33 | # software distributed under the License is distributed on an 34 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 35 | # KIND, either express or implied. See the License for the 36 | # specific language governing permissions and limitations 37 | # under the License. 38 | # 39 | """ 40 | This is the core interface of the function api. 41 | 42 | The process method is called for every message of the input topic of the 43 | function. The incoming input bytes are deserialized using the serde. 44 | The process function can optionally emit an output 45 | """ 46 | from abc import abstractmethod 47 | 48 | 49 | class Function(object): 50 | """Interface for Pulsar Function""" 51 | 52 | @abstractmethod 53 | def process(self, input, context): 54 | """Process input message""" 55 | pass 56 | -------------------------------------------------------------------------------- /examples/rpc_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | 22 | import pulsar 23 | 24 | 25 | DEFAULT_CLIENT_TOPIC = 'rpc-client-topic' 26 | DEFAULT_SERVER_TOPIC = 'rpc-server-topic' 27 | 28 | 29 | class RPCServer(object): 30 | def __init__(self, 31 | client_topic=DEFAULT_CLIENT_TOPIC, 32 | server_topic=DEFAULT_SERVER_TOPIC): 33 | self.client_topic = client_topic 34 | self.server_topic = server_topic 35 | 36 | self.client = pulsar.Client('pulsar://localhost:6650') 37 | self.producer = self.client.create_producer(client_topic) 38 | self.consumer = \ 39 | self.client.subscribe(server_topic, 40 | 'rpc-server', 41 | pulsar.ConsumerType.Shared, 42 | message_listener=self.on_response) 43 | 44 | def on_response(self, consumer, message): 45 | print('Received from {0}: {1}'.format(message.partition_key(), 46 | message.data().decode('utf-8'))) 47 | 48 | self.producer.send('{} bar'.format(message.data().decode('utf-8')), 49 | partition_key=message.partition_key()) 50 | consumer.acknowledge(message) 51 | 52 | def start(self): 53 | self.consumer.resume_message_listener() 54 | 55 | 56 | rpc_server = RPCServer() 57 | rpc_server.start() 58 | 59 | try: 60 | while True: 61 | pass 62 | except KeyboardInterrupt: 63 | print('Interrupted.') 64 | -------------------------------------------------------------------------------- /tests/reader_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from unittest import TestCase, main 22 | import time 23 | 24 | from pulsar import Client, MessageId 25 | 26 | class ReaderTest(TestCase): 27 | 28 | def setUp(self): 29 | self._client: Client = Client('pulsar://localhost:6650') 30 | 31 | def tearDown(self) -> None: 32 | self._client.close() 33 | 34 | def test_has_message_available_after_seek(self): 35 | topic = f'test_has_message_available_after_seek-{time.time()}' 36 | producer = self._client.create_producer(topic) 37 | reader = self._client.create_reader(topic, start_message_id=MessageId.earliest) 38 | 39 | producer.send('msg-0'.encode()) 40 | self.assertTrue(reader.has_message_available()) 41 | 42 | reader.seek(MessageId.latest) 43 | self.assertFalse(reader.has_message_available()) 44 | 45 | producer.send('msg-1'.encode()) 46 | self.assertTrue(reader.has_message_available()) 47 | 48 | def test_seek_latest_message_id(self): 49 | topic = f'test_seek_latest_message_id-{time.time()}' 50 | producer = self._client.create_producer(topic) 51 | msg_id = producer.send('msg'.encode()) 52 | 53 | reader = self._client.create_reader(topic, 54 | start_message_id=MessageId.latest) 55 | self.assertFalse(reader.has_message_available()) 56 | reader.close() 57 | 58 | reader = self._client.create_reader(topic, 59 | start_message_id=MessageId.latest, 60 | start_message_id_inclusive=True) 61 | self.assertTrue(reader.has_message_available()) 62 | msg = reader.read_next(3000) 63 | self.assertEqual(msg.message_id(), msg_id) 64 | reader.close() 65 | 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /src/reader.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | #include 21 | 22 | namespace py = pybind11; 23 | 24 | Message Reader_readNext(Reader& reader) { 25 | return waitForAsyncValue([&](ReadNextCallback callback) { reader.readNextAsync(callback); }); 26 | } 27 | 28 | Message Reader_readNextTimeout(Reader& reader, int timeoutMs) { 29 | Message msg; 30 | Result res; 31 | Py_BEGIN_ALLOW_THREADS res = reader.readNext(msg, timeoutMs); 32 | Py_END_ALLOW_THREADS 33 | 34 | CHECK_RESULT(res); 35 | return msg; 36 | } 37 | 38 | bool Reader_hasMessageAvailable(Reader& reader) { 39 | return waitForAsyncValue( 40 | [&](HasMessageAvailableCallback callback) { reader.hasMessageAvailableAsync(callback); }); 41 | } 42 | 43 | void Reader_close(Reader& reader) { 44 | waitForAsyncResult([&](ResultCallback callback) { reader.closeAsync(callback); }); 45 | } 46 | 47 | void Reader_seek(Reader& reader, const MessageId& msgId) { 48 | waitForAsyncResult([&](ResultCallback callback) { reader.seekAsync(msgId, callback); }); 49 | } 50 | 51 | void Reader_seek_timestamp(Reader& reader, uint64_t timestamp) { 52 | waitForAsyncResult([&](ResultCallback callback) { reader.seekAsync(timestamp, callback); }); 53 | } 54 | 55 | bool Reader_is_connected(Reader& reader) { return reader.isConnected(); } 56 | 57 | void export_reader(py::module_& m) { 58 | using namespace py; 59 | 60 | class_(m, "Reader") 61 | .def("topic", &Reader::getTopic, return_value_policy::copy) 62 | .def("read_next", &Reader_readNext) 63 | .def("read_next", &Reader_readNextTimeout) 64 | .def("has_message_available", &Reader_hasMessageAvailable) 65 | .def("close", &Reader_close) 66 | .def("seek", &Reader_seek) 67 | .def("seek", &Reader_seek_timestamp) 68 | .def("is_connected", &Reader_is_connected); 69 | } 70 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: "CodeQL" 21 | 22 | on: 23 | push: 24 | branches: [ "main" ] 25 | pull_request: 26 | branches: [ "main" ] 27 | schedule: 28 | - cron: '27 21 * * 6' 29 | 30 | jobs: 31 | analyze: 32 | name: Analyze 33 | runs-on: 'ubuntu-latest' 34 | timeout-minutes: 360 35 | permissions: 36 | # required for all workflows 37 | security-events: write 38 | 39 | # only required for workflows in private repositories 40 | actions: read 41 | contents: read 42 | 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | language: [ 'c-cpp', 'python' ] 47 | 48 | steps: 49 | - name: Checkout repository 50 | uses: actions/checkout@v4 51 | 52 | # Initializes the CodeQL tools for scanning. 53 | - name: Initialize CodeQL 54 | uses: github/codeql-action/init@v3 55 | with: 56 | languages: ${{ matrix.language }} 57 | # If you wish to specify custom queries, you can do so here or in a config file. 58 | # By default, queries listed here will override any specified in a config file. 59 | # Prefix the list here with "+" to use these queries and those in the config file. 60 | 61 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 62 | # queries: security-extended,security-and-quality 63 | 64 | - uses: actions/setup-python@v5 65 | with: 66 | python-version: "3.12" 67 | 68 | - name: Install Pulsar C++ client 69 | run: build-support/install-dependencies.sh 70 | 71 | - name: CMake 72 | run: cmake . 73 | 74 | - name: Build 75 | run: make -j8 76 | 77 | - name: Perform CodeQL Analysis 78 | uses: github/codeql-action/analyze@v3 79 | with: 80 | category: "/language:${{matrix.language}}" 81 | -------------------------------------------------------------------------------- /examples/rpc_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | 22 | import pulsar 23 | import threading 24 | import uuid 25 | 26 | 27 | DEFAULT_CLIENT_TOPIC = 'rpc-client-topic' 28 | DEFAULT_SERVER_TOPIC = 'rpc-server-topic' 29 | UUID = str(uuid.uuid4()) 30 | NUM_CLIENT = 0 31 | LOCK = threading.Lock() 32 | 33 | 34 | class RPCClient(object): 35 | 36 | def __init__(self, 37 | client_topic=DEFAULT_CLIENT_TOPIC, 38 | server_topic=DEFAULT_SERVER_TOPIC): 39 | self.client_topic = client_topic 40 | self.server_topic = server_topic 41 | 42 | global NUM_CLIENT 43 | with LOCK: 44 | self.client_no = NUM_CLIENT 45 | NUM_CLIENT += 1 46 | 47 | self.response = None 48 | self.partition_key = '{0}_{1}'.format(UUID, self.client_no) 49 | self.client = pulsar.Client('pulsar://localhost:6650') 50 | self.producer = self.client.create_producer(server_topic) 51 | self.consumer = \ 52 | self.client.subscribe(client_topic, 53 | 'rpc-client-{}'.format(self.partition_key), 54 | message_listener=self.on_response) 55 | 56 | self.consumer.resume_message_listener() 57 | 58 | def on_response(self, consumer, message): 59 | if message.partition_key() == self.partition_key \ 60 | and consumer.topic() == self.client_topic: 61 | msg = message.data().decode('utf-8') 62 | print('Received: {0}'.format(msg)) 63 | self.response = msg 64 | consumer.acknowledge(message) 65 | 66 | def call(self, message): 67 | self.response = None 68 | self.producer.send(message.encode('utf-8'), partition_key=self.partition_key) 69 | 70 | while self.response is None: 71 | pass 72 | 73 | return self.response 74 | 75 | 76 | msg = 'foo' 77 | rpc_client = RPCClient() 78 | ret = rpc_client.call(msg) 79 | 80 | print('RPCClient message sent: {0}, result: {1}'.format(msg, ret)) 81 | -------------------------------------------------------------------------------- /src/utils.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #pragma once 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include "exceptions.h" 29 | 30 | using namespace pulsar; 31 | namespace py = pybind11; 32 | 33 | inline void CHECK_RESULT(Result res) { 34 | if (res != ResultOk) { 35 | raiseException(res); 36 | } 37 | } 38 | 39 | namespace internal { 40 | 41 | void waitForResult(std::promise& promise); 42 | 43 | } // namespace internal 44 | 45 | void waitForAsyncResult(std::function func); 46 | 47 | template 48 | inline T waitForAsyncValue(std::function)> func) { 49 | auto resultPromise = std::make_shared>(); 50 | auto valuePromise = std::make_shared>(); 51 | 52 | { 53 | py::gil_scoped_release release; 54 | 55 | func([resultPromise, valuePromise](Result result, const T& value) { 56 | valuePromise->set_value(value); 57 | resultPromise->set_value(result); 58 | }); 59 | } 60 | 61 | internal::waitForResult(*resultPromise); 62 | return valuePromise->get_future().get(); 63 | } 64 | 65 | struct CryptoKeyReaderWrapper { 66 | CryptoKeyReaderPtr cryptoKeyReader; 67 | 68 | CryptoKeyReaderWrapper(); 69 | CryptoKeyReaderWrapper(const std::string& publicKeyPath, const std::string& privateKeyPath); 70 | }; 71 | 72 | class CaptivePythonObjectMixin { 73 | protected: 74 | PyObject* _captive; 75 | 76 | CaptivePythonObjectMixin(PyObject* captive) { 77 | _captive = captive; 78 | PyGILState_STATE state = PyGILState_Ensure(); 79 | Py_XINCREF(_captive); 80 | PyGILState_Release(state); 81 | } 82 | 83 | ~CaptivePythonObjectMixin() { 84 | if (Py_IsInitialized()) { 85 | PyGILState_STATE state = PyGILState_Ensure(); 86 | Py_XDECREF(_captive); 87 | PyGILState_Release(state); 88 | } 89 | } 90 | }; 91 | -------------------------------------------------------------------------------- /src/authentication.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace py = pybind11; 26 | using namespace pulsar; 27 | 28 | void export_authentication(py::module_& m) { 29 | using namespace py; 30 | 31 | class_>(m, "Authentication") 32 | .def("getAuthMethodName", &Authentication::getAuthMethodName) 33 | .def("getAuthData", &Authentication::getAuthData) 34 | .def_static("create", static_cast( 35 | &AuthFactory::create)); 36 | 37 | class_>(m, "AuthenticationTLS") 38 | .def(init()) 39 | .def_static("create", static_cast( 40 | &AuthTls::create)); 41 | 42 | class_>(m, "AuthenticationToken") 43 | .def(init()) 44 | .def_static("create", static_cast(&AuthToken::create)) 45 | .def_static("create", static_cast(&AuthToken::create)); 46 | 47 | class_>(m, "AuthenticationAthenz") 48 | .def(init()) 49 | .def_static("create", static_cast(&AuthAthenz::create)); 50 | 51 | class_>(m, "AuthenticationOauth2") 52 | .def(init()) 53 | .def_static("create", static_cast(&AuthOauth2::create)); 54 | 55 | class_>(m, "AuthenticationBasic") 56 | .def(init()) 57 | .def_static("create", static_cast(&AuthBasic::create)) 60 | .def_static("create", static_cast(&AuthBasic::create)); 61 | } 62 | -------------------------------------------------------------------------------- /pkg/build-wheel-inside-docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | set -e -x 22 | 23 | cd /pulsar-client-python 24 | ROOT_DIR=$PWD 25 | source build-support/dep-url.sh 26 | 27 | # Build cpp wheels 28 | if [[ $ARCH == "aarch64" ]]; then 29 | export VCPKG_FORCE_SYSTEM_BINARIES=1 30 | fi 31 | PULSAR_CPP_VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') 32 | 33 | if [ $CPP_BINARY_TYPE == "rpm" ]; then 34 | # The pre-built RPM packages have incompatible ABI with manylinux, so we have to build from source 35 | download_dependency ./dependencies.yaml pulsar-cpp 36 | cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} 37 | 38 | git clone https://github.com/microsoft/vcpkg.git 39 | cd vcpkg 40 | 41 | # manylinux does not have ninja in the system package manager 42 | git clone https://github.com/ninja-build/ninja.git 43 | cd ninja 44 | git checkout release 45 | ./configure.py --bootstrap 46 | mv ninja /usr/bin/ 47 | cd .. 48 | ./bootstrap-vcpkg.sh 49 | cd .. 50 | if [ $PULSAR_CPP_VERSION == "3.7.0" ]; then 51 | patch lib/CMakeLists.txt $ROOT_DIR/pkg/manylinux/pulsar-client-cpp-3.7.0.patch 52 | fi 53 | cmake -B build-cpp -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON 54 | cmake --build build-cpp -j8 --target install 55 | cd .. 56 | rm -rf apache-pulsar-client-cpp-$(PULSAR_CPP_VERSION) 57 | else # apk 58 | if [ $ARCH == "aarch64" ]; then 59 | APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-arm64/aarch64 60 | else 61 | APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-x86_64/x86_64 62 | fi 63 | curl -O -L $APK_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-r0.apk 64 | curl -O -L $APK_ROOT_DIR/apache-pulsar-client-dev-$PULSAR_CPP_VERSION-r0.apk 65 | apk add --allow-untrusted *.apk 66 | fi 67 | 68 | download_dependency $PWD/dependencies.yaml pybind11 69 | rm -rf pybind11 70 | mv pybind11-* pybind11 71 | 72 | cmake -B build -DCMAKE_BUILD_TYPE=Release 73 | cmake --build build -j8 74 | mv build/lib_pulsar.so . 75 | 76 | ./setup.py bdist_wheel 77 | 78 | # Audit wheel is required to convert a wheel that is tagged as generic 79 | # 'linux' into a 'multilinux' wheel. 80 | # Only wheel files tagged as multilinux can be uploaded to PyPI 81 | # Audit wheel will make sure no external dependencies are needed for 82 | # the shared library and that only symbols supported by most linux 83 | # distributions are used. 84 | auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_*.whl 85 | -------------------------------------------------------------------------------- /tests/asyncio_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | import asyncio 22 | import pulsar 23 | from pulsar.asyncio import ( 24 | Client, 25 | PulsarException, 26 | ) 27 | from unittest import ( 28 | main, 29 | IsolatedAsyncioTestCase, 30 | ) 31 | 32 | service_url = 'pulsar://localhost:6650' 33 | 34 | class AsyncioTest(IsolatedAsyncioTestCase): 35 | 36 | async def asyncSetUp(self) -> None: 37 | self._client = Client(service_url, 38 | operation_timeout_seconds=5) 39 | 40 | async def asyncTearDown(self) -> None: 41 | await self._client.close() 42 | 43 | async def test_batch_send(self): 44 | producer = await self._client.create_producer('awaitio-test-batch-send') 45 | tasks = [] 46 | for i in range(5): 47 | tasks.append(asyncio.create_task(producer.send(f'msg-{i}'.encode()))) 48 | msg_ids = await asyncio.gather(*tasks) 49 | self.assertEqual(len(msg_ids), 5) 50 | ledger_id = msg_ids[0].ledger_id() 51 | entry_id = msg_ids[0].entry_id() 52 | # These messages should be in the same entry 53 | for i in range(5): 54 | msg_id = msg_ids[i] 55 | print(f'{i} was sent to {msg_id}') 56 | self.assertIsInstance(msg_id, pulsar.MessageId) 57 | self.assertEqual(msg_ids[i].ledger_id(), ledger_id) 58 | self.assertEqual(msg_ids[i].entry_id(), entry_id) 59 | self.assertEqual(msg_ids[i].batch_index(), i) 60 | 61 | async def test_create_producer_failure(self): 62 | try: 63 | await self._client.create_producer('tenant/ns/awaitio-test-send-failure') 64 | self.fail() 65 | except PulsarException as e: 66 | self.assertEqual(e.error(), pulsar.Result.Timeout) 67 | 68 | async def test_send_failure(self): 69 | producer = await self._client.create_producer('awaitio-test-send-failure') 70 | try: 71 | await producer.send(('x' * 1024 * 1024 * 10).encode()) 72 | self.fail() 73 | except PulsarException as e: 74 | self.assertEqual(e.error(), pulsar.Result.MessageTooBig) 75 | 76 | async def test_close_producer(self): 77 | producer = await self._client.create_producer('awaitio-test-close-producer') 78 | await producer.close() 79 | try: 80 | await producer.close() 81 | self.fail() 82 | except PulsarException as e: 83 | self.assertEqual(e.error(), pulsar.Result.AlreadyClosed) 84 | 85 | if __name__ == '__main__': 86 | main() 87 | -------------------------------------------------------------------------------- /src/table_view.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include "utils.h" 28 | 29 | namespace py = pybind11; 30 | using namespace pulsar; 31 | 32 | void export_table_view(py::module_& m) { 33 | py::class_(m, "TableViewConfiguration") 34 | .def(py::init<>()) 35 | .def("subscription_name", 36 | [](TableViewConfiguration& config, const std::string& name) { config.subscriptionName = name; }) 37 | .def("schema", 38 | [](TableViewConfiguration& config, const SchemaInfo& schema) { config.schemaInfo = schema; }); 39 | 40 | py::class_(m, "TableView") 41 | .def(py::init<>()) 42 | .def("get", 43 | [](const TableView& view, const std::string& key) -> std::pair { 44 | py::gil_scoped_release release; 45 | std::string value; 46 | bool available = view.getValue(key, value); 47 | py::gil_scoped_acquire acquire; 48 | if (available) { 49 | return std::make_pair(true, py::bytes(std::move(value))); 50 | } else { 51 | return std::make_pair(false, py::bytes()); 52 | } 53 | }) 54 | .def("size", &TableView::size, py::call_guard()) 55 | .def("for_each", 56 | [](TableView& view, std::function callback) { 57 | py::gil_scoped_release release; 58 | view.forEach([callback](const std::string& key, const std::string& value) { 59 | py::gil_scoped_acquire acquire; 60 | callback(key, py::bytes(value)); 61 | }); 62 | }) 63 | .def("for_each_and_listen", 64 | [](TableView& view, std::function callback) { 65 | py::gil_scoped_release release; 66 | view.forEachAndListen([callback](const std::string& key, const std::string& value) { 67 | py::gil_scoped_acquire acquire; 68 | callback(key, py::bytes(value)); 69 | }); 70 | }) 71 | .def("close", [](TableView& view) { 72 | waitForAsyncResult([&view](ResultCallback callback) { view.closeAsync(callback); }); 73 | }); 74 | } 75 | -------------------------------------------------------------------------------- /pulsar/functions/serde.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # -*- encoding: utf-8 -*- 21 | 22 | # Licensed to the Apache Software Foundation (ASF) under one 23 | # or more contributor license agreements. See the NOTICE file 24 | # distributed with this work for additional information 25 | # regarding copyright ownership. The ASF licenses this file 26 | # to you under the Apache License, Version 2.0 (the 27 | # "License"); you may not use this file except in compliance 28 | # with the License. You may obtain a copy of the License at 29 | # 30 | # http://www.apache.org/licenses/LICENSE-2.0 31 | # 32 | # Unless required by applicable law or agreed to in writing, 33 | # software distributed under the License is distributed on an 34 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 35 | # KIND, either express or implied. See the License for the 36 | # specific language governing permissions and limitations 37 | # under the License. 38 | # 39 | """ 40 | SerDe defines the interface for serialization/deserialization. 41 | 42 | Every time a message is read from pulsar topic, the serde is invoked to 43 | serialize the bytes into an object before invoking the process method. 44 | Anytime a python object needs to be written back to pulsar, it is 45 | serialized into bytes before writing. 46 | """ 47 | import pickle 48 | from abc import abstractmethod 49 | 50 | 51 | class SerDe(object): 52 | """Interface for Serialization/Deserialization""" 53 | 54 | @abstractmethod 55 | def serialize(self, input): 56 | """Serialize input message into bytes""" 57 | pass 58 | 59 | @abstractmethod 60 | def deserialize(self, input_bytes): 61 | """Serialize input_bytes into an object""" 62 | pass 63 | 64 | 65 | class PickleSerDe(SerDe): 66 | """Pickle based serializer""" 67 | 68 | def serialize(self, input): 69 | return pickle.dumps(input) 70 | 71 | def deserialize(self, input_bytes): 72 | return pickle.loads(input_bytes) 73 | 74 | 75 | class IdentitySerDe(SerDe): 76 | """Simple Serde that just conversion to string and back""" 77 | 78 | def __init__(self): 79 | self._types = [int, float, complex, str] 80 | 81 | def serialize(self, input): 82 | if type(input) in self._types: 83 | return str(input).encode('utf-8') 84 | if type(input) == bytes: 85 | return input 86 | raise TypeError("IdentitySerde cannot serialize object of type %s" % type(input)) 87 | 88 | def deserialize(self, input_bytes): 89 | for typ in self._types: 90 | try: 91 | return typ(input_bytes.decode('utf-8')) 92 | except: 93 | pass 94 | return input_bytes 95 | -------------------------------------------------------------------------------- /tests/custom_logger_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from unittest import TestCase, main 22 | import asyncio 23 | import logging 24 | import threading 25 | from pulsar import Client 26 | 27 | class CustomLoggingTest(TestCase): 28 | 29 | serviceUrl = 'pulsar://localhost:6650' 30 | 31 | def test_async_func_with_custom_logger(self): 32 | # boost::python::call may fail in C++ destructors, even worse, calls 33 | # to PyErr_Print could corrupt the Python interpreter. 34 | # See https://github.com/boostorg/python/issues/374 for details. 35 | # This test is to verify these functions won't be called in C++ destructors 36 | # so that Python's async function works well. 37 | client = Client( 38 | self.serviceUrl, 39 | logger=logging.getLogger('custom-logger') 40 | ) 41 | 42 | async def async_get(value): 43 | consumer = client.subscribe('test_async_get', 'sub') 44 | consumer.close() 45 | return value 46 | 47 | value = 'foo' 48 | result = asyncio.run(async_get(value)) 49 | self.assertEqual(value, result) 50 | 51 | client.close() 52 | 53 | def test_logger_thread_leaks(self): 54 | def _do_connect(close): 55 | logger = logging.getLogger(str(threading.current_thread().ident)) 56 | logger.setLevel(logging.INFO) 57 | client = Client( 58 | service_url="pulsar://localhost:6650", 59 | io_threads=4, 60 | message_listener_threads=4, 61 | operation_timeout_seconds=1, 62 | log_conf_file_path=None, 63 | authentication=None, 64 | logger=logger, 65 | ) 66 | client.get_topic_partitions("persistent://public/default/partitioned_topic_name_test") 67 | if close: 68 | client.close() 69 | 70 | for should_close in (True, False): 71 | self.assertEqual(threading.active_count(), 1, "Explicit close: {}; baseline is 1 thread".format(should_close)) 72 | _do_connect(should_close) 73 | self.assertEqual(threading.active_count(), 1, "Explicit close: {}; synchronous connect doesn't leak threads".format(should_close)) 74 | threads = [] 75 | for _ in range(10): 76 | threads.append(threading.Thread(target=_do_connect, args=(should_close))) 77 | threads[-1].start() 78 | for thread in threads: 79 | thread.join() 80 | assert threading.active_count() == 1, "Explicit close: {}; threaded connect in parallel doesn't leak threads".format(should_close) 81 | 82 | if __name__ == '__main__': 83 | logging.basicConfig(level=logging.DEBUG) 84 | main() 85 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from setuptools import setup 22 | from distutils.core import Extension 23 | from os import environ, path 24 | import platform 25 | 26 | from distutils.command import build_ext 27 | 28 | 29 | def get_version(): 30 | root = path.dirname(path.realpath(__file__)) 31 | version_file = path.join(root, 'pulsar', '__about__.py') 32 | version = {} 33 | with open(version_file) as fp: 34 | exec(fp.read(), version) 35 | return version['__version__'] 36 | 37 | 38 | def get_name(): 39 | postfix = environ.get('NAME_POSTFIX', '') 40 | base = 'pulsar-client' 41 | return base + postfix 42 | 43 | 44 | VERSION = get_version() 45 | NAME = get_name() 46 | 47 | print('NAME: %s' % NAME) 48 | print('VERSION: %s' % VERSION) 49 | 50 | 51 | # This is a workaround to have setuptools to include 52 | # the already compiled _pulsar.so library 53 | class my_build_ext(build_ext.build_ext): 54 | def build_extension(self, ext): 55 | import shutil 56 | import os.path 57 | 58 | try: 59 | os.makedirs(os.path.dirname(self.get_ext_fullpath(ext.name))) 60 | except OSError as e: 61 | if e.errno != 17: # already exists 62 | raise 63 | if 'Windows' in platform.platform(): 64 | shutil.copyfile('_pulsar.pyd', self.get_ext_fullpath(ext.name)) 65 | else: 66 | try: 67 | shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name)) 68 | except FileNotFoundError: 69 | shutil.copyfile('lib_pulsar.so', self.get_ext_fullpath(ext.name)) 70 | 71 | 72 | # Core Client dependencies 73 | dependencies = [ 74 | 'certifi', 75 | ] 76 | 77 | extras_require = {} 78 | 79 | # functions dependencies 80 | extras_require["functions"] = sorted( 81 | { 82 | "protobuf>=3.6.1", 83 | "grpcio>=1.59.3", 84 | "apache-bookkeeper-client>=4.16.1", 85 | "prometheus_client", 86 | "ratelimit" 87 | } 88 | ) 89 | 90 | # avro dependencies 91 | extras_require["avro"] = sorted( 92 | { 93 | "fastavro>=1.9.2" 94 | } 95 | ) 96 | 97 | # all dependencies 98 | extras_require["all"] = sorted(set(sum(extras_require.values(), []))) 99 | 100 | setup( 101 | name=NAME, 102 | version=VERSION, 103 | packages=['pulsar', 'pulsar.schema', 'pulsar.functions'], 104 | cmdclass={'build_ext': my_build_ext}, 105 | ext_modules=[Extension('_pulsar', [])], 106 | 107 | author="Pulsar Devs", 108 | author_email="dev@pulsar.apache.org", 109 | description="Apache Pulsar Python client library", 110 | license="Apache License v2.0", 111 | url="https://pulsar.apache.org/", 112 | install_requires=dependencies, 113 | extras_require=extras_require, 114 | ) 115 | -------------------------------------------------------------------------------- /src/exceptions.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #pragma once 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | struct PulsarException : std::exception { 27 | const pulsar::Result _result; 28 | std::string _msg = "Pulsar error: "; 29 | PulsarException(pulsar::Result res) : _result(res) { _msg += strResult(res); } 30 | const char* what() const noexcept override { return _msg.c_str(); } 31 | }; 32 | 33 | void raiseException(pulsar::Result result); 34 | 35 | #define INHERIT_PULSAR_EXCEPTION(name) \ 36 | struct name : PulsarException { \ 37 | using PulsarException::PulsarException; \ 38 | }; 39 | 40 | INHERIT_PULSAR_EXCEPTION(UnknownError) 41 | INHERIT_PULSAR_EXCEPTION(InvalidConfiguration) 42 | INHERIT_PULSAR_EXCEPTION(Timeout) 43 | INHERIT_PULSAR_EXCEPTION(LookupError) 44 | INHERIT_PULSAR_EXCEPTION(ConnectError) 45 | INHERIT_PULSAR_EXCEPTION(ReadError) 46 | INHERIT_PULSAR_EXCEPTION(AuthenticationError) 47 | INHERIT_PULSAR_EXCEPTION(AuthorizationError) 48 | INHERIT_PULSAR_EXCEPTION(ErrorGettingAuthenticationData) 49 | INHERIT_PULSAR_EXCEPTION(BrokerMetadataError) 50 | INHERIT_PULSAR_EXCEPTION(BrokerPersistenceError) 51 | INHERIT_PULSAR_EXCEPTION(ChecksumError) 52 | INHERIT_PULSAR_EXCEPTION(ConsumerBusy) 53 | INHERIT_PULSAR_EXCEPTION(NotConnected) 54 | INHERIT_PULSAR_EXCEPTION(AlreadyClosed) 55 | INHERIT_PULSAR_EXCEPTION(InvalidMessage) 56 | INHERIT_PULSAR_EXCEPTION(ConsumerNotInitialized) 57 | INHERIT_PULSAR_EXCEPTION(ProducerNotInitialized) 58 | INHERIT_PULSAR_EXCEPTION(ProducerBusy) 59 | INHERIT_PULSAR_EXCEPTION(TooManyLookupRequestException) 60 | INHERIT_PULSAR_EXCEPTION(InvalidTopicName) 61 | INHERIT_PULSAR_EXCEPTION(InvalidUrl) 62 | INHERIT_PULSAR_EXCEPTION(ServiceUnitNotReady) 63 | INHERIT_PULSAR_EXCEPTION(OperationNotSupported) 64 | INHERIT_PULSAR_EXCEPTION(ProducerBlockedQuotaExceededError) 65 | INHERIT_PULSAR_EXCEPTION(ProducerBlockedQuotaExceededException) 66 | INHERIT_PULSAR_EXCEPTION(ProducerQueueIsFull) 67 | INHERIT_PULSAR_EXCEPTION(MessageTooBig) 68 | INHERIT_PULSAR_EXCEPTION(TopicNotFound) 69 | INHERIT_PULSAR_EXCEPTION(SubscriptionNotFound) 70 | INHERIT_PULSAR_EXCEPTION(ConsumerNotFound) 71 | INHERIT_PULSAR_EXCEPTION(UnsupportedVersionError) 72 | INHERIT_PULSAR_EXCEPTION(TopicTerminated) 73 | INHERIT_PULSAR_EXCEPTION(CryptoError) 74 | INHERIT_PULSAR_EXCEPTION(IncompatibleSchema) 75 | INHERIT_PULSAR_EXCEPTION(ConsumerAssignError) 76 | INHERIT_PULSAR_EXCEPTION(CumulativeAcknowledgementNotAllowedError) 77 | INHERIT_PULSAR_EXCEPTION(TransactionCoordinatorNotFoundError) 78 | INHERIT_PULSAR_EXCEPTION(InvalidTxnStatusError) 79 | INHERIT_PULSAR_EXCEPTION(NotAllowedError) 80 | INHERIT_PULSAR_EXCEPTION(TransactionConflict) 81 | INHERIT_PULSAR_EXCEPTION(TransactionNotFound) 82 | INHERIT_PULSAR_EXCEPTION(ProducerFenced) 83 | INHERIT_PULSAR_EXCEPTION(MemoryBufferIsFull) 84 | INHERIT_PULSAR_EXCEPTION(Interrupted) 85 | 86 | #undef INHERIT_PULSAR_EXCEPTION 87 | -------------------------------------------------------------------------------- /src/producer.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace py = pybind11; 26 | 27 | MessageId Producer_send(Producer& producer, const Message& message) { 28 | return waitForAsyncValue( 29 | [&](SendCallback callback) { producer.sendAsync(message, callback); }); 30 | } 31 | 32 | void Producer_sendAsync(Producer& producer, const Message& msg, SendCallback callback) { 33 | Py_BEGIN_ALLOW_THREADS producer.sendAsync(msg, callback); 34 | Py_END_ALLOW_THREADS 35 | 36 | if (PyErr_CheckSignals() == -1) { 37 | PyErr_SetInterrupt(); 38 | } 39 | } 40 | 41 | void Producer_flush(Producer& producer) { 42 | waitForAsyncResult([&](ResultCallback callback) { producer.flushAsync(callback); }); 43 | } 44 | 45 | void Producer_close(Producer& producer) { 46 | waitForAsyncResult([&](ResultCallback callback) { producer.closeAsync(callback); }); 47 | } 48 | 49 | void Producer_closeAsync(Producer& producer, ResultCallback callback) { 50 | py::gil_scoped_release release; 51 | producer.closeAsync(callback); 52 | } 53 | 54 | void export_producer(py::module_& m) { 55 | using namespace py; 56 | 57 | class_(m, "Producer") 58 | .def(init<>()) 59 | .def("topic", &Producer::getTopic, "return the topic to which producer is publishing to", 60 | return_value_policy::copy) 61 | .def("producer_name", &Producer::getProducerName, 62 | "return the producer name which could have been assigned by the system or specified by the " 63 | "client", 64 | return_value_policy::copy) 65 | .def("last_sequence_id", &Producer::getLastSequenceId) 66 | .def("send", &Producer_send, 67 | "Publish a message on the topic associated with this Producer.\n" 68 | "\n" 69 | "This method will block until the message will be accepted and persisted\n" 70 | "by the broker. In case of errors, the client library will try to\n" 71 | "automatically recover and use a different broker.\n" 72 | "\n" 73 | "If it wasn't possible to successfully publish the message within the sendTimeout,\n" 74 | "an error will be returned.\n" 75 | "\n" 76 | "This method is equivalent to asyncSend() and wait until the callback is triggered.\n" 77 | "\n" 78 | "@param msg message to publish\n") 79 | .def("send_async", &Producer_sendAsync) 80 | .def("flush", &Producer_flush, 81 | "Flush all the messages buffered in the client and wait until all messages have been\n" 82 | "successfully persisted\n") 83 | .def("close", &Producer_close) 84 | .def("close_async", &Producer_closeAsync) 85 | .def("is_connected", &Producer::isConnected); 86 | } 87 | -------------------------------------------------------------------------------- /pulsar/tableview.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | """ 21 | The TableView implementation. 22 | """ 23 | 24 | from typing import Any, Callable, Optional 25 | from pulsar.schema.schema import Schema 26 | import _pulsar 27 | 28 | class TableView(): 29 | 30 | def __init__(self, table_view: _pulsar.TableView, topic: str, 31 | subscription: Optional[str], schema: Schema) -> None: 32 | self._table_view = table_view 33 | self._topic = topic 34 | self._subscription = subscription 35 | self._schema = schema 36 | 37 | def get(self, key: str) -> Optional[Any]: 38 | """ 39 | Return the value associated with the given key in the table view. 40 | 41 | Parameters 42 | ---------- 43 | key: str 44 | The message key 45 | 46 | Returns 47 | ------- 48 | Optional[Any] 49 | The value associated with the key, or None if the key does not exist. 50 | """ 51 | pair = self._table_view.get(key) 52 | if pair[0]: 53 | return self._schema.decode(pair[1]) 54 | else: 55 | return None 56 | 57 | def for_each(self, callback: Callable[[str, Any], None]) -> None: 58 | """ 59 | Iterate over all entries in the table view and call the callback function 60 | with the key and value for each entry. 61 | 62 | Parameters 63 | ---------- 64 | callback: Callable[[str, Any], None] 65 | The callback function to call for each entry. 66 | """ 67 | self._table_view.for_each(lambda k, v: callback(k, self._schema.decode(v))) 68 | 69 | def for_each_and_listen(self, callback: Callable[[str, Any], None]) -> None: 70 | """ 71 | Iterate over all entries in the table view and call the callback function 72 | with the key and value for each entry, then listen for changes. The callback 73 | will be called when a new entry is added or an existing entry is updated. 74 | 75 | Parameters 76 | ---------- 77 | callback: Callable[[str, Any], None] 78 | The callback function to call for each entry. 79 | """ 80 | self._table_view.for_each_and_listen(lambda k, v: callback(k, self._schema.decode(v))) 81 | 82 | def close(self) -> None: 83 | """ 84 | Close the table view. 85 | """ 86 | self._table_view.close() 87 | 88 | def __len__(self) -> int: 89 | """ 90 | Return the number of entries in the table view. 91 | """ 92 | return self._table_view.size() 93 | 94 | def __str__(self) -> str: 95 | if self._subscription is None: 96 | return f"TableView(topic={self._topic})" 97 | else: 98 | return f"TableView(topic={self._topic}, subscription={self._subscription})" 99 | 100 | def __repr__(self) -> str: 101 | return self.__str__() 102 | -------------------------------------------------------------------------------- /cmake_modules/FindClangTools.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | # Tries to find the clang-tidy and clang-format modules 15 | # 16 | # Usage of this module as follows: 17 | # 18 | # find_package(ClangTools) 19 | # 20 | # Variables used by this module, they can change the default behaviour and need 21 | # to be set before calling find_package: 22 | # 23 | # ClangToolsBin_HOME - 24 | # When set, this path is inspected instead of standard library binary locations 25 | # to find clang-tidy and clang-format 26 | # 27 | # This module defines 28 | # CLANG_TIDY_BIN, The path to the clang tidy binary 29 | # CLANG_TIDY_FOUND, Whether clang tidy was found 30 | # CLANG_FORMAT_BIN, The path to the clang format binary 31 | # CLANG_TIDY_FOUND, Whether clang format was found 32 | 33 | list(APPEND CLANG_SEARCH_PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin /opt/homebrew/bin) 34 | if (WIN32) 35 | list(APPEND CLANG_SEARCH_PATHS "C:/Program Files/LLVM/bin" "C:/Program Files (x86)/LLVM/bin") 36 | endif() 37 | 38 | find_program(CLANG_TIDY_BIN 39 | NAMES clang-tidy-4.0 40 | clang-tidy-3.9 41 | clang-tidy-3.8 42 | clang-tidy-3.7 43 | clang-tidy-3.6 44 | clang-tidy 45 | PATHS ${CLANG_SEARCH_PATHS} 46 | NO_DEFAULT_PATH 47 | ) 48 | 49 | if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) 50 | set(CLANG_TIDY_FOUND 0) 51 | message("clang-tidy not found") 52 | else() 53 | set(CLANG_TIDY_FOUND 1) 54 | message("clang-tidy found at ${CLANG_TIDY_BIN}") 55 | endif() 56 | 57 | if (CLANG_FORMAT_VERSION) 58 | find_program(CLANG_FORMAT_BIN 59 | NAMES clang-format-${CLANG_FORMAT_VERSION} 60 | PATHS ${CLANG_SEARCH_PATHS} 61 | NO_DEFAULT_PATH 62 | ) 63 | 64 | # If not found yet, search alternative locations 65 | if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE) 66 | # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/ 67 | STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}") 68 | STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}") 69 | if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0") 70 | find_program(CLANG_FORMAT_BIN 71 | NAMES clang-format 72 | PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin 73 | NO_DEFAULT_PATH 74 | ) 75 | else() 76 | find_program(CLANG_FORMAT_BIN 77 | NAMES clang-format 78 | PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin 79 | NO_DEFAULT_PATH 80 | ) 81 | endif() 82 | endif() 83 | else() 84 | find_program(CLANG_FORMAT_BIN 85 | NAMES clang-format-5 86 | clang-format-5.0 87 | clang-format 88 | PATHS ${CLANG_SEARCH_PATHS} 89 | NO_DEFAULT_PATH 90 | ) 91 | endif() 92 | 93 | if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) 94 | set(CLANG_FORMAT_FOUND 0) 95 | message("clang-format not found") 96 | else() 97 | set(CLANG_FORMAT_FOUND 1) 98 | message("clang-format found at ${CLANG_FORMAT_BIN}") 99 | endif() 100 | 101 | -------------------------------------------------------------------------------- /tests/oauth2_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from unittest import TestCase, main 22 | from pulsar import AuthenticationOauth2, AuthenticationError, Client 23 | import base64 24 | import os 25 | 26 | # This test should run against the standalone that is set up with 27 | # build-support/docker-compose-pulsar-oauth2.yml 28 | class Oauth2Test(TestCase): 29 | 30 | service_url = 'pulsar://localhost:6650' 31 | 32 | def test_invalid_private_key(self): 33 | def test_create_client(auth_params_string): 34 | client = Client(self.service_url, authentication=AuthenticationOauth2(auth_params_string)) 35 | with self.assertRaises(AuthenticationError): 36 | client.create_producer('oauth2-test-base64') 37 | client.close() 38 | 39 | test_create_client('{"private_key":"xxx:yyy"}') 40 | test_create_client('{"private_key":"data:"}') 41 | test_create_client('{"private_key":"data:application/x-pem"}') 42 | test_create_client('{"private_key":"data:application/json;xxx"}') 43 | 44 | def test_key_file(self): 45 | path = (os.path.dirname(os.path.abspath(__file__)) 46 | + '/test-conf/cpp_credentials_file.json') 47 | auth = AuthenticationOauth2(f'''{{ 48 | "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", 49 | "private_key": "{path}", 50 | "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/" 51 | }}''') 52 | client = Client(self.service_url, authentication=auth) 53 | producer = client.create_producer('oauth2-test-base64') 54 | producer.close() 55 | client.close() 56 | 57 | def test_base64(self): 58 | credentials = '''{ 59 | "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", 60 | "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" 61 | }''' 62 | base64_credentials = base64.b64encode(credentials.encode()).decode() 63 | auth = AuthenticationOauth2(f'''{{ 64 | "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", 65 | "private_key": "data:application/json;base64,{base64_credentials}", 66 | "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/" 67 | }}''') 68 | client = Client(self.service_url, authentication=auth) 69 | producer = client.create_producer('oauth2-test-base64') 70 | producer.close() 71 | client.close() 72 | 73 | def test_wrong_secret(self): 74 | credentials = '''{ 75 | "client_id": "my-id", 76 | "client_secret":"my-secret" 77 | }''' 78 | base64_credentials = base64.b64encode(credentials.encode()).decode() 79 | auth = AuthenticationOauth2(f'''{{ 80 | "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", 81 | "private_key": "data:application/json;base64,{base64_credentials}", 82 | "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/" 83 | }}''') 84 | client = Client(self.service_url, authentication=auth) 85 | with self.assertRaises(AuthenticationError): 86 | client.create_producer('oauth2-test-base64') 87 | client.close() 88 | 89 | if __name__ == '__main__': 90 | main() 91 | -------------------------------------------------------------------------------- /pulsar/schema/schema.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | 21 | from abc import abstractmethod 22 | import json 23 | import _pulsar 24 | import enum 25 | 26 | 27 | class Schema(object): 28 | def __init__(self, record_cls, schema_type, schema_definition, schema_name): 29 | self._record_cls = record_cls 30 | self._schema_info = _pulsar.SchemaInfo(schema_type, schema_name, 31 | json.dumps(schema_definition, indent=True)) 32 | 33 | @abstractmethod 34 | def encode(self, obj): 35 | pass 36 | 37 | @abstractmethod 38 | def decode(self, data): 39 | pass 40 | 41 | def decode_message(self, msg: _pulsar.Message): 42 | return self.decode(msg.data()) 43 | 44 | def schema_info(self): 45 | return self._schema_info 46 | 47 | def attach_client(self, client: _pulsar.Client): 48 | self._client = client 49 | 50 | def _validate_object_type(self, obj): 51 | if not isinstance(obj, self._record_cls): 52 | raise TypeError('Invalid record obj of type ' + str(type(obj)) 53 | + ' - expected type is ' + str(self._record_cls)) 54 | 55 | 56 | class BytesSchema(Schema): 57 | def __init__(self): 58 | super(BytesSchema, self).__init__(bytes, _pulsar.SchemaType.BYTES, None, 'BYTES') 59 | 60 | def encode(self, data): 61 | self._validate_object_type(data) 62 | return data 63 | 64 | def decode(self, data): 65 | return data 66 | 67 | def __str__(self): 68 | return 'BytesSchema' 69 | 70 | 71 | class StringSchema(Schema): 72 | def __init__(self): 73 | super(StringSchema, self).__init__(str, _pulsar.SchemaType.STRING, None, 'STRING') 74 | 75 | def encode(self, obj): 76 | self._validate_object_type(obj) 77 | return obj.encode('utf-8') 78 | 79 | def decode(self, data): 80 | return data.decode('utf-8') 81 | 82 | def __str__(self): 83 | return 'StringSchema' 84 | 85 | 86 | def remove_reserved_key(data): 87 | if '_default' in data: 88 | del data['_default'] 89 | if '_required' in data: 90 | del data['_required'] 91 | if '_required_default' in data: 92 | del data['_required_default'] 93 | 94 | 95 | class JsonSchema(Schema): 96 | 97 | def __init__(self, record_cls): 98 | super(JsonSchema, self).__init__(record_cls, _pulsar.SchemaType.JSON, 99 | record_cls.schema(), 'JSON') 100 | 101 | def _get_serialized_value(self, o): 102 | if isinstance(o, enum.Enum): 103 | return o.value 104 | elif isinstance(o, bytes): 105 | return o.decode() 106 | else: 107 | data = o.__dict__.copy() 108 | remove_reserved_key(data) 109 | return data 110 | 111 | def encode(self, obj): 112 | self._validate_object_type(obj) 113 | # Copy the dict of the object as to not modify the provided object via the reference provided 114 | data = obj.__dict__.copy() 115 | remove_reserved_key(data) 116 | return json.dumps(data, default=self._get_serialized_value, indent=True).encode('utf-8') 117 | 118 | def decode(self, data): 119 | return self._record_cls(**json.loads(data)) 120 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | cmake_minimum_required(VERSION 3.18) 21 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") 22 | 23 | project (pulsar-client-python) 24 | set(CMAKE_PREFIX_PATH ${PROJECT_SOURCE_DIR}/pybind11/include ${CMAKE_PREFIX_PATH}) 25 | option(LINK_STATIC "Link against static libraries" OFF) 26 | MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) 27 | 28 | if (NOT CMAKE_BUILD_TYPE) 29 | set(CMAKE_BUILD_TYPE Release) 30 | endif () 31 | MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) 32 | set(THREADS_PREFER_PTHREAD_FLAG TRUE) 33 | find_package(Threads REQUIRED) 34 | MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) 35 | 36 | if (MSVC) 37 | add_compile_options(/wd4819) 38 | endif () 39 | 40 | if (LINK_STATIC) 41 | find_library(PULSAR_LIBRARY NAMES pulsarwithdeps pulsarWithDeps.lib) 42 | add_definitions("-DPULSAR_STATIC") 43 | else () 44 | find_library(PULSAR_LIBRARY NAMES pulsar libpulsar) 45 | endif() 46 | message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}") 47 | 48 | find_path(PULSAR_INCLUDE pulsar/Client.h) 49 | message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}") 50 | 51 | SET(CMAKE_CXX_STANDARD 17) 52 | 53 | find_package (Python3 REQUIRED COMPONENTS Development.Module) 54 | MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS}) 55 | 56 | find_path(PYBIND11_INCLUDE_DIRS NAMES "pybind11/pybind11.h") 57 | message(STATUS "PYBIND11_INCLUDE_DIRS: " ${PYBIND11_INCLUDE_DIRS}) 58 | 59 | ######################################################################################################################## 60 | 61 | INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} ${PYBIND11_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) 62 | 63 | file(GLOB SOURCES src/*.cc) 64 | ADD_LIBRARY(_pulsar SHARED ${SOURCES}) 65 | if (MSVC) 66 | set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd) 67 | else () 68 | set(CMAKE_SHARED_LIBRARY_SUFFIX .so) 69 | endif () 70 | 71 | if (NOT APPLE AND NOT MSVC) 72 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_PYTHON}") 73 | endif() 74 | 75 | if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 76 | set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -Qunused-arguments -undefined dynamic_lookup") 77 | endif() 78 | 79 | set(PYTHON_WRAPPER_LIBS 80 | ${PULSAR_LIBRARY} 81 | ) 82 | set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) 83 | 84 | message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") 85 | 86 | if (LINK_STATIC AND NOT MSVC) 87 | if (APPLE) 88 | set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") 89 | target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) 90 | else () 91 | set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++") 92 | target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) 93 | endif () 94 | elseif (LINK_STATIC) # MSVC 95 | set_property(TARGET _pulsar PROPERTY 96 | MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") 97 | target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) 98 | else() 99 | target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) 100 | endif () 101 | install(TARGETS _pulsar DESTINATION ${CMAKE_SOURCE_DIR}) 102 | 103 | find_package(ClangTools) 104 | set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") 105 | add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py 106 | ${CLANG_FORMAT_BIN} 107 | 0 108 | ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt 109 | ${CMAKE_SOURCE_DIR}/src) 110 | 111 | # `make check-format` option (for CI test) 112 | add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py 113 | ${CLANG_FORMAT_BIN} 114 | 1 115 | ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt 116 | ${CMAKE_SOURCE_DIR}/src) 117 | -------------------------------------------------------------------------------- /tests/table_view_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | from typing import Callable 22 | from unittest import TestCase, main 23 | import time 24 | 25 | from pulsar import Client 26 | from pulsar.schema.schema import StringSchema 27 | 28 | class TableViewTest(TestCase): 29 | 30 | def setUp(self): 31 | self._client: Client = Client('pulsar://localhost:6650') 32 | 33 | def tearDown(self): 34 | self._client.close() 35 | 36 | def test_get(self): 37 | topic = f'table_view_test_get-{time.time()}' 38 | table_view = self._client.create_table_view(topic) 39 | self.assertEqual(len(table_view), 0) 40 | 41 | producer = self._client.create_producer(topic) 42 | producer.send(b'value-0', partition_key='key-0') 43 | producer.send(b'\xba\xd0\xba\xd0', partition_key='key-1') # an invalid UTF-8 bytes 44 | 45 | self._wait_for_assertion(lambda: self.assertEqual(len(table_view), 2)) 46 | self.assertEqual(table_view.get('key-0'), b'value-0') 47 | self.assertEqual(table_view.get('key-1'), b'\xba\xd0\xba\xd0') 48 | 49 | producer.send(b'value-1', partition_key='key-0') 50 | self._wait_for_assertion(lambda: self.assertEqual(table_view.get('key-0'), b'value-1')) 51 | 52 | producer.close() 53 | table_view.close() 54 | 55 | def test_for_each(self): 56 | topic = f'table_view_test_for_each-{time.time()}' 57 | table_view = self._client.create_table_view(topic) 58 | producer = self._client.create_producer(topic) 59 | producer.send(b'value-0', partition_key='key-0') 60 | producer.send(b'value-1', partition_key='key-1') 61 | self._wait_for_assertion(lambda: self.assertEqual(len(table_view), 2)) 62 | 63 | d = dict() 64 | table_view.for_each(lambda key, value: d.__setitem__(key, value)) 65 | self.assertEqual(d, { 66 | 'key-0': b'value-0', 67 | 'key-1': b'value-1' 68 | }) 69 | 70 | def listener(key: str, value: str): 71 | if len(value) == 0: 72 | d.pop(key) 73 | else: 74 | d[key] = value 75 | 76 | d.clear() 77 | table_view.for_each_and_listen(listener) 78 | self.assertEqual(d, { 79 | 'key-0': b'value-0', 80 | 'key-1': b'value-1' 81 | }) 82 | 83 | producer.send(b'value-0-new', partition_key='key-0') 84 | producer.send(b'', partition_key='key-1') 85 | producer.send(b'value-2', partition_key='key-2') 86 | def assert_latest_values(): 87 | self.assertEqual(d, { 88 | 'key-0': b'value-0-new', 89 | 'key-2': b'value-2' 90 | }) 91 | self._wait_for_assertion(assert_latest_values) 92 | 93 | def test_schema(self): 94 | topic = f'table_view_test_schema-{time.time()}' 95 | table_view = self._client.create_table_view(topic, schema=StringSchema()) 96 | producer = self._client.create_producer(topic, schema=StringSchema()) 97 | producer.send('value', partition_key='key') 98 | 99 | self._wait_for_assertion(lambda: self.assertEqual(table_view.get('key'), 'value')) 100 | self.assertEqual(table_view.get('missed-key'), None) 101 | 102 | entries = dict() 103 | table_view.for_each(lambda key, value: entries.__setitem__(key, value)) 104 | self.assertEqual(entries, {'key': 'value'}) 105 | 106 | entries.clear() 107 | table_view.for_each_and_listen(lambda key, value: entries.__setitem__(key, value)) 108 | self.assertEqual(entries, {'key': 'value'}) 109 | 110 | producer.send('new-value', partition_key='key') 111 | self._wait_for_assertion(lambda: self.assertEqual(table_view.get('key'), 'new-value')) 112 | 113 | def _wait_for_assertion(self, assertion: Callable, timeout=5) -> None: 114 | start_time = time.time() 115 | while time.time() - start_time < timeout: 116 | try: 117 | assertion() 118 | return 119 | except AssertionError: 120 | time.sleep(0.1) 121 | assertion() 122 | 123 | if __name__ == "__main__": 124 | main() 125 | -------------------------------------------------------------------------------- /src/client.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace py = pybind11; 26 | 27 | Producer Client_createProducer(Client& client, const std::string& topic, const ProducerConfiguration& conf) { 28 | return waitForAsyncValue( 29 | [&](CreateProducerCallback callback) { client.createProducerAsync(topic, conf, callback); }); 30 | } 31 | 32 | void Client_createProducerAsync(Client& client, const std::string& topic, ProducerConfiguration conf, 33 | CreateProducerCallback callback) { 34 | py::gil_scoped_release release; 35 | client.createProducerAsync(topic, conf, callback); 36 | } 37 | 38 | Consumer Client_subscribe(Client& client, const std::string& topic, const std::string& subscriptionName, 39 | const ConsumerConfiguration& conf) { 40 | return waitForAsyncValue( 41 | [&](SubscribeCallback callback) { client.subscribeAsync(topic, subscriptionName, conf, callback); }); 42 | } 43 | 44 | Consumer Client_subscribe_topics(Client& client, const std::vector& topics, 45 | const std::string& subscriptionName, const ConsumerConfiguration& conf) { 46 | return waitForAsyncValue( 47 | [&](SubscribeCallback callback) { client.subscribeAsync(topics, subscriptionName, conf, callback); }); 48 | } 49 | 50 | Consumer Client_subscribe_pattern(Client& client, const std::string& topic_pattern, 51 | const std::string& subscriptionName, const ConsumerConfiguration& conf) { 52 | return waitForAsyncValue([&](SubscribeCallback callback) { 53 | client.subscribeWithRegexAsync(topic_pattern, subscriptionName, conf, callback); 54 | }); 55 | } 56 | 57 | Reader Client_createReader(Client& client, const std::string& topic, const MessageId& startMessageId, 58 | const ReaderConfiguration& conf) { 59 | return waitForAsyncValue( 60 | [&](ReaderCallback callback) { client.createReaderAsync(topic, startMessageId, conf, callback); }); 61 | } 62 | 63 | std::vector Client_getTopicPartitions(Client& client, const std::string& topic) { 64 | return waitForAsyncValue>( 65 | [&](GetPartitionsCallback callback) { client.getPartitionsForTopicAsync(topic, callback); }); 66 | } 67 | 68 | SchemaInfo Client_getSchemaInfo(Client& client, const std::string& topic, int64_t version) { 69 | return waitForAsyncValue([&](std::function callback) { 70 | client.getSchemaInfoAsync(topic, version, callback); 71 | }); 72 | } 73 | 74 | void Client_close(Client& client) { 75 | waitForAsyncResult([&](ResultCallback callback) { client.closeAsync(callback); }); 76 | } 77 | 78 | void Client_closeAsync(Client& client, ResultCallback callback) { 79 | py::gil_scoped_release release; 80 | client.closeAsync(callback); 81 | } 82 | 83 | void export_client(py::module_& m) { 84 | py::class_>(m, "Client") 85 | .def(py::init()) 86 | .def("create_producer", &Client_createProducer) 87 | .def("create_producer_async", &Client_createProducerAsync) 88 | .def("subscribe", &Client_subscribe) 89 | .def("subscribe_topics", &Client_subscribe_topics) 90 | .def("subscribe_pattern", &Client_subscribe_pattern) 91 | .def("create_reader", &Client_createReader) 92 | .def("create_table_view", [](Client& client, const std::string& topic, 93 | const TableViewConfiguration& config) { 94 | return waitForAsyncValue([&](TableViewCallback callback) { 95 | client.createTableViewAsync(topic, config, callback); 96 | }); 97 | }) 98 | .def("get_topic_partitions", &Client_getTopicPartitions) 99 | .def("get_schema_info", &Client_getSchemaInfo) 100 | .def("close", &Client_close) 101 | .def("close_async", &Client_closeAsync) 102 | .def("shutdown", &Client::shutdown); 103 | } 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 21 | 22 | # Pulsar Python client library 23 | 24 | Pulsar Python clients support a variety of Pulsar features to enable building applications connecting to your Pulsar cluster. For the supported Pulsar features, see [Client Feature Matrix](https://pulsar.apache.org/client-feature-matrix/). 25 | 26 | ## Requirements 27 | 28 | - Python 3.10, 3.11, 3.12, 3.13 or 3.14 29 | - A C++ compiler that supports C++11 30 | - CMake >= 3.18 31 | - [Pulsar C++ client library](https://github.com/apache/pulsar-client-cpp) 32 | - [PyBind11](https://github.com/pybind/pybind11) 33 | 34 | PyBind11 is a header-only library and a submodule, so you can simply download the submodule so that CMake can find this dependency. 35 | 36 | ```bash 37 | git submodule update --init 38 | ``` 39 | 40 | You can also download the pybind11 directly like: 41 | 42 | ```bash 43 | pip3 install pyyaml 44 | export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) 45 | curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz 46 | tar zxf v${PYBIND11_VERSION}.tar.gz 47 | mv pybind11-${PYBIND11_VERSION} pybind11 48 | ``` 49 | 50 | After that, you only need to install the Pulsar C++ client dependency into the system path. You can [install the pre-built binaries](https://pulsar.apache.org/docs/next/client-libraries-cpp/#installation) or [build from source](https://github.com/apache/pulsar-client-cpp#compilation). 51 | 52 | ## Install the Python wheel 53 | 54 | Make sure the PyBind11 submodule has been downloaded and the Pulsar C++ client has been installed. Then run the following commands: 55 | 56 | ```bash 57 | cmake -B build 58 | cmake --build build 59 | cmake --install build 60 | python3 ./setup.py bdist_wheel 61 | python3 -m pip install dist/pulsar_client-*.whl --force-reinstall 62 | ``` 63 | 64 | > **NOTE** 65 | > 66 | > 1. The separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` to be under the project directory. 67 | > 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before. 68 | > 3. On Windows, the Python command is `py` instead of `python3`. 69 | 70 | ## Running examples 71 | 72 | You can run `python3 -c 'import pulsar'` to see whether the wheel has been installed successfully. If it fails, check whether dependencies (e.g., `libpulsar.so`) are in the system path. If not, make sure the dependencies are in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS). 73 | 74 | Then you can run examples as a simple end-to-end test. 75 | 76 | ```bash 77 | # In terminal 1 78 | python3 ./examples/consumer.py 79 | ``` 80 | 81 | ```bash 82 | # In terminal 2 83 | python3 ./examples/producer.py 84 | ``` 85 | 86 | Before executing the commands above, you must ensure the Pulsar service is running. See [here](https://pulsar.apache.org/docs/getting-started-standalone) for quick start. 87 | 88 | ## Unit tests 89 | 90 | Before running the unit tests, you must run a Pulsar service with all things set up: 91 | 92 | ```bash 93 | ./build-support/pulsar-test-service-start.sh 94 | ``` 95 | 96 | The command above runs a Pulsar standalone in a Docker container. You can run `./build-support/pulsar-test-service-stop.sh` to stop it. 97 | 98 | Run all unit tests: 99 | 100 | ```bash 101 | ./tests/run-unit-tests.sh 102 | ``` 103 | 104 | Run a single unit test (e.g., `PulsarTest.test_tls_auth`): 105 | 106 | ```bash 107 | python3 ./tests/pulsar_test.py 'PulsarTest.test_tls_auth' 108 | ``` 109 | 110 | ## Generate API docs 111 | 112 | Pulsar Python Client uses [pydoctor](https://github.com/twisted/pydoctor) to generate API docs. To generate by yourself, you need to install the Python library first. Then run the following command in the root path of this repository: 113 | 114 | ```bash 115 | sudo python3 -m pip install pydoctor 116 | cp $(python3 -c 'import _pulsar, os; print(_pulsar.__file__)') ./_pulsar.so 117 | pydoctor --make-html \ 118 | --docformat=numpy --theme=readthedocs \ 119 | --intersphinx=https://docs.python.org/3/objects.inv \ 120 | --html-output= \ 121 | --introspect-c-modules \ 122 | ./_pulsar.so \ 123 | pulsar 124 | ``` 125 | 126 | Then the index page will be generated in `/index.html`. 127 | 128 | ## Contribute 129 | 130 | We welcome contributions from the open source community! 131 | 132 | If your contribution adds Pulsar features for Python clients, you need to update both the [Pulsar docs](https://pulsar.apache.org/docs/client-libraries/) and the [Client Feature Matrix](https://pulsar.apache.org/client-feature-matrix/). See [Contribution Guide](https://pulsar.apache.org/contribute/site-intro/#pages) for more details. 133 | -------------------------------------------------------------------------------- /tests/test-conf/client-cert.pem: -------------------------------------------------------------------------------- 1 | Certificate: 2 | Data: 3 | Version: 1 (0x0) 4 | Serial Number: 4097 (0x1001) 5 | Signature Algorithm: sha256WithRSAEncryption 6 | Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org 7 | Validity 8 | Not Before: Feb 17 16:56:55 2021 GMT 9 | Not After : Feb 12 16:56:55 2041 GMT 10 | Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=admin/emailAddress=dev@pulsar.apache.org 11 | Subject Public Key Info: 12 | Public Key Algorithm: rsaEncryption 13 | Public-Key: (2048 bit) 14 | Modulus: 15 | 00:ab:61:f5:12:b1:e1:ae:19:01:3e:59:4a:c6:ca: 16 | 00:0c:96:e8:76:3a:83:20:d9:af:3a:e1:11:20:12: 17 | e0:e4:d0:70:8f:4b:7b:af:e1:89:ef:9b:c5:a9:c2: 18 | ed:ae:24:8d:bb:42:6e:ec:59:11:3f:f5:63:59:61: 19 | 18:9f:70:b6:76:88:e2:ca:79:15:cc:fb:9c:5e:5c: 20 | bb:a1:d7:f0:d8:11:d4:17:34:1e:81:7e:0b:0e:05: 21 | be:5d:fa:d6:46:af:e1:95:d8:a0:5d:c5:2f:d9:a9: 22 | 8f:69:64:49:95:f7:42:16:6a:84:2b:2e:af:91:73: 23 | 3d:b6:d4:44:56:9a:61:43:49:15:22:ae:90:5d:04: 24 | 29:90:4e:b2:41:34:73:3e:a2:48:05:1c:bc:8e:1b: 25 | 0b:c1:d5:df:56:32:40:e9:91:a2:7b:de:31:2b:67: 26 | f1:8e:d6:c5:c0:87:57:70:29:f9:af:db:57:a0:2e: 27 | 8c:30:0a:a7:47:39:33:4c:d7:2d:32:aa:48:29:bd: 28 | c4:48:c5:58:52:07:c4:99:b1:cc:66:da:ac:28:4d: 29 | c1:bc:1f:44:3f:a3:63:61:bd:ff:48:61:76:04:b2: 30 | 7d:1c:6e:9c:ee:82:bb:f7:60:1c:7a:a0:98:be:2d: 31 | 70:43:2f:64:bf:d2:0f:20:25:f7:c7:7d:70:05:b8: 32 | 2e:bf 33 | Exponent: 65537 (0x10001) 34 | Signature Algorithm: sha256WithRSAEncryption 35 | 1c:31:b8:0f:a1:03:28:a0:da:31:ec:34:ce:e0:fd:01:99:9d: 36 | 9b:ad:f8:03:5d:20:85:18:de:ca:b5:ea:61:c9:3b:65:42:9c: 37 | e5:21:73:d2:06:41:4b:a9:3a:fb:7f:ff:45:f3:5a:4a:ab:5a: 38 | 86:cd:57:6a:5f:13:c0:ae:7e:ad:5c:6e:c3:c4:e7:b7:d3:14: 39 | bf:86:fe:f2:d1:70:0e:fc:98:50:a7:fe:53:62:5a:2d:f5:63: 40 | 2c:ee:4a:7c:dd:32:3e:d1:52:3a:1f:15:38:4b:2a:4a:ee:27: 41 | a9:d8:92:a8:33:92:83:c9:3a:09:5a:01:66:0e:68:da:8f:82: 42 | c0:18:cc:78:ea:c5:db:09:7c:2f:61:c3:51:f8:58:7a:27:d7: 43 | 92:c0:ff:f8:29:d7:a0:e9:54:17:8d:48:a8:ff:5e:92:ee:81: 44 | 6c:37:90:1c:93:28:8c:d2:f5:b1:20:96:d3:1d:0f:c0:7f:db: 45 | 0c:6d:65:7f:3a:55:e5:c9:9a:ad:09:91:a5:57:cb:fc:bf:df: 46 | 69:bd:6b:87:94:5b:d0:cf:3b:8b:48:41:3d:56:b6:1d:3f:e7: 47 | f6:b6:58:f7:54:2a:dd:da:60:68:db:9b:70:04:8b:19:c3:44: 48 | bf:1d:b4:28:b9:f8:ea:ad:d3:1a:6e:64:72:b1:61:6a:f3:e1: 49 | d4:68:56:7b:0e:ad:4c:53:1e:d2:2e:1c:bc:b7:82:59:af:65: 50 | d2:fd:ef:89:7c:34:8f:51:a1:4e:9d:7e:dc:c7:97:68:ea:aa: 51 | e5:67:ed:be:dc:38:74:0e:c3:6f:fd:08:62:54:d8:1f:15:d1: 52 | 25:fc:21:f6:8c:f9:2f:65:5e:07:b9:e9:56:ba:48:14:5c:0d: 53 | 18:ba:f8:83:54:5b:b6:27:0c:36:2c:20:29:9c:c2:68:c5:3a: 54 | 0f:a5:d6:5f:7c:aa:f9:a6:2a:2b:69:c5:b1:39:e7:1c:02:31: 55 | 5b:f5:82:de:c9:4e:8d:33:dc:94:02:44:0a:44:95:75:7b:a1: 56 | e7:ee:92:fc:35:93:73:8c:22:c1:32:ea:39:17:ca:d0:87:fc: 57 | 4d:8e:04:f8:59:66:d3:14:3f:59:ad:76:14:20:16:7b:77:4f: 58 | 94:58:f8:85:5c:ba:b3:69:ed:7f:75:54:9a:1a:88:21:5d:04: 59 | 57:87:85:e2:d4:0e:1b:61:7f:5d:36:dc:72:a1:9d:0b:c8:ce: 60 | 19:69:49:fa:1b:bb:3f:3d:1b:4d:81:42:95:4e:d8:0b:04:d1: 61 | 08:6d:15:b3:ae:52:41:12:ff:e1:90:c4:7d:52:88:55:8b:87: 62 | 83:06:48:8b:fc:3a:a7:47:0e:6c:a8:4c:9e:b0:aa:da:50:f5: 63 | 97:97:98:3e:9d:18:ef:43 64 | -----BEGIN CERTIFICATE----- 65 | MIIEqzCCApMCAhABMA0GCSqGSIb3DQEBCwUAMIGmMQswCQYDVQQGEwJVUzETMBEG 66 | A1UECAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpB 67 | cGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYD 68 | VQQDDAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hl 69 | Lm9yZzAeFw0yMTAyMTcxNjU2NTVaFw00MTAyMTIxNjU2NTVaMIGOMQswCQYDVQQG 70 | EwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEjMCEGA1UECgwaQXBhY2hlIFNvZnR3 71 | YXJlIEZvdW5kYXRpb24xDzANBgNVBAsMBlB1bHNhcjEOMAwGA1UEAwwFYWRtaW4x 72 | JDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCCASIwDQYJKoZI 73 | hvcNAQEBBQADggEPADCCAQoCggEBAKth9RKx4a4ZAT5ZSsbKAAyW6HY6gyDZrzrh 74 | ESAS4OTQcI9Le6/hie+bxanC7a4kjbtCbuxZET/1Y1lhGJ9wtnaI4sp5Fcz7nF5c 75 | u6HX8NgR1Bc0HoF+Cw4Fvl361kav4ZXYoF3FL9mpj2lkSZX3QhZqhCsur5FzPbbU 76 | RFaaYUNJFSKukF0EKZBOskE0cz6iSAUcvI4bC8HV31YyQOmRonveMStn8Y7WxcCH 77 | V3Ap+a/bV6AujDAKp0c5M0zXLTKqSCm9xEjFWFIHxJmxzGbarChNwbwfRD+jY2G9 78 | /0hhdgSyfRxunO6Cu/dgHHqgmL4tcEMvZL/SDyAl98d9cAW4Lr8CAwEAATANBgkq 79 | hkiG9w0BAQsFAAOCAgEAHDG4D6EDKKDaMew0zuD9AZmdm634A10ghRjeyrXqYck7 80 | ZUKc5SFz0gZBS6k6+3//RfNaSqtahs1Xal8TwK5+rVxuw8Tnt9MUv4b+8tFwDvyY 81 | UKf+U2JaLfVjLO5KfN0yPtFSOh8VOEsqSu4nqdiSqDOSg8k6CVoBZg5o2o+CwBjM 82 | eOrF2wl8L2HDUfhYeifXksD/+CnXoOlUF41IqP9eku6BbDeQHJMojNL1sSCW0x0P 83 | wH/bDG1lfzpV5cmarQmRpVfL/L/fab1rh5Rb0M87i0hBPVa2HT/n9rZY91Qq3dpg 84 | aNubcASLGcNEvx20KLn46q3TGm5kcrFhavPh1GhWew6tTFMe0i4cvLeCWa9l0v3v 85 | iXw0j1GhTp1+3MeXaOqq5Wftvtw4dA7Db/0IYlTYHxXRJfwh9oz5L2VeB7npVrpI 86 | FFwNGLr4g1RbticMNiwgKZzCaMU6D6XWX3yq+aYqK2nFsTnnHAIxW/WC3slOjTPc 87 | lAJECkSVdXuh5+6S/DWTc4wiwTLqORfK0If8TY4E+Flm0xQ/Wa12FCAWe3dPlFj4 88 | hVy6s2ntf3VUmhqIIV0EV4eF4tQOG2F/XTbccqGdC8jOGWlJ+hu7Pz0bTYFClU7Y 89 | CwTRCG0Vs65SQRL/4ZDEfVKIVYuHgwZIi/w6p0cObKhMnrCq2lD1l5eYPp0Y70M= 90 | -----END CERTIFICATE----- 91 | -------------------------------------------------------------------------------- /pulsar/asyncio.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | """ 21 | The Pulsar Python client APIs that work with the asyncio module. 22 | """ 23 | 24 | import asyncio 25 | import functools 26 | from typing import Any 27 | 28 | import _pulsar 29 | import pulsar 30 | 31 | class PulsarException(BaseException): 32 | """ 33 | The exception that wraps the Pulsar error code 34 | """ 35 | 36 | def __init__(self, result: pulsar.Result) -> None: 37 | """ 38 | Create the Pulsar exception. 39 | 40 | Parameters 41 | ---------- 42 | result: pulsar.Result 43 | The error code of the underlying Pulsar APIs. 44 | """ 45 | self._result = result 46 | 47 | def error(self) -> pulsar.Result: 48 | """ 49 | Returns the Pulsar error code. 50 | """ 51 | return self._result 52 | 53 | def __str__(self): 54 | """ 55 | Convert the exception to string. 56 | """ 57 | return f'{self._result.value} {self._result.name}' 58 | 59 | class Producer: 60 | """ 61 | The Pulsar message producer, used to publish messages on a topic. 62 | """ 63 | 64 | def __init__(self, producer: _pulsar.Producer) -> None: 65 | """ 66 | Create the producer. 67 | Users should not call this constructor directly. Instead, create the 68 | producer via `Client.create_producer`. 69 | 70 | Parameters 71 | ---------- 72 | producer: _pulsar.Producer 73 | The underlying Producer object from the C extension. 74 | """ 75 | self._producer: _pulsar.Producer = producer 76 | 77 | async def send(self, content: bytes) -> pulsar.MessageId: 78 | """ 79 | Send a message asynchronously. 80 | 81 | parameters 82 | ---------- 83 | content: bytes 84 | The message payload 85 | 86 | Returns 87 | ------- 88 | pulsar.MessageId 89 | The message id that represents the persisted position of the message. 90 | 91 | Raises 92 | ------ 93 | PulsarException 94 | """ 95 | builder = _pulsar.MessageBuilder() 96 | builder.content(content) 97 | future = asyncio.get_running_loop().create_future() 98 | self._producer.send_async(builder.build(), functools.partial(_set_future, future)) 99 | msg_id = await future 100 | return pulsar.MessageId( 101 | msg_id.partition(), 102 | msg_id.ledger_id(), 103 | msg_id.entry_id(), 104 | msg_id.batch_index(), 105 | ) 106 | 107 | async def close(self) -> None: 108 | """ 109 | Close the producer. 110 | 111 | Raises 112 | ------ 113 | PulsarException 114 | """ 115 | future = asyncio.get_running_loop().create_future() 116 | self._producer.close_async(functools.partial(_set_future, future, value=None)) 117 | await future 118 | 119 | class Client: 120 | """ 121 | The asynchronous version of `pulsar.Client`. 122 | """ 123 | 124 | def __init__(self, service_url, **kwargs) -> None: 125 | """ 126 | See `pulsar.Client.__init__` 127 | """ 128 | self._client: _pulsar.Client = pulsar.Client(service_url, **kwargs)._client 129 | 130 | async def create_producer(self, topic: str) -> Producer: 131 | """ 132 | Create a new producer on a given topic 133 | 134 | Parameters 135 | ---------- 136 | topic: str 137 | The topic name 138 | 139 | Returns 140 | ------- 141 | Producer 142 | The producer created 143 | 144 | Raises 145 | ------ 146 | PulsarException 147 | """ 148 | future = asyncio.get_running_loop().create_future() 149 | conf = _pulsar.ProducerConfiguration() 150 | # TODO: add more configs 151 | self._client.create_producer_async(topic, conf, functools.partial(_set_future, future)) 152 | return Producer(await future) 153 | 154 | async def close(self) -> None: 155 | """ 156 | Close the client and all the associated producers and consumers 157 | 158 | Raises 159 | ------ 160 | PulsarException 161 | """ 162 | future = asyncio.get_running_loop().create_future() 163 | self._client.close_async(functools.partial(_set_future, future, value=None)) 164 | await future 165 | 166 | def _set_future(future: asyncio.Future, result: _pulsar.Result, value: Any): 167 | def complete(): 168 | if result == _pulsar.Result.Ok: 169 | future.set_result(value) 170 | else: 171 | future.set_exception(PulsarException(result)) 172 | future.get_loop().call_soon_threadsafe(complete) 173 | -------------------------------------------------------------------------------- /src/consumer.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace py = pybind11; 26 | 27 | void Consumer_unsubscribe(Consumer& consumer) { 28 | waitForAsyncResult([&consumer](ResultCallback callback) { consumer.unsubscribeAsync(callback); }); 29 | } 30 | 31 | Message Consumer_receive(Consumer& consumer) { 32 | return waitForAsyncValue([&](ReceiveCallback callback) { consumer.receiveAsync(callback); }); 33 | } 34 | 35 | Message Consumer_receive_timeout(Consumer& consumer, int timeoutMs) { 36 | Message msg; 37 | Result res; 38 | Py_BEGIN_ALLOW_THREADS res = consumer.receive(msg, timeoutMs); 39 | Py_END_ALLOW_THREADS 40 | 41 | CHECK_RESULT(res); 42 | return msg; 43 | } 44 | 45 | Messages Consumer_batch_receive(Consumer& consumer) { 46 | Messages msgs; 47 | Result res; 48 | Py_BEGIN_ALLOW_THREADS res = consumer.batchReceive(msgs); 49 | Py_END_ALLOW_THREADS CHECK_RESULT(res); 50 | return msgs; 51 | } 52 | 53 | void Consumer_acknowledge(Consumer& consumer, const Message& msg) { 54 | waitForAsyncResult([&](ResultCallback callback) { consumer.acknowledgeAsync(msg, callback); }); 55 | } 56 | 57 | void Consumer_acknowledge_message_id(Consumer& consumer, const MessageId& msgId) { 58 | waitForAsyncResult([&](ResultCallback callback) { consumer.acknowledgeAsync(msgId, callback); }); 59 | } 60 | 61 | void Consumer_negative_acknowledge(Consumer& consumer, const Message& msg) { 62 | Py_BEGIN_ALLOW_THREADS consumer.negativeAcknowledge(msg); 63 | Py_END_ALLOW_THREADS 64 | } 65 | 66 | void Consumer_negative_acknowledge_message_id(Consumer& consumer, const MessageId& msgId) { 67 | Py_BEGIN_ALLOW_THREADS consumer.negativeAcknowledge(msgId); 68 | Py_END_ALLOW_THREADS 69 | } 70 | 71 | void Consumer_acknowledge_cumulative(Consumer& consumer, const Message& msg) { 72 | waitForAsyncResult([&](ResultCallback callback) { consumer.acknowledgeCumulativeAsync(msg, callback); }); 73 | } 74 | 75 | void Consumer_acknowledge_cumulative_message_id(Consumer& consumer, const MessageId& msgId) { 76 | waitForAsyncResult( 77 | [&](ResultCallback callback) { consumer.acknowledgeCumulativeAsync(msgId, callback); }); 78 | } 79 | 80 | void Consumer_close(Consumer& consumer) { 81 | waitForAsyncResult([&consumer](ResultCallback callback) { consumer.closeAsync(callback); }); 82 | } 83 | 84 | void Consumer_pauseMessageListener(Consumer& consumer) { CHECK_RESULT(consumer.pauseMessageListener()); } 85 | 86 | void Consumer_resumeMessageListener(Consumer& consumer) { CHECK_RESULT(consumer.resumeMessageListener()); } 87 | 88 | void Consumer_seek(Consumer& consumer, const MessageId& msgId) { 89 | waitForAsyncResult([msgId, &consumer](ResultCallback callback) { consumer.seekAsync(msgId, callback); }); 90 | } 91 | 92 | void Consumer_seek_timestamp(Consumer& consumer, uint64_t timestamp) { 93 | waitForAsyncResult( 94 | [timestamp, &consumer](ResultCallback callback) { consumer.seekAsync(timestamp, callback); }); 95 | } 96 | 97 | bool Consumer_is_connected(Consumer& consumer) { return consumer.isConnected(); } 98 | 99 | MessageId Consumer_get_last_message_id(Consumer& consumer) { 100 | MessageId msgId; 101 | Result res; 102 | Py_BEGIN_ALLOW_THREADS res = consumer.getLastMessageId(msgId); 103 | Py_END_ALLOW_THREADS 104 | 105 | CHECK_RESULT(res); 106 | return msgId; 107 | } 108 | 109 | void export_consumer(py::module_& m) { 110 | py::class_(m, "Consumer") 111 | .def(py::init<>()) 112 | .def("topic", &Consumer::getTopic, "return the topic this consumer is subscribed to", 113 | py::return_value_policy::copy) 114 | .def("subscription_name", &Consumer::getSubscriptionName, py::return_value_policy::copy) 115 | .def("consumer_name", &Consumer::getConsumerName, py::return_value_policy::copy) 116 | .def("unsubscribe", &Consumer_unsubscribe) 117 | .def("receive", &Consumer_receive) 118 | .def("receive", &Consumer_receive_timeout) 119 | .def("batch_receive", &Consumer_batch_receive) 120 | .def("acknowledge", &Consumer_acknowledge) 121 | .def("acknowledge", &Consumer_acknowledge_message_id) 122 | .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative) 123 | .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative_message_id) 124 | .def("negative_acknowledge", &Consumer_negative_acknowledge) 125 | .def("negative_acknowledge", &Consumer_negative_acknowledge_message_id) 126 | .def("close", &Consumer_close) 127 | .def("pause_message_listener", &Consumer_pauseMessageListener) 128 | .def("resume_message_listener", &Consumer_resumeMessageListener) 129 | .def("redeliver_unacknowledged_messages", &Consumer::redeliverUnacknowledgedMessages) 130 | .def("seek", &Consumer_seek) 131 | .def("seek", &Consumer_seek_timestamp) 132 | .def("is_connected", &Consumer_is_connected) 133 | .def("get_last_message_id", &Consumer_get_last_message_id); 134 | } 135 | -------------------------------------------------------------------------------- /src/message.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace py = pybind11; 27 | 28 | void export_message(py::module_& m) { 29 | using namespace py; 30 | 31 | PyDateTime_IMPORT; 32 | 33 | MessageBuilder& (MessageBuilder::*MessageBuilderSetContentString)(const std::string&) = 34 | &MessageBuilder::setContent; 35 | 36 | class_(m, "MessageBuilder") 37 | .def(init<>()) 38 | .def("content", MessageBuilderSetContentString, return_value_policy::reference) 39 | .def("property", &MessageBuilder::setProperty, return_value_policy::reference) 40 | .def("properties", &MessageBuilder::setProperties, return_value_policy::reference) 41 | .def("sequence_id", &MessageBuilder::setSequenceId, return_value_policy::reference) 42 | .def("deliver_after", &MessageBuilder::setDeliverAfter, return_value_policy::reference) 43 | .def("deliver_at", &MessageBuilder::setDeliverAt, return_value_policy::reference) 44 | .def("partition_key", &MessageBuilder::setPartitionKey, return_value_policy::reference) 45 | .def("ordering_key", &MessageBuilder::setOrderingKey, return_value_policy::reference) 46 | .def("event_timestamp", &MessageBuilder::setEventTimestamp, return_value_policy::reference) 47 | .def("replication_clusters", &MessageBuilder::setReplicationClusters, return_value_policy::reference) 48 | .def("disable_replication", &MessageBuilder::disableReplication, return_value_policy::reference) 49 | .def("build", &MessageBuilder::build); 50 | 51 | class_(m, "MessageId") 52 | .def(init()) 53 | .def("__str__", 54 | [](const MessageId& msgId) { 55 | std::ostringstream oss; 56 | oss << msgId; 57 | return oss.str(); 58 | }) 59 | .def("__repr__", 60 | [](const MessageId& msgId) { 61 | std::ostringstream oss; 62 | oss << msgId; 63 | return oss.str(); 64 | }) 65 | .def("__eq__", &MessageId::operator==) 66 | .def("__ne__", &MessageId::operator!=) 67 | .def("__le__", &MessageId::operator<=) 68 | .def("__lt__", &MessageId::operator<) 69 | .def("__ge__", &MessageId::operator>=) 70 | .def("__gt__", &MessageId::operator>) 71 | .def("ledger_id", &MessageId::ledgerId) 72 | .def("entry_id", &MessageId::entryId) 73 | .def("batch_index", &MessageId::batchIndex) 74 | .def("partition", &MessageId::partition) 75 | .def( 76 | "topic_name", 77 | [](MessageId& msgId, const std::string& topicName) { msgId.setTopicName(topicName); }, 78 | return_value_policy::copy) 79 | .def_property_readonly_static("earliest", [](object) { return MessageId::earliest(); }) 80 | .def_property_readonly_static("latest", [](object) { return MessageId::latest(); }) 81 | .def("serialize", 82 | [](const MessageId& msgId) { 83 | std::string serialized; 84 | msgId.serialize(serialized); 85 | return bytes(serialized); 86 | }) 87 | .def_static("deserialize", &MessageId::deserialize); 88 | 89 | class_(m, "Message") 90 | .def(init<>()) 91 | .def("properties", &Message::getProperties) 92 | .def("data", [](const Message& msg) { return bytes(msg.getDataAsString()); }) 93 | .def("length", &Message::getLength) 94 | .def("partition_key", &Message::getPartitionKey, return_value_policy::copy) 95 | .def("ordering_key", &Message::getOrderingKey, return_value_policy::copy) 96 | .def("publish_timestamp", &Message::getPublishTimestamp) 97 | .def("event_timestamp", &Message::getEventTimestamp) 98 | .def("message_id", &Message::getMessageId, return_value_policy::copy) 99 | .def("__str__", 100 | [](const Message& msg) { 101 | std::ostringstream oss; 102 | oss << msg; 103 | return oss.str(); 104 | }) 105 | .def("topic_name", &Message::getTopicName, return_value_policy::copy) 106 | .def("redelivery_count", &Message::getRedeliveryCount) 107 | .def("int_schema_version", &Message::getLongSchemaVersion) 108 | .def("schema_version", &Message::getSchemaVersion, return_value_policy::copy) 109 | .def("producer_name", &Message::getProducerName, return_value_policy::copy); 110 | 111 | MessageBatch& (MessageBatch::*MessageBatchParseFromString)(const std::string& payload, 112 | uint32_t batchSize) = &MessageBatch::parseFrom; 113 | 114 | class_(m, "MessageBatch") 115 | .def(init<>()) 116 | .def("with_message_id", &MessageBatch::withMessageId, return_value_policy::reference) 117 | .def("parse_from", MessageBatchParseFromString, return_value_policy::reference) 118 | .def("messages", &MessageBatch::messages, return_value_policy::copy); 119 | } 120 | -------------------------------------------------------------------------------- /pulsar/schema/schema_avro.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | import _pulsar 21 | import io 22 | import json 23 | import logging 24 | import enum 25 | 26 | from . import Record 27 | from .schema import Schema 28 | 29 | try: 30 | import fastavro 31 | HAS_AVRO = True 32 | except ImportError: 33 | HAS_AVRO = False 34 | 35 | if HAS_AVRO: 36 | class AvroSchema(Schema): 37 | def __init__(self, record_cls, schema_definition=None): 38 | if record_cls is None and schema_definition is None: 39 | raise AssertionError("The param record_cls and schema_definition shouldn't be both None.") 40 | 41 | if record_cls is not None: 42 | self._schema = record_cls.schema() 43 | else: 44 | self._schema = schema_definition 45 | self._writer_schemas = dict() 46 | self._logger = logging.getLogger() 47 | super(AvroSchema, self).__init__(record_cls, _pulsar.SchemaType.AVRO, self._schema, 'AVRO') 48 | 49 | def _get_serialized_value(self, x): 50 | if isinstance(x, enum.Enum): 51 | return x.name 52 | elif isinstance(x, Record): 53 | return self.encode_dict(x.__dict__) 54 | elif isinstance(x, list): 55 | arr = [] 56 | for item in x: 57 | arr.append(self._get_serialized_value(item)) 58 | return arr 59 | elif isinstance(x, dict): 60 | return self.encode_dict(x) 61 | else: 62 | return x 63 | 64 | def encode(self, obj): 65 | buffer = io.BytesIO() 66 | m = obj 67 | if self._record_cls is not None: 68 | self._validate_object_type(obj) 69 | m = self.encode_dict(obj.__dict__) 70 | elif not isinstance(obj, dict): 71 | raise ValueError('If using the custom schema, the record data should be dict type.') 72 | 73 | fastavro.schemaless_writer(buffer, self._schema, m) 74 | return buffer.getvalue() 75 | 76 | def encode_dict(self, d): 77 | obj = {} 78 | for k, v in d.items(): 79 | obj[k] = self._get_serialized_value(v) 80 | return obj 81 | 82 | def decode(self, data): 83 | return self._decode_bytes(data, self._schema) 84 | 85 | def decode_message(self, msg: _pulsar.Message): 86 | if self._client is None: 87 | return self.decode(msg.data()) 88 | topic = msg.topic_name() 89 | version = msg.int_schema_version() 90 | try: 91 | writer_schema = self._get_writer_schema(topic, version) 92 | return self._decode_bytes(msg.data(), writer_schema) 93 | except Exception as e: 94 | msg_id = msg.message_id() 95 | self._logger.warn(f'Failed to decode {msg_id} with schema {topic} version {version}: {e}') 96 | return self._decode_bytes(msg.data(), self._schema) 97 | 98 | def _get_writer_schema(self, topic: str, version: int) -> 'dict': 99 | if self._writer_schemas.get(topic) is None: 100 | self._writer_schemas[topic] = dict() 101 | writer_schema = self._writer_schemas[topic].get(version) 102 | if writer_schema is not None: 103 | return writer_schema 104 | if self._client is None: 105 | return self._schema 106 | 107 | self._logger.info('Downloading schema of %s version %d...', topic, version) 108 | info = self._client.get_schema_info(topic, version) 109 | self._logger.info('Downloaded schema of %s version %d', topic, version) 110 | if info.schema_type() != _pulsar.SchemaType.AVRO: 111 | raise RuntimeError(f'The schema type of topic "{topic}" and version {version}' 112 | f' is {info.schema_type()}') 113 | writer_schema = json.loads(info.schema()) 114 | self._writer_schemas[topic][version] = writer_schema 115 | return writer_schema 116 | 117 | def _decode_bytes(self, data: bytes, writer_schema: dict): 118 | buffer = io.BytesIO(data) 119 | # If the record names are different between the writer schema and the reader schema, 120 | # schemaless_reader will fail with fastavro._read_common.SchemaResolutionError. 121 | # So we make the record name fields consistent here. 122 | reader_schema: dict = self._schema 123 | writer_schema['name'] = reader_schema['name'] 124 | d = fastavro.schemaless_reader(buffer, writer_schema, reader_schema) 125 | if self._record_cls is not None: 126 | return self._record_cls(**d) 127 | else: 128 | return d 129 | 130 | else: 131 | class AvroSchema(Schema): 132 | def __init__(self, _record_cls, _schema_definition=None): 133 | raise Exception("Avro library support was not found. Make sure to install Pulsar client " + 134 | "with Avro support: pip3 install 'pulsar-client[avro]'") 135 | 136 | def encode(self, obj): 137 | pass 138 | 139 | def decode(self, data): 140 | pass 141 | -------------------------------------------------------------------------------- /pkg/mac/build-mac-wheels.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | set -e -x 22 | 23 | if [[ $# -ne 2 ]]; then 24 | echo "Usage: $0 " 25 | exit 1 26 | fi 27 | 28 | ROOT_DIR=$(git rev-parse --show-toplevel) 29 | cd "${ROOT_DIR}" 30 | 31 | source build-support/dep-url.sh 32 | 33 | CACHE_DIR=$ROOT_DIR/.pulsar-mac-build 34 | PREFIX=${CACHE_DIR}/install 35 | mkdir -p $PREFIX 36 | 37 | mkdir -p $PREFIX/lib/ 38 | if [ ! -f $PREFIX/lib/libpulsarwithdeps.a ]; then 39 | VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') 40 | curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-arm64.zip 41 | curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-x86_64.zip 42 | 43 | unzip -q macos-arm64.zip -d arm64 44 | unzip -q macos-x86_64.zip -d x86_64 45 | libtool -static -o libpulsarwithdeps.a arm64/lib/libpulsarwithdeps.a x86_64/lib/libpulsarwithdeps.a 46 | 47 | mv arm64/include/ $PREFIX/ 48 | mv libpulsarwithdeps.a $PREFIX/lib/ 49 | rm -rf arm64/ x86_64/ macos-arm64.zip macos-x86_64.zip 50 | fi 51 | 52 | PYTHON_VERSION=$1 53 | PYTHON_VERSION_LONG=$2 54 | 55 | # When building Python from source, it will read this environment variable to determine the minimum supported macOS version 56 | export MACOSX_DEPLOYMENT_TARGET=13 57 | pushd $CACHE_DIR 58 | 59 | # We need to build OpenSSL from source to have universal2 binaries 60 | OPENSSL_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep openssl | awk '{print $2}') 61 | OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') 62 | if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then 63 | echo "Building OpenSSL" 64 | download_dependency $ROOT_DIR/dependencies.yaml openssl 65 | tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz 66 | 67 | mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 68 | pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 69 | echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c 70 | CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ 71 | ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc 72 | make -j8 73 | make install_sw 74 | popd 75 | 76 | tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz 77 | mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 78 | pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 79 | echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c 80 | CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ 81 | ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc 82 | make -j8 83 | make install_sw 84 | popd 85 | 86 | # Create universal binaries 87 | lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ 88 | -output $PREFIX/lib/libssl.a 89 | lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ 90 | -output $PREFIX/lib/libcrypto.a 91 | 92 | touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done 93 | else 94 | echo "Using cached OpenSSL" 95 | fi 96 | 97 | if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then 98 | echo "Building Python $PYTHON_VERSION_LONG" 99 | curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz 100 | tar xfz Python-${PYTHON_VERSION_LONG}.tgz 101 | 102 | pushd Python-${PYTHON_VERSION_LONG} 103 | export CFLAGS="-fPIC -O3" 104 | ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 --with-openssl=$PREFIX 105 | make -j16 106 | make install 107 | 108 | curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl 109 | export SSL_CERT_FILE=/etc/ssl/cert.pem 110 | $PREFIX/bin/pip3 install wheel setuptools 111 | $PREFIX/bin/pip3 install wheel-*.whl 112 | 113 | touch .done 114 | popd 115 | else 116 | echo "Using cached Python $PYTHON_VERSION_LONG" 117 | fi 118 | 119 | PYBIND11_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep pybind11 | awk '{print $2}') 120 | if [ ! -f pybind11/.done ]; then 121 | download_dependency $ROOT_DIR/dependencies.yaml pybind11 122 | mkdir -p $PREFIX/include/ 123 | cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/ 124 | mkdir -p pybind11 125 | touch pybind11/.done 126 | fi 127 | 128 | popd # $CACHE_DIR 129 | 130 | PYTHON_CLIENT_VERSION=$(grep -v '^#' pulsar/__about__.py | cut -d "=" -f2 | sed "s/'//g") 131 | 132 | echo "Build wheel for Python $PYTHON_VERSION" 133 | 134 | PY_EXE=$PREFIX/bin/python3 135 | PIP_EXE=$PREFIX/bin/pip3 136 | 137 | ARCHS='arm64;x86_64' 138 | PIP_TAG='universal2' 139 | 140 | cmake -B build \ 141 | -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ 142 | -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ 143 | -DCMAKE_BUILD_TYPE=Release \ 144 | -DCMAKE_PREFIX_PATH=$PREFIX \ 145 | -DLINK_STATIC=ON \ 146 | -DPython3_ROOT_DIR=$PREFIX 147 | cmake --build build --config Release -j8 148 | cp -f build/lib_pulsar.so . 149 | 150 | $PY_EXE setup.py bdist_wheel 151 | 152 | PY_SPEC=$(echo $PYTHON_VERSION | sed 's/\.//g') 153 | 154 | cd /tmp 155 | $PIP_EXE install --no-dependencies --force-reinstall \ 156 | $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx*_${PIP_TAG}.whl 157 | $PY_EXE -c 'import pulsar' 158 | -------------------------------------------------------------------------------- /.github/workflows/ci-build-release-wheels.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: Build release wheels 21 | on: 22 | push: 23 | tags: 24 | - '*' 25 | 26 | concurrency: 27 | group: ${{ github.workflow }}-${{ github.ref }} 28 | cancel-in-progress: true 29 | 30 | jobs: 31 | 32 | linux-wheel: 33 | name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} 34 | runs-on: ubuntu-latest 35 | timeout-minutes: 300 36 | 37 | strategy: 38 | fail-fast: false 39 | matrix: 40 | image: 41 | - {name: 'manylinux', py_suffix: ''} 42 | - {name: 'manylinux_musl', py_suffix: '-alpine'} 43 | python: 44 | - {version: '3.10', spec: 'cp310-cp310'} 45 | - {version: '3.11', spec: 'cp311-cp311'} 46 | - {version: '3.12', spec: 'cp312-cp312'} 47 | - {version: '3.13', spec: 'cp313-cp313'} 48 | - {version: '3.14', spec: 'cp314-cp314'} 49 | cpu: 50 | - {arch: 'x86_64', platform: 'x86_64'} 51 | - {arch: 'aarch64', platform: 'arm64'} 52 | 53 | steps: 54 | - name: checkout 55 | uses: actions/checkout@v3 56 | 57 | - name: Set up QEMU 58 | uses: docker/setup-qemu-action@v2 59 | 60 | - uses: docker/setup-buildx-action@v2 61 | 62 | - name: Build Manylinux Docker image 63 | uses: docker/build-push-action@v3 64 | with: 65 | context: ./pkg/${{matrix.image.name}} 66 | load: true 67 | tags: build:latest 68 | platforms: linux/${{matrix.cpu.arch}} 69 | build-args: | 70 | PLATFORM=${{matrix.cpu.platform}} 71 | ARCH=${{matrix.cpu.arch}} 72 | PYTHON_VERSION=${{matrix.python.version}} 73 | PYTHON_SPEC=${{matrix.python.spec}} 74 | cache-from: type=gha 75 | cache-to: type=gha,mode=max 76 | 77 | - name: Build wheel file 78 | run: | 79 | docker run -i -v $PWD:/pulsar-client-python build:latest \ 80 | /pulsar-client-python/pkg/build-wheel-inside-docker.sh 81 | 82 | - name: Test wheel file 83 | run: | 84 | docker run -i -v $PWD:/pulsar-client-python \ 85 | --platform linux/${{matrix.cpu.arch}} \ 86 | python:${{matrix.python.version}}${{matrix.image.py_suffix}} \ 87 | /pulsar-client-python/pkg/test-wheel.sh 88 | 89 | - name: Upload artifacts 90 | uses: actions/upload-artifact@v4 91 | with: 92 | name: wheel-${{matrix.image.name}}-py${{matrix.python.version}}-${{matrix.cpu.platform}} 93 | path: wheelhouse/*.whl 94 | 95 | mac-wheels: 96 | name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} 97 | runs-on: macos-14 98 | timeout-minutes: 300 99 | 100 | strategy: 101 | fail-fast: false 102 | matrix: 103 | py: 104 | - {version: '3.10', version_long: '3.10.15'} 105 | - {version: '3.11', version_long: '3.11.11'} 106 | - {version: '3.12', version_long: '3.12.8'} 107 | - {version: '3.13', version_long: '3.13.1'} 108 | - {version: '3.14', version_long: '3.14.0'} 109 | 110 | steps: 111 | - name: checkout 112 | uses: actions/checkout@v3 113 | 114 | - name: Build and test Mac wheels 115 | run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} 116 | 117 | - name: Upload artifacts 118 | uses: actions/upload-artifact@v4 119 | with: 120 | name: wheel-mac-py${{matrix.py.version}} 121 | path: dist/*.whl 122 | 123 | windows-wheels: 124 | name: Wheel Windows - Py ${{matrix.python.version}} 125 | runs-on: windows-2022 126 | env: 127 | PULSAR_CPP_DIR: 'C:\\pulsar-cpp' 128 | timeout-minutes: 300 129 | 130 | strategy: 131 | fail-fast: false 132 | matrix: 133 | python: 134 | - {version: '3.10'} 135 | - {version: '3.11'} 136 | - {version: '3.12'} 137 | - {version: '3.13'} 138 | - {version: '3.14'} 139 | 140 | steps: 141 | - uses: actions/checkout@v3 142 | 143 | - uses: actions/setup-python@v4 144 | with: 145 | python-version: ${{ matrix.python.version }} 146 | 147 | - name: Download Pulsar C++ client on Windows 148 | shell: bash 149 | run: | 150 | source ./build-support/dep-url.sh 151 | BASE_URL=$(pulsar_cpp_base_url $(grep pulsar-cpp dependencies.yaml | awk '{print $2}')) 152 | mkdir -p ${{ env.PULSAR_CPP_DIR }} 153 | cd ${{ env.PULSAR_CPP_DIR }} 154 | curl -O -L ${BASE_URL}/x64-windows-static.tar.gz 155 | tar zxf x64-windows-static.tar.gz 156 | mv x64-windows-static/* . 157 | ls -l ${{ env.PULSAR_CPP_DIR }} 158 | 159 | - name: Configure CMake 160 | shell: bash 161 | run: | 162 | pip3 install pyyaml 163 | export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) 164 | curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz 165 | tar zxf v${PYBIND11_VERSION}.tar.gz 166 | rm -rf pybind11 167 | mv pybind11-${PYBIND11_VERSION} pybind11 168 | cmake -B build -A x64 \ 169 | -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \ 170 | -DLINK_STATIC=ON 171 | 172 | - name: Build Python wheel 173 | shell: bash 174 | run: | 175 | cmake --build build --config Release --target install 176 | python -m pip install wheel setuptools 177 | python setup.py bdist_wheel 178 | python -m pip install ./dist/*.whl 179 | python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' 180 | 181 | - name: Upload artifacts 182 | uses: actions/upload-artifact@v4 183 | with: 184 | name: wheel-windows-py${{matrix.python.version}} 185 | path: dist/*.whl 186 | -------------------------------------------------------------------------------- /pulsar/functions/context.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # -*- encoding: utf-8 -*- 21 | 22 | # Licensed to the Apache Software Foundation (ASF) under one 23 | # or more contributor license agreements. See the NOTICE file 24 | # distributed with this work for additional information 25 | # regarding copyright ownership. The ASF licenses this file 26 | # to you under the Apache License, Version 2.0 (the 27 | # "License"); you may not use this file except in compliance 28 | # with the License. You may obtain a copy of the License at 29 | # 30 | # http://www.apache.org/licenses/LICENSE-2.0 31 | # 32 | # Unless required by applicable law or agreed to in writing, 33 | # software distributed under the License is distributed on an 34 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 35 | # KIND, either express or implied. See the License for the 36 | # specific language governing permissions and limitations 37 | # under the License. 38 | # 39 | """ 40 | Context defines context information available during processing of a request. 41 | """ 42 | from abc import abstractmethod 43 | 44 | 45 | class Context(object): 46 | """Interface defining information available at process time""" 47 | 48 | @abstractmethod 49 | def get_message_id(self): 50 | """Return the messageid of the current message that we are processing""" 51 | pass 52 | 53 | @abstractmethod 54 | def get_message_key(self): 55 | """Return the key of the current message that we are processing""" 56 | pass 57 | 58 | @abstractmethod 59 | def get_message_eventtime(self): 60 | """Return the event time of the current message that we are processing""" 61 | pass 62 | 63 | @abstractmethod 64 | def get_message_properties(self): 65 | """Return the message properties kv map of the current message that we are processing""" 66 | pass 67 | 68 | @abstractmethod 69 | def get_current_message_topic_name(self): 70 | """Returns the topic name of the message that we are processing""" 71 | pass 72 | 73 | @abstractmethod 74 | def get_function_tenant(self): 75 | """Returns the tenant of the message that's being processed""" 76 | pass 77 | 78 | @abstractmethod 79 | def get_function_namespace(self): 80 | """Returns the namespace of the message that's being processed""" 81 | 82 | @abstractmethod 83 | def get_function_name(self): 84 | """Returns the function name that we are a part of""" 85 | pass 86 | 87 | @abstractmethod 88 | def get_function_id(self): 89 | """Returns the function id that we are a part of""" 90 | pass 91 | 92 | @abstractmethod 93 | def get_instance_id(self): 94 | """Returns the instance id that is executing the function""" 95 | pass 96 | 97 | @abstractmethod 98 | def get_function_version(self): 99 | """Returns the version of function that we are executing""" 100 | pass 101 | 102 | @abstractmethod 103 | def get_logger(self): 104 | """Returns the logger object that can be used to do logging""" 105 | pass 106 | 107 | @abstractmethod 108 | def get_user_config_value(self, key): 109 | """Returns the value of the user-defined config. If the key doesn't exist, None is returned""" 110 | pass 111 | 112 | @abstractmethod 113 | def get_user_config_map(self): 114 | """Returns the entire user-defined config as a dict 115 | (the dict will be empty if no user-defined config is supplied)""" 116 | pass 117 | 118 | @abstractmethod 119 | def get_secret(self, secret_name): 120 | """Returns the secret value associated with the name. None if nothing was found""" 121 | pass 122 | 123 | @abstractmethod 124 | def get_partition_key(self): 125 | """Returns partition key of the input message is one exists""" 126 | pass 127 | 128 | @abstractmethod 129 | def get_ordering_key(self): 130 | """Returns ordering key of the input message, if one exists""" 131 | pass 132 | 133 | @abstractmethod 134 | def record_metric(self, metric_name, metric_value): 135 | """Records the metric_value. metric_value has to satisfy isinstance(metric_value, numbers.Number)""" 136 | pass 137 | 138 | @abstractmethod 139 | def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, 140 | compression_type=None, callback=None, message_conf=None): 141 | """Publishes message to topic_name by first serializing the message using serde_class_name serde 142 | The message will have properties specified if any 143 | 144 | The available options for message_conf: 145 | 146 | properties, 147 | partition_key, 148 | ordering_key, 149 | sequence_id, 150 | replication_clusters, 151 | disable_replication, 152 | event_timestamp 153 | 154 | """ 155 | pass 156 | 157 | @abstractmethod 158 | def get_input_topics(self): 159 | """Returns the input topics of function""" 160 | pass 161 | 162 | @abstractmethod 163 | def get_output_topic(self): 164 | """Returns the output topic of function""" 165 | pass 166 | 167 | @abstractmethod 168 | def get_output_serde_class_name(self): 169 | """return output Serde class""" 170 | pass 171 | 172 | @abstractmethod 173 | def ack(self, msgid, topic): 174 | """ack this message id""" 175 | pass 176 | 177 | @abstractmethod 178 | def incr_counter(self, key, amount): 179 | """incr the counter of a given key in the managed state""" 180 | pass 181 | 182 | @abstractmethod 183 | def get_counter(self, key): 184 | """get the counter of a given key in the managed state""" 185 | pass 186 | 187 | @abstractmethod 188 | def del_counter(self, key): 189 | """delete the counter of a given key in the managed state""" 190 | pass 191 | 192 | @abstractmethod 193 | def put_state(self, key, value): 194 | """update the value of a given key in the managed state""" 195 | pass 196 | 197 | @abstractmethod 198 | def get_state(self, key): 199 | """get the value of a given key in the managed state""" 200 | pass 201 | -------------------------------------------------------------------------------- /tests/test-conf/broker-cert.pem: -------------------------------------------------------------------------------- 1 | Certificate: 2 | Data: 3 | Version: 3 (0x2) 4 | Serial Number: 4098 (0x1002) 5 | Signature Algorithm: sha256WithRSAEncryption 6 | Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org 7 | Validity 8 | Not Before: Feb 17 17:00:44 2021 GMT 9 | Not After : Feb 12 17:00:44 2041 GMT 10 | Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=localhost/emailAddress=dev@pulsar.apache.org 11 | Subject Public Key Info: 12 | Public Key Algorithm: rsaEncryption 13 | Public-Key: (2048 bit) 14 | Modulus: 15 | 00:9b:2a:6f:24:02:23:f7:ff:e6:75:61:ca:07:a8: 16 | c0:ab:e9:8d:eb:51:2e:64:f7:9e:9b:d4:b4:be:3a: 17 | fa:f4:6e:c6:92:8f:38:4d:08:cd:89:15:3e:2c:c4: 18 | 99:6d:cb:58:80:fc:e0:4d:d6:7d:f6:82:ab:0d:94: 19 | f2:e2:45:c9:d3:15:95:57:0a:6c:86:dc:78:64:3b: 20 | 34:4b:01:7c:5d:de:4f:d4:21:1a:5d:27:a0:a5:70: 21 | 7a:2e:02:50:e1:19:b4:b9:05:df:99:0d:8b:cc:62: 22 | dc:10:73:fa:72:8b:38:7f:d3:56:54:61:50:bb:92: 23 | ff:09:71:09:c7:bd:04:43:3c:8c:9c:8b:32:d1:05: 24 | 04:8a:c6:89:d8:78:56:4d:da:2f:f4:ec:34:37:26: 25 | b5:87:e4:3f:26:c9:41:60:ba:31:10:19:be:f8:0c: 26 | a4:0a:85:19:59:e2:00:5d:b7:c0:bd:d1:2e:fc:a6: 27 | 34:8b:85:2a:cc:05:f6:fb:e4:00:e6:74:95:ff:02: 28 | 6f:43:7f:39:a7:c2:83:8e:5b:38:40:c9:42:c8:bc: 29 | 26:72:36:35:64:c2:54:22:11:87:e8:65:8f:3d:e9: 30 | 41:a7:6d:19:88:9a:20:9b:9a:52:e7:d2:cb:b3:e0: 31 | 2e:8f:c1:56:54:bc:6d:14:30:73:c5:d7:8e:d0:5a: 32 | 5e:cd 33 | Exponent: 65537 (0x10001) 34 | X509v3 extensions: 35 | X509v3 Basic Constraints: 36 | CA:FALSE 37 | Netscape Cert Type: 38 | SSL Server 39 | Netscape Comment: 40 | OpenSSL Generated Server Certificate 41 | X509v3 Subject Key Identifier: 42 | 49:3C:B2:98:30:CE:7F:79:7A:C6:8B:57:CA:24:9F:12:82:1E:5D:EF 43 | X509v3 Authority Key Identifier: 44 | keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71 45 | DirName:/C=US/ST=California/L=Palo Alto/O=Apache Software Foundation/OU=Pulsar/CN=Pulsar CA/emailAddress=dev@pulsar.apache.org 46 | serial:52:7B:B4:00:96:60:B4:26:85:BE:01:82:B8:B8:E2:8C:72:EF:5B:90 47 | 48 | X509v3 Key Usage: critical 49 | Digital Signature, Key Encipherment 50 | X509v3 Extended Key Usage: 51 | TLS Web Server Authentication 52 | Signature Algorithm: sha256WithRSAEncryption 53 | 0f:bd:af:39:0c:2c:dc:8f:7e:06:0d:27:df:35:c7:8d:5a:03: 54 | 68:97:f6:dc:d6:d3:39:0e:b4:76:48:7d:e1:1c:a9:4b:83:fa: 55 | 52:00:ab:28:93:2d:06:76:0c:14:35:3c:f1:8e:3b:af:c8:d0: 56 | 27:1f:58:d4:71:22:5f:05:a6:9e:73:c6:a5:5e:2a:e6:fb:eb: 57 | fc:73:52:87:ca:8a:2a:f9:1e:5f:e2:b9:bd:01:27:9f:7c:61: 58 | a6:97:ad:a0:ab:4e:fb:cc:fa:c8:77:6a:65:1b:ae:60:5e:fb: 59 | 97:14:8c:40:d7:96:c6:2c:64:59:c0:52:52:7c:2d:98:4b:f4: 60 | 72:da:83:f7:c6:4f:32:42:ce:df:02:dd:5f:eb:58:42:f9:62: 61 | a1:9a:05:ef:13:48:27:af:a3:7f:23:eb:e0:dc:1d:8f:96:2a: 62 | 88:47:f7:e4:75:6f:a9:15:f6:44:f1:6d:39:3a:2c:df:a7:82: 63 | cc:7e:aa:9c:1c:c0:a7:7d:68:31:4a:4e:21:b8:9f:17:90:4b: 64 | f1:68:23:ef:a7:53:fc:a9:a8:35:6b:8f:4c:5e:d4:ea:b0:8a: 65 | 27:9a:86:89:ce:f2:5d:03:35:80:fc:45:e8:87:66:0f:32:b5: 66 | 2a:f5:1b:79:0e:09:8b:90:40:20:fb:e3:27:8a:c9:92:c1:53: 67 | 97:10:5a:8c:50:ef:02:46:7e:ec:68:c8:1e:26:66:0e:1d:d6: 68 | 6c:82:e7:38:14:e8:cb:45:77:29:5f:2c:1a:9d:d7:54:21:8a: 69 | cf:0f:b7:0c:ae:fe:d6:fb:fb:c3:07:3e:33:df:59:25:1c:73: 70 | d4:87:73:14:b4:76:16:8a:3f:82:05:7b:42:0a:55:0c:79:24: 71 | 3c:58:31:3f:e0:3e:9f:4e:d0:0e:fd:77:b7:13:2c:d3:d0:46: 72 | cc:80:09:0f:50:56:8b:6e:6e:91:b2:5b:c8:2f:4d:86:dc:72: 73 | 00:de:08:0d:5e:3e:96:1f:12:7d:3b:0d:4d:71:d5:c8:a8:06: 74 | ba:00:23:ec:10:4c:a4:c3:6f:bc:f0:d7:b1:cf:57:3f:3b:79: 75 | db:80:87:35:c7:4e:7f:bb:38:30:0a:9f:fe:5a:86:f5:97:ce: 76 | 24:38:79:fd:a0:dc:0b:82:11:a1:ea:0c:e9:16:65:e0:c0:54: 77 | 80:ad:6e:55:18:ac:27:35:3a:b0:20:70:62:8e:5d:a2:33:53: 78 | 8c:ce:f9:ee:a1:27:cb:db:e5:9a:5e:e6:f7:80:93:84:63:04: 79 | 26:58:ab:23:bb:94:80:d0:a0:55:a2:8a:ed:bc:0f:c3:41:d2: 80 | 26:a5:b9:8d:8a:45:e8:a1:fc:e8:ee:7a:64:93:ed:d6:ef:a2: 81 | 51:d7:c9:0a:31:39:35:4a 82 | -----BEGIN CERTIFICATE----- 83 | MIIGPDCCBCSgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwgaYxCzAJBgNVBAYTAlVT 84 | MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNV 85 | BAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIx 86 | EjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5h 87 | cGFjaGUub3JnMB4XDTIxMDIxNzE3MDA0NFoXDTQxMDIxMjE3MDA0NFowgZIxCzAJ 88 | BgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMSMwIQYDVQQKDBpBcGFjaGUg 89 | U29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQDDAls 90 | b2NhbGhvc3QxJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCC 91 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJsqbyQCI/f/5nVhygeowKvp 92 | jetRLmT3npvUtL46+vRuxpKPOE0IzYkVPizEmW3LWID84E3WffaCqw2U8uJFydMV 93 | lVcKbIbceGQ7NEsBfF3eT9QhGl0noKVwei4CUOEZtLkF35kNi8xi3BBz+nKLOH/T 94 | VlRhULuS/wlxCce9BEM8jJyLMtEFBIrGidh4Vk3aL/TsNDcmtYfkPybJQWC6MRAZ 95 | vvgMpAqFGVniAF23wL3RLvymNIuFKswF9vvkAOZ0lf8Cb0N/OafCg45bOEDJQsi8 96 | JnI2NWTCVCIRh+hljz3pQadtGYiaIJuaUufSy7PgLo/BVlS8bRQwc8XXjtBaXs0C 97 | AwEAAaOCAYQwggGAMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCG 98 | SAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUw 99 | HQYDVR0OBBYEFEk8spgwzn95esaLV8oknxKCHl3vMIHmBgNVHSMEgd4wgduAFNKy 100 | PbGkfEhLNuGn3tj8upK6p8RxoYGspIGpMIGmMQswCQYDVQQGEwJVUzETMBEGA1UE 101 | CAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpBcGFj 102 | aGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQD 103 | DAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9y 104 | Z4IUUnu0AJZgtCaFvgGCuLjijHLvW5AwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQM 105 | MAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQAPva85DCzcj34GDSffNceN 106 | WgNol/bc1tM5DrR2SH3hHKlLg/pSAKsoky0GdgwUNTzxjjuvyNAnH1jUcSJfBaae 107 | c8alXirm++v8c1KHyooq+R5f4rm9ASeffGGml62gq077zPrId2plG65gXvuXFIxA 108 | 15bGLGRZwFJSfC2YS/Ry2oP3xk8yQs7fAt1f61hC+WKhmgXvE0gnr6N/I+vg3B2P 109 | liqIR/fkdW+pFfZE8W05Oizfp4LMfqqcHMCnfWgxSk4huJ8XkEvxaCPvp1P8qag1 110 | a49MXtTqsIonmoaJzvJdAzWA/EXoh2YPMrUq9Rt5DgmLkEAg++MnismSwVOXEFqM 111 | UO8CRn7saMgeJmYOHdZsguc4FOjLRXcpXywanddUIYrPD7cMrv7W+/vDBz4z31kl 112 | HHPUh3MUtHYWij+CBXtCClUMeSQ8WDE/4D6fTtAO/Xe3EyzT0EbMgAkPUFaLbm6R 113 | slvIL02G3HIA3ggNXj6WHxJ9Ow1NcdXIqAa6ACPsEEykw2+88Nexz1c/O3nbgIc1 114 | x05/uzgwCp/+Wob1l84kOHn9oNwLghGh6gzpFmXgwFSArW5VGKwnNTqwIHBijl2i 115 | M1OMzvnuoSfL2+WaXub3gJOEYwQmWKsju5SA0KBVoortvA/DQdImpbmNikXoofzo 116 | 7npkk+3W76JR18kKMTk1Sg== 117 | -----END CERTIFICATE----- 118 | -------------------------------------------------------------------------------- /src/enums.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "utils.h" 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace pulsar; 28 | namespace py = pybind11; 29 | 30 | void export_enums(py::module_& m) { 31 | using namespace py; 32 | 33 | enum_(m, "KeySharedMode") 34 | .value("AutoSplit", AUTO_SPLIT) 35 | .value("Sticky", STICKY); 36 | 37 | enum_(m, "PartitionsRoutingMode") 38 | .value("UseSinglePartition", ProducerConfiguration::UseSinglePartition) 39 | .value("RoundRobinDistribution", ProducerConfiguration::RoundRobinDistribution) 40 | .value("CustomPartition", ProducerConfiguration::CustomPartition); 41 | 42 | enum_(m, "CompressionType") 43 | .value("NONE", CompressionNone) // Don't use 'None' since it's a keyword in py3 44 | .value("LZ4", CompressionLZ4) 45 | .value("ZLib", CompressionZLib) 46 | .value("ZSTD", CompressionZSTD) 47 | .value("SNAPPY", CompressionSNAPPY); 48 | 49 | enum_(m, "ConsumerType") 50 | .value("Exclusive", ConsumerExclusive) 51 | .value("Shared", ConsumerShared) 52 | .value("Failover", ConsumerFailover) 53 | .value("KeyShared", ConsumerKeyShared); 54 | 55 | enum_(m, "Result", "Collection of return codes") 56 | .value("Ok", ResultOk) 57 | .value("UnknownError", ResultUnknownError) 58 | .value("InvalidConfiguration", ResultInvalidConfiguration) 59 | .value("Timeout", ResultTimeout) 60 | .value("LookupError", ResultLookupError) 61 | .value("ConnectError", ResultConnectError) 62 | .value("ReadError", ResultReadError) 63 | .value("AuthenticationError", ResultAuthenticationError) 64 | .value("AuthorizationError", ResultAuthorizationError) 65 | .value("ErrorGettingAuthenticationData", ResultErrorGettingAuthenticationData) 66 | .value("BrokerMetadataError", ResultBrokerMetadataError) 67 | .value("BrokerPersistenceError", ResultBrokerPersistenceError) 68 | .value("ChecksumError", ResultChecksumError) 69 | .value("ConsumerBusy", ResultConsumerBusy) 70 | .value("NotConnected", ResultNotConnected) 71 | .value("AlreadyClosed", ResultAlreadyClosed) 72 | .value("InvalidMessage", ResultInvalidMessage) 73 | .value("ConsumerNotInitialized", ResultConsumerNotInitialized) 74 | .value("ProducerNotInitialized", ResultProducerNotInitialized) 75 | .value("ProducerBusy", ResultProducerBusy) 76 | .value("TooManyLookupRequestException", ResultTooManyLookupRequestException) 77 | .value("InvalidTopicName", ResultInvalidTopicName) 78 | .value("InvalidUrl", ResultInvalidUrl) 79 | .value("ServiceUnitNotReady", ResultServiceUnitNotReady) 80 | .value("OperationNotSupported", ResultOperationNotSupported) 81 | .value("ProducerBlockedQuotaExceededError", ResultProducerBlockedQuotaExceededError) 82 | .value("ProducerBlockedQuotaExceededException", ResultProducerBlockedQuotaExceededException) 83 | .value("ProducerQueueIsFull", ResultProducerQueueIsFull) 84 | .value("MessageTooBig", ResultMessageTooBig) 85 | .value("TopicNotFound", ResultTopicNotFound) 86 | .value("SubscriptionNotFound", ResultSubscriptionNotFound) 87 | .value("ConsumerNotFound", ResultConsumerNotFound) 88 | .value("UnsupportedVersionError", ResultUnsupportedVersionError) 89 | .value("TopicTerminated", ResultTopicTerminated) 90 | .value("CryptoError", ResultCryptoError) 91 | .value("IncompatibleSchema", ResultIncompatibleSchema) 92 | .value("ConsumerAssignError", ResultConsumerAssignError) 93 | .value("CumulativeAcknowledgementNotAllowedError", ResultCumulativeAcknowledgementNotAllowedError) 94 | .value("TransactionCoordinatorNotFoundError", ResultTransactionCoordinatorNotFoundError) 95 | .value("InvalidTxnStatusError", ResultInvalidTxnStatusError) 96 | .value("NotAllowedError", ResultNotAllowedError) 97 | .value("TransactionConflict", ResultTransactionConflict) 98 | .value("TransactionNotFound", ResultTransactionNotFound) 99 | .value("ProducerFenced", ResultProducerFenced) 100 | .value("MemoryBufferIsFull", ResultMemoryBufferIsFull) 101 | .value("Interrupted", pulsar::ResultInterrupted); 102 | 103 | enum_(m, "SchemaType", "Supported schema types") 104 | .value("NONE", pulsar::NONE) 105 | .value("STRING", pulsar::STRING) 106 | .value("INT8", pulsar::INT8) 107 | .value("INT16", pulsar::INT16) 108 | .value("INT32", pulsar::INT32) 109 | .value("INT64", pulsar::INT64) 110 | .value("FLOAT", pulsar::FLOAT) 111 | .value("DOUBLE", pulsar::DOUBLE) 112 | .value("BYTES", pulsar::BYTES) 113 | .value("JSON", pulsar::JSON) 114 | .value("PROTOBUF", pulsar::PROTOBUF) 115 | .value("AVRO", pulsar::AVRO) 116 | .value("AUTO_CONSUME", pulsar::AUTO_CONSUME) 117 | .value("AUTO_PUBLISH", pulsar::AUTO_PUBLISH) 118 | .value("KEY_VALUE", pulsar::KEY_VALUE); 119 | 120 | enum_(m, "InitialPosition", "Supported initial position") 121 | .value("Latest", InitialPositionLatest) 122 | .value("Earliest", InitialPositionEarliest); 123 | 124 | enum_(m, "RegexSubscriptionMode", "Regex subscription mode") 125 | .value("PersistentOnly", PersistentOnly) 126 | .value("NonPersistentOnly", NonPersistentOnly) 127 | .value("AllTopics", AllTopics); 128 | 129 | enum_(m, "BatchingType", "Supported batching types") 130 | .value("Default", ProducerConfiguration::DefaultBatching) 131 | .value("KeyBased", ProducerConfiguration::KeyBasedBatching); 132 | 133 | enum_(m, "ProducerAccessMode", "Producer Access Mode") 134 | .value("Shared", ProducerConfiguration::ProducerAccessMode::Shared) 135 | .value("Exclusive", ProducerConfiguration::ProducerAccessMode::Exclusive) 136 | .value("WaitForExclusive", ProducerConfiguration::ProducerAccessMode::WaitForExclusive) 137 | .value("ExclusiveWithFencing", ProducerConfiguration::ProducerAccessMode::ExclusiveWithFencing); 138 | 139 | enum_(m, "LoggerLevel") 140 | .value("Debug", Logger::LEVEL_DEBUG) 141 | .value("Info", Logger::LEVEL_INFO) 142 | .value("Warn", Logger::LEVEL_WARN) 143 | .value("Error", Logger::LEVEL_ERROR); 144 | 145 | enum_(m, "ConsumerCryptoFailureAction") 146 | .value("FAIL", ConsumerCryptoFailureAction::FAIL) 147 | .value("DISCARD", ConsumerCryptoFailureAction::DISCARD) 148 | .value("CONSUME", ConsumerCryptoFailureAction::CONSUME); 149 | } 150 | -------------------------------------------------------------------------------- /tests/test-conf/cacert.pem: -------------------------------------------------------------------------------- 1 | Certificate: 2 | Data: 3 | Version: 3 (0x2) 4 | Serial Number: 5 | 52:7b:b4:00:96:60:b4:26:85:be:01:82:b8:b8:e2:8c:72:ef:5b:90 6 | Signature Algorithm: sha256WithRSAEncryption 7 | Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org 8 | Validity 9 | Not Before: Feb 17 16:43:44 2021 GMT 10 | Not After : Feb 12 16:43:44 2041 GMT 11 | Subject: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org 12 | Subject Public Key Info: 13 | Public Key Algorithm: rsaEncryption 14 | Public-Key: (4096 bit) 15 | Modulus: 16 | 00:b1:3c:7d:ab:4a:54:72:37:2a:92:94:0a:66:46: 17 | af:8c:ed:f4:2e:f3:87:1a:d0:c7:9d:23:35:1b:61: 18 | 74:69:ca:f7:f5:3e:95:9c:86:f2:21:34:f8:0b:ed: 19 | 45:76:22:ec:75:52:c0:67:db:2f:ba:da:25:3f:e1: 20 | 5b:ac:da:15:dd:a5:75:24:b2:12:f0:b0:ce:fd:ab: 21 | 44:06:a9:09:f6:b0:8e:8f:83:53:16:69:fa:9c:cc: 22 | 00:fa:dd:13:f3:da:fd:f2:bf:88:8e:c4:f8:1a:6f: 23 | ab:4d:f8:32:81:80:7e:51:7a:99:2d:94:cd:f3:5d: 24 | 1c:58:b2:44:f1:96:12:46:56:bd:60:8f:65:32:b7: 25 | d4:4b:7b:f3:23:88:2d:9b:a4:c4:c9:52:ea:9f:66: 26 | c1:74:be:4b:91:c6:b9:57:ec:c1:cc:81:bb:03:d5: 27 | fa:a0:46:4f:9a:a7:3e:3c:27:26:2b:97:eb:69:53: 28 | 04:75:50:97:d6:0d:90:b1:37:9f:64:df:70:4d:d9: 29 | b3:e3:b7:cc:76:50:d9:3c:9b:4c:ac:e9:26:2e:cf: 30 | ac:47:42:14:b7:60:00:0a:de:42:47:66:0c:c7:7a: 31 | b9:4d:f4:fb:c2:6a:45:78:ec:b0:b4:ce:b3:1f:50: 32 | 25:96:13:0c:55:0a:e0:d6:76:f7:1f:e1:16:e6:41: 33 | d6:72:6a:49:17:12:d9:05:8f:dc:56:b6:31:b3:b7: 34 | 9c:e3:d8:a9:99:8a:1d:3b:9d:d9:59:44:ee:46:88: 35 | 11:5f:ab:fa:38:a9:8b:d2:23:15:8b:af:1a:de:66: 36 | ba:7d:51:95:37:94:91:aa:01:01:d7:83:19:4b:5d: 37 | 8d:f4:18:39:ef:e3:32:d0:62:c8:12:50:4e:91:c2: 38 | ac:58:73:68:bb:92:20:fc:14:e5:1a:86:bd:40:4c: 39 | 94:e0:7d:0d:9c:08:57:ae:00:44:38:94:a3:3d:64: 40 | 99:43:f8:e3:12:90:14:0f:5d:63:e2:c6:07:ea:d0: 41 | 4c:8e:cf:e0:ae:34:be:86:4f:fc:58:e2:ea:f5:23: 42 | 82:37:96:02:57:1b:b4:29:ca:fd:68:a0:48:79:e8: 43 | 31:97:9a:5a:0e:2b:b4:b0:84:bb:57:4e:5f:4f:a7: 44 | 43:45:97:d7:de:05:fc:2f:6c:3e:f5:53:26:56:a3: 45 | a5:da:52:69:57:8e:a0:4b:27:50:f9:ad:6e:76:a6: 46 | 29:cc:06:94:dd:d0:ac:c6:18:22:a0:e2:bb:ed:d5: 47 | e4:97:f7:ac:23:df:75:30:41:97:07:3f:d3:12:8e: 48 | c5:a4:ef:ce:40:e8:3b:57:24:19:33:1b:ee:8a:0e: 49 | dd:0c:70:f2:1a:87:35:d9:71:d8:18:a7:9c:47:db: 50 | 93:51:c3 51 | Exponent: 65537 (0x10001) 52 | X509v3 extensions: 53 | X509v3 Subject Key Identifier: 54 | D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71 55 | X509v3 Authority Key Identifier: 56 | keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71 57 | 58 | X509v3 Basic Constraints: critical 59 | CA:TRUE 60 | X509v3 Key Usage: critical 61 | Digital Signature, Certificate Sign, CRL Sign 62 | Signature Algorithm: sha256WithRSAEncryption 63 | 14:3d:7c:15:86:de:aa:5a:30:5d:d4:f2:bc:5f:10:d2:af:fe: 64 | 91:d7:ee:f3:b8:5f:ce:e4:c9:b2:01:c3:16:da:66:8e:7e:b1: 65 | c1:e3:30:ff:1d:73:d0:9c:20:3d:54:32:57:ae:07:80:4a:24: 66 | 6e:7e:32:a3:e7:23:4d:5c:31:54:8b:c1:1b:c5:bc:20:5d:43: 67 | 62:93:e0:2e:a7:01:77:39:cf:fd:ec:4c:57:09:4f:2b:ad:ac: 68 | b6:c0:be:5a:a3:ea:12:ac:5a:7f:60:23:81:bb:9a:fa:5f:7a: 69 | 67:a9:31:c3:34:af:db:ff:32:22:83:40:c2:7d:2f:39:5e:8a: 70 | 29:44:73:5f:6e:b4:f4:a2:ae:60:1f:8e:ef:91:9a:49:bb:a6: 71 | 90:2b:e0:44:95:24:8b:37:90:18:2d:41:32:8a:8e:07:8d:ea: 72 | 75:62:b8:9c:ec:73:6f:12:54:23:6d:40:00:74:c7:d3:fb:b7: 73 | 95:06:7d:cc:6d:8e:2c:d0:8b:11:06:8a:b7:43:1a:d7:e9:98: 74 | f4:c6:ef:ad:2a:75:08:fb:07:8f:20:36:7a:86:1a:cf:f7:d6: 75 | 96:ad:ed:71:59:d1:81:56:18:8d:98:c2:c0:44:e5:29:7a:7c: 76 | c0:e3:d7:fb:b8:f5:b2:50:53:8a:cf:38:ff:99:aa:bb:28:51: 77 | 60:e8:05:91:e1:ee:86:90:90:9b:87:60:63:38:cf:54:a5:82: 78 | 74:0f:40:b5:d2:6a:c5:a9:98:22:59:4e:fb:a5:81:e2:7b:0e: 79 | 3f:71:f3:24:17:1e:c5:89:fc:ae:ed:f3:69:65:02:b8:1e:98: 80 | bc:37:c6:25:36:f8:ca:99:60:8e:13:3b:33:ec:91:b3:eb:04: 81 | 6d:41:97:3e:35:c0:97:ed:66:12:25:44:23:f3:2e:fa:9c:2e: 82 | c2:ba:dd:f3:63:d7:5b:b2:72:03:4d:3b:fb:5e:29:d6:5c:02: 83 | 32:93:47:d1:4c:77:4a:58:c5:aa:81:ab:67:84:80:81:14:28: 84 | e1:db:11:16:6d:31:50:7a:47:b2:a8:2d:15:a1:c4:63:1b:ce: 85 | d5:e1:d7:57:dc:1a:71:e0:55:9f:6d:fb:be:e6:99:e8:89:be: 86 | 2c:e0:19:5e:cd:02:79:52:ee:93:56:9f:dc:d7:de:31:9b:2a: 87 | c8:91:48:a0:c7:44:7d:72:32:27:c3:2b:d8:e8:6b:94:67:b5: 88 | 1d:9d:99:25:23:d9:24:b5:ed:4b:f2:18:2d:88:f5:d4:36:bb: 89 | 53:8c:a8:b1:7f:05:13:d7:8d:89:9d:55:33:90:bc:60:99:cf: 90 | 05:ba:bd:cb:c5:61:f9:c5:1a:f7:46:9c:40:90:dd:83:aa:7a: 91 | 1f:ab:5c:10:8d:26:27:1e 92 | -----BEGIN CERTIFICATE----- 93 | MIIGPzCCBCegAwIBAgIUUnu0AJZgtCaFvgGCuLjijHLvW5AwDQYJKoZIhvcNAQEL 94 | BQAwgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQH 95 | DAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9u 96 | MQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3 97 | DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMB4XDTIxMDIxNzE2NDM0NFoXDTQx 98 | MDIxMjE2NDM0NFowgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh 99 | MRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBG 100 | b3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEk 101 | MCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMIICIjANBgkqhkiG 102 | 9w0BAQEFAAOCAg8AMIICCgKCAgEAsTx9q0pUcjcqkpQKZkavjO30LvOHGtDHnSM1 103 | G2F0acr39T6VnIbyITT4C+1FdiLsdVLAZ9svutolP+FbrNoV3aV1JLIS8LDO/atE 104 | BqkJ9rCOj4NTFmn6nMwA+t0T89r98r+IjsT4Gm+rTfgygYB+UXqZLZTN810cWLJE 105 | 8ZYSRla9YI9lMrfUS3vzI4gtm6TEyVLqn2bBdL5Lkca5V+zBzIG7A9X6oEZPmqc+ 106 | PCcmK5fraVMEdVCX1g2QsTefZN9wTdmz47fMdlDZPJtMrOkmLs+sR0IUt2AACt5C 107 | R2YMx3q5TfT7wmpFeOywtM6zH1AllhMMVQrg1nb3H+EW5kHWcmpJFxLZBY/cVrYx 108 | s7ec49ipmYodO53ZWUTuRogRX6v6OKmL0iMVi68a3ma6fVGVN5SRqgEB14MZS12N 109 | 9Bg57+My0GLIElBOkcKsWHNou5Ig/BTlGoa9QEyU4H0NnAhXrgBEOJSjPWSZQ/jj 110 | EpAUD11j4sYH6tBMjs/grjS+hk/8WOLq9SOCN5YCVxu0Kcr9aKBIeegxl5paDiu0 111 | sIS7V05fT6dDRZfX3gX8L2w+9VMmVqOl2lJpV46gSydQ+a1udqYpzAaU3dCsxhgi 112 | oOK77dXkl/esI991MEGXBz/TEo7FpO/OQOg7VyQZMxvuig7dDHDyGoc12XHYGKec 113 | R9uTUcMCAwEAAaNjMGEwHQYDVR0OBBYEFNKyPbGkfEhLNuGn3tj8upK6p8RxMB8G 114 | A1UdIwQYMBaAFNKyPbGkfEhLNuGn3tj8upK6p8RxMA8GA1UdEwEB/wQFMAMBAf8w 115 | DgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQAUPXwVht6qWjBd1PK8 116 | XxDSr/6R1+7zuF/O5MmyAcMW2maOfrHB4zD/HXPQnCA9VDJXrgeASiRufjKj5yNN 117 | XDFUi8EbxbwgXUNik+AupwF3Oc/97ExXCU8rray2wL5ao+oSrFp/YCOBu5r6X3pn 118 | qTHDNK/b/zIig0DCfS85XoopRHNfbrT0oq5gH47vkZpJu6aQK+BElSSLN5AYLUEy 119 | io4Hjep1Yric7HNvElQjbUAAdMfT+7eVBn3MbY4s0IsRBoq3QxrX6Zj0xu+tKnUI 120 | +wePIDZ6hhrP99aWre1xWdGBVhiNmMLAROUpenzA49f7uPWyUFOKzzj/maq7KFFg 121 | 6AWR4e6GkJCbh2BjOM9UpYJ0D0C10mrFqZgiWU77pYHiew4/cfMkFx7Fifyu7fNp 122 | ZQK4Hpi8N8YlNvjKmWCOEzsz7JGz6wRtQZc+NcCX7WYSJUQj8y76nC7Cut3zY9db 123 | snIDTTv7XinWXAIyk0fRTHdKWMWqgatnhICBFCjh2xEWbTFQekeyqC0VocRjG87V 124 | 4ddX3Bpx4FWfbfu+5pnoib4s4BlezQJ5Uu6TVp/c194xmyrIkUigx0R9cjInwyvY 125 | 6GuUZ7UdnZklI9kkte1L8hgtiPXUNrtTjKixfwUT142JnVUzkLxgmc8Fur3LxWH5 126 | xRr3RpxAkN2Dqnofq1wQjSYnHg== 127 | -----END CERTIFICATE----- 128 | -------------------------------------------------------------------------------- /src/exceptions.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | #include "exceptions.h" 20 | #include 21 | #include 22 | 23 | using namespace pulsar; 24 | namespace py = pybind11; 25 | 26 | #define CASE_RESULT(className) \ 27 | case pulsar::Result##className: \ 28 | throw className{pulsar::Result##className}; 29 | 30 | void raiseException(pulsar::Result result) { 31 | switch (result) { 32 | CASE_RESULT(UnknownError) 33 | CASE_RESULT(InvalidConfiguration) 34 | CASE_RESULT(Timeout) 35 | CASE_RESULT(LookupError) 36 | CASE_RESULT(ConnectError) 37 | CASE_RESULT(ReadError) 38 | CASE_RESULT(AuthenticationError) 39 | CASE_RESULT(AuthorizationError) 40 | CASE_RESULT(ErrorGettingAuthenticationData) 41 | CASE_RESULT(BrokerMetadataError) 42 | CASE_RESULT(BrokerPersistenceError) 43 | CASE_RESULT(ChecksumError) 44 | CASE_RESULT(ConsumerBusy) 45 | CASE_RESULT(NotConnected) 46 | CASE_RESULT(AlreadyClosed) 47 | CASE_RESULT(InvalidMessage) 48 | CASE_RESULT(ConsumerNotInitialized) 49 | CASE_RESULT(ProducerNotInitialized) 50 | CASE_RESULT(ProducerBusy) 51 | CASE_RESULT(TooManyLookupRequestException) 52 | CASE_RESULT(InvalidTopicName) 53 | CASE_RESULT(InvalidUrl) 54 | CASE_RESULT(ServiceUnitNotReady) 55 | CASE_RESULT(OperationNotSupported) 56 | CASE_RESULT(ProducerBlockedQuotaExceededError) 57 | CASE_RESULT(ProducerBlockedQuotaExceededException) 58 | CASE_RESULT(ProducerQueueIsFull) 59 | CASE_RESULT(MessageTooBig) 60 | CASE_RESULT(TopicNotFound) 61 | CASE_RESULT(SubscriptionNotFound) 62 | CASE_RESULT(ConsumerNotFound) 63 | CASE_RESULT(UnsupportedVersionError) 64 | CASE_RESULT(TopicTerminated) 65 | CASE_RESULT(CryptoError) 66 | CASE_RESULT(IncompatibleSchema) 67 | CASE_RESULT(ConsumerAssignError) 68 | CASE_RESULT(CumulativeAcknowledgementNotAllowedError) 69 | CASE_RESULT(TransactionCoordinatorNotFoundError) 70 | CASE_RESULT(InvalidTxnStatusError) 71 | CASE_RESULT(NotAllowedError) 72 | CASE_RESULT(TransactionConflict) 73 | CASE_RESULT(TransactionNotFound) 74 | CASE_RESULT(ProducerFenced) 75 | CASE_RESULT(MemoryBufferIsFull) 76 | CASE_RESULT(Interrupted) 77 | default: 78 | return; 79 | } 80 | } 81 | 82 | // There is no std::hash specification for an enum in Clang compiler of macOS for C++11 83 | template <> 84 | struct std::hash { 85 | std::size_t operator()(const Result& result) const noexcept { 86 | return std::hash()(static_cast(result)); 87 | } 88 | }; 89 | 90 | using PythonExceptionMap = std::unordered_map>; 91 | static PythonExceptionMap createPythonExceptionMap(py::module_& m, py::exception& base) { 92 | PythonExceptionMap exceptions; 93 | exceptions[ResultUnknownError] = {m, "UnknownError", base}; 94 | exceptions[ResultInvalidConfiguration] = {m, "InvalidConfiguration", base}; 95 | exceptions[ResultTimeout] = {m, "Timeout", base}; 96 | exceptions[ResultLookupError] = {m, "LookupError", base}; 97 | exceptions[ResultConnectError] = {m, "ConnectError", base}; 98 | exceptions[ResultReadError] = {m, "ReadError", base}; 99 | exceptions[ResultAuthenticationError] = {m, "AuthenticationError", base}; 100 | exceptions[ResultAuthorizationError] = {m, "AuthorizationError", base}; 101 | exceptions[ResultErrorGettingAuthenticationData] = {m, "ErrorGettingAuthenticationData", base}; 102 | exceptions[ResultBrokerMetadataError] = {m, "BrokerMetadataError", base}; 103 | exceptions[ResultBrokerPersistenceError] = {m, "BrokerPersistenceError", base}; 104 | exceptions[ResultChecksumError] = {m, "ChecksumError", base}; 105 | exceptions[ResultConsumerBusy] = {m, "ConsumerBusy", base}; 106 | exceptions[ResultNotConnected] = {m, "NotConnected", base}; 107 | exceptions[ResultAlreadyClosed] = {m, "AlreadyClosed", base}; 108 | exceptions[ResultInvalidMessage] = {m, "InvalidMessage", base}; 109 | exceptions[ResultConsumerNotInitialized] = {m, "ConsumerNotInitialized", base}; 110 | exceptions[ResultProducerNotInitialized] = {m, "ProducerNotInitialized", base}; 111 | exceptions[ResultProducerBusy] = {m, "ProducerBusy", base}; 112 | exceptions[ResultTooManyLookupRequestException] = {m, "TooManyLookupRequestException", base}; 113 | exceptions[ResultInvalidTopicName] = {m, "InvalidTopicName", base}; 114 | exceptions[ResultInvalidUrl] = {m, "InvalidUrl", base}; 115 | exceptions[ResultServiceUnitNotReady] = {m, "ServiceUnitNotReady", base}; 116 | exceptions[ResultOperationNotSupported] = {m, "OperationNotSupported", base}; 117 | exceptions[ResultProducerBlockedQuotaExceededError] = {m, "ProducerBlockedQuotaExceededError", base}; 118 | exceptions[ResultProducerBlockedQuotaExceededException] = {m, "ProducerBlockedQuotaExceededException", 119 | base}; 120 | exceptions[ResultProducerQueueIsFull] = {m, "ProducerQueueIsFull", base}; 121 | exceptions[ResultMessageTooBig] = {m, "MessageTooBig", base}; 122 | exceptions[ResultTopicNotFound] = {m, "TopicNotFound", base}; 123 | exceptions[ResultSubscriptionNotFound] = {m, "SubscriptionNotFound", base}; 124 | exceptions[ResultConsumerNotFound] = {m, "ConsumerNotFound", base}; 125 | exceptions[ResultUnsupportedVersionError] = {m, "UnsupportedVersionError", base}; 126 | exceptions[ResultTopicTerminated] = {m, "TopicTerminated", base}; 127 | exceptions[ResultCryptoError] = {m, "CryptoError", base}; 128 | exceptions[ResultIncompatibleSchema] = {m, "IncompatibleSchema", base}; 129 | exceptions[ResultConsumerAssignError] = {m, "ConsumerAssignError", base}; 130 | exceptions[ResultCumulativeAcknowledgementNotAllowedError] = { 131 | m, "CumulativeAcknowledgementNotAllowedError", base}; 132 | exceptions[ResultTransactionCoordinatorNotFoundError] = {m, "TransactionCoordinatorNotFoundError", base}; 133 | exceptions[ResultInvalidTxnStatusError] = {m, "InvalidTxnStatusError", base}; 134 | exceptions[ResultNotAllowedError] = {m, "NotAllowedError", base}; 135 | exceptions[ResultTransactionConflict] = {m, "TransactionConflict", base}; 136 | exceptions[ResultTransactionNotFound] = {m, "TransactionNotFound", base}; 137 | exceptions[ResultProducerFenced] = {m, "ProducerFenced", base}; 138 | exceptions[ResultMemoryBufferIsFull] = {m, "MemoryBufferIsFull", base}; 139 | exceptions[ResultInterrupted] = {m, "Interrupted", base}; 140 | return exceptions; 141 | } 142 | 143 | void export_exceptions(py::module_& m) { 144 | static py::exception base{m, "PulsarException"}; 145 | static auto exceptions = createPythonExceptionMap(m, base); 146 | py::register_exception_translator([](std::exception_ptr e) { 147 | try { 148 | if (e) { 149 | std::rethrow_exception(e); 150 | } 151 | } catch (const PulsarException& e) { 152 | auto it = exceptions.find(e._result); 153 | if (it != exceptions.end()) { 154 | PyErr_SetString(it->second.ptr(), e.what()); 155 | } else { 156 | base(e.what()); 157 | } 158 | } catch (const std::invalid_argument& e) { 159 | PyErr_SetString(PyExc_ValueError, e.what()); 160 | } catch (const std::exception& e) { 161 | PyErr_SetString(PyExc_RuntimeError, e.what()); 162 | } 163 | }); 164 | } 165 | -------------------------------------------------------------------------------- /.github/workflows/ci-pr-validation.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: PR validation 21 | on: 22 | pull_request: 23 | branches: ['main'] 24 | 25 | concurrency: 26 | group: ${{ github.workflow }}-${{ github.ref }} 27 | cancel-in-progress: true 28 | 29 | jobs: 30 | check-and-lint: 31 | name: Lint and check code 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v3 35 | - uses: actions/setup-python@v4 36 | with: 37 | python-version: 3.x 38 | - name: Install deps 39 | run: pip install -U pydoctor 40 | - name: Check docs syntax 41 | run: | 42 | pydoctor --make-html \ 43 | --html-viewsource-base=https://github.com/apache/pulsar-client-python/tree/main \ 44 | --docformat=numpy --theme=readthedocs \ 45 | --intersphinx=https://docs.python.org/3/objects.inv \ 46 | --html-output=apidocs \ 47 | pulsar 48 | 49 | unit-tests: 50 | name: Run unit tests for Python ${{matrix.version}} 51 | runs-on: ubuntu-latest 52 | timeout-minutes: 120 53 | 54 | strategy: 55 | fail-fast: false 56 | matrix: 57 | version: ['3.10', '3.14'] 58 | 59 | steps: 60 | - name: checkout 61 | uses: actions/checkout@v4 62 | 63 | - uses: actions/setup-python@v5 64 | with: 65 | python-version: "${{matrix.version}}" 66 | 67 | - name: Install Pulsar C++ client 68 | run: build-support/install-dependencies.sh 69 | 70 | - name: CMake 71 | run: cmake . 72 | 73 | - name: Build 74 | run: make -j8 75 | 76 | - name: Python install 77 | run: | 78 | python3 -m pip install -U pip setuptools wheel requests 79 | python3 setup.py bdist_wheel 80 | WHEEL=$(find dist -name '*.whl') 81 | pip3 install ${WHEEL}[avro] 82 | 83 | - name: Run Oauth2 tests 84 | run: | 85 | docker compose -f ./build-support/docker-compose-pulsar-oauth2.yml up -d 86 | # Wait until the namespace is created, currently there is no good way to check it via CLI 87 | sleep 10 88 | python3 tests/oauth2_test.py 89 | docker compose -f ./build-support/docker-compose-pulsar-oauth2.yml down 90 | 91 | - name: Start Pulsar service 92 | run: ./build-support/pulsar-test-service-start.sh 93 | 94 | - name: Run unit tests 95 | run: ./tests/run-unit-tests.sh 96 | 97 | - name: Stop Pulsar service 98 | run: ./build-support/pulsar-test-service-stop.sh 99 | 100 | - name: Test functions install 101 | run: | 102 | WHEEL=$(find dist -name '*.whl') 103 | pip3 install ${WHEEL}[all] --force-reinstall 104 | 105 | linux-wheel: 106 | name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} 107 | needs: unit-tests 108 | runs-on: ubuntu-latest 109 | timeout-minutes: 300 110 | 111 | strategy: 112 | fail-fast: false 113 | matrix: 114 | image: 115 | - {name: 'manylinux', py_suffix: ''} 116 | - {name: 'manylinux_musl', py_suffix: '-alpine'} 117 | python: 118 | - {version: '3.14', spec: 'cp314-cp314'} 119 | cpu: 120 | - {arch: 'x86_64', platform: 'x86_64'} 121 | 122 | steps: 123 | - name: checkout 124 | uses: actions/checkout@v3 125 | 126 | - name: Set up QEMU 127 | uses: docker/setup-qemu-action@v2 128 | 129 | - uses: docker/setup-buildx-action@v2 130 | 131 | - name: Build Manylinux Docker image 132 | uses: docker/build-push-action@v3 133 | with: 134 | context: ./pkg/${{matrix.image.name}} 135 | load: true 136 | tags: build:latest 137 | platforms: linux/${{matrix.cpu.arch}} 138 | build-args: | 139 | PLATFORM=${{matrix.cpu.platform}} 140 | ARCH=${{matrix.cpu.arch}} 141 | PYTHON_VERSION=${{matrix.python.version}} 142 | PYTHON_SPEC=${{matrix.python.spec}} 143 | cache-from: type=gha 144 | cache-to: type=gha,mode=max 145 | 146 | - name: Build wheel file 147 | run: | 148 | docker run -i -v $PWD:/pulsar-client-python build:latest \ 149 | /pulsar-client-python/pkg/build-wheel-inside-docker.sh 150 | 151 | - name: Test wheel file 152 | run: | 153 | docker run -i -v $PWD:/pulsar-client-python python:${{matrix.python.version}}${{matrix.image.py_suffix}} \ 154 | /pulsar-client-python/pkg/test-wheel.sh 155 | 156 | mac-wheels: 157 | name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} 158 | needs: unit-tests 159 | runs-on: macos-14 160 | timeout-minutes: 300 161 | 162 | strategy: 163 | matrix: 164 | py: 165 | - {version: '3.14', version_long: '3.14.0'} 166 | 167 | steps: 168 | - name: checkout 169 | uses: actions/checkout@v3 170 | 171 | - name: Build and test Mac wheels 172 | run: | 173 | pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} 174 | 175 | windows-wheels: 176 | name: "Python ${{ matrix.python.version }} Wheel on Windows x64" 177 | needs: unit-tests 178 | runs-on: windows-2022 179 | timeout-minutes: 120 180 | 181 | env: 182 | PULSAR_CPP_DIR: 'C:\\pulsar-cpp' 183 | strategy: 184 | fail-fast: false 185 | matrix: 186 | python: 187 | - version: '3.14' 188 | 189 | steps: 190 | - uses: actions/checkout@v3 191 | 192 | - uses: actions/setup-python@v4 193 | with: 194 | python-version: ${{ matrix.python.version }} 195 | 196 | - name: Download Pulsar C++ client on Windows 197 | shell: bash 198 | run: | 199 | source ./build-support/dep-url.sh 200 | BASE_URL=$(pulsar_cpp_base_url $(grep pulsar-cpp dependencies.yaml | awk '{print $2}')) 201 | mkdir -p ${{ env.PULSAR_CPP_DIR }} 202 | cd ${{ env.PULSAR_CPP_DIR }} 203 | curl -O -L ${BASE_URL}/x64-windows-static.tar.gz 204 | tar zxf x64-windows-static.tar.gz 205 | mv x64-windows-static/* . 206 | ls -l ${{ env.PULSAR_CPP_DIR }} 207 | 208 | - name: Configure CMake 209 | shell: bash 210 | run: | 211 | pip3 install pyyaml 212 | export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) 213 | curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz 214 | tar zxf v${PYBIND11_VERSION}.tar.gz 215 | rm -rf pybind11 216 | mv pybind11-${PYBIND11_VERSION} pybind11 217 | cmake -B build -A x64 \ 218 | -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \ 219 | -DLINK_STATIC=ON 220 | 221 | - name: Build Python wheel 222 | shell: bash 223 | run: | 224 | cmake --build build --config Release --target install 225 | python -m pip install wheel setuptools 226 | python setup.py bdist_wheel 227 | python -m pip install ./dist/*.whl 228 | python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' 229 | 230 | check-completion: 231 | name: Check Completion 232 | runs-on: ubuntu-latest 233 | if: ${{ always() }} 234 | needs: [ 235 | check-and-lint, 236 | unit-tests, 237 | linux-wheel, 238 | mac-wheels, 239 | windows-wheels 240 | ] 241 | steps: 242 | - run: | 243 | if [[ ! ( \ 244 | "${{ needs.check-and-lint.result }}" == "success" \ 245 | && "${{ needs.unit-tests.result }}" == "success" \ 246 | && "${{ needs.linux-wheel.result }}" == "success" \ 247 | && "${{ needs.mac-wheels.result }}" == "success" \ 248 | && "${{ needs.windows-wheels.result }}" == "success" \ 249 | ) ]]; then 250 | echo "Required jobs haven't been completed successfully." 251 | exit 1 252 | fi 253 | --------------------------------------------------------------------------------