├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── appengine-push ├── README.md ├── app.yaml ├── appengine_config.py ├── constants.py ├── js │ └── pubsub.js ├── main.py ├── pubsub_utils.py ├── requirements.txt ├── templates │ └── pubsub.html └── test_deploy.py ├── client-secret.json.enc ├── cmdline-pull ├── .gitignore ├── README.md ├── pubsub_sample.py ├── requirements.txt └── test_pubsub_sample.py ├── gce-cmdline-publisher ├── README.md └── traffic_pubsub_generator.py ├── grpc ├── README.md ├── pubsub_sample.py └── requirements.txt └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Copied from 2 | # https://github.com/github/gitignore/blob/master/Python.gitignore 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # Installer logs 28 | pip-log.txt 29 | pip-delete-this-directory.txt 30 | 31 | # Unit test / coverage reports 32 | htmlcov/ 33 | .tox/ 34 | .coverage 35 | .cache 36 | nosetests.xml 37 | coverage.xml 38 | 39 | # Translations 40 | *.mo 41 | *.pot 42 | 43 | # Django stuff: 44 | *.log 45 | 46 | # Sphinx documentation 47 | docs/_build/ 48 | client-secret.json 49 | 50 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | branches: 4 | only: 5 | - master 6 | 7 | cache: 8 | directories: 9 | - ${HOME}/gcloud/ 10 | 11 | install: 12 | - pip install tox 13 | 14 | env: 15 | globals: 16 | - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin 17 | - CLOUDSDK_CORE_DISABLE_PROMPTS=1 18 | 19 | before_install: 20 | - openssl aes-256-cbc -K $encrypted_a53bb0208314_key -iv $encrypted_a53bb0208314_iv -in client-secret.json.enc -out client-secret.json -d 21 | - if [ ! -d ${HOME}/gcloud/google-cloud-sdk ]; then 22 | mkdir -p ${HOME}/gcloud && 23 | wget https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz --directory-prefix=${HOME}/gcloud && 24 | cd ${HOME}/gcloud && 25 | tar xzf google-cloud-sdk.tar.gz && 26 | ./google-cloud-sdk/install.sh --usage-reporting false --path-update false --command-completion false && 27 | cd ${TRAVIS_BUILD_DIR}; 28 | fi 29 | - gcloud -q components update app 30 | - if [ -a client-secret.json ]; then 31 | gcloud auth activate-service-account --key-file client-secret.json; 32 | fi 33 | - gcloud config set project cloud-pubsub-sample-test 34 | - mkdir -p appengine-push/lib 35 | - pip install -t appengine-push/lib -r appengine-push/requirements.txt 36 | - gcloud -q app deploy --project cloud-pubsub-sample-test --version=py appengine-push/app.yaml 37 | 38 | script: 39 | - tox 40 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributor License Agreements 2 | ------------------------------ 3 | 4 | Before we can accept your pull requests you'll need to sign a Contributor License Agreement (CLA): 5 | 6 | * If you are an individual writing original source code and you own the intellectual property, then you'll need to sign an [individual CLA](https://developers.google.com/open-source/cla/individual). 7 | 8 | * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate>). 9 | 10 | You can sign these electronically (just scroll to the bottom). After that, we'll be able to accept your pull requests. 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloud Pub/Sub samples for Python 2 | 3 | ## Overview 4 | 5 | This repository contains several samples for Cloud Pub/Sub service 6 | with Python. 7 | 8 | - appengine-push 9 | 10 | A sample for push subscription running on [Google App Engine][1]. 11 | 12 | - cmdline-pull 13 | 14 | A command line sample for pull subscription. 15 | 16 | - gce-cmdline-publisher 17 | 18 | A Python command-line script that publishes to a topic using data from a large traffic sensor dataset. 19 | 20 | ## Run tests 21 | 22 | Here are instructions to run the tests. You need a cloud project with 23 | Cloud Pub/Sub enabled. 24 | 25 | ```bash 26 | $ pip install tox 27 | $ export GOOGLE_APPLICATION_CREDENTIALS=your-service-account-json-file 28 | $ export TEST_PROJECT_ID={YOUR_PROJECT_ID} 29 | $ tox 30 | ``` 31 | 32 | ## Licensing 33 | 34 | See LICENSE 35 | 36 | [1]: https://developers.google.com/appengine/ 37 | -------------------------------------------------------------------------------- /appengine-push/README.md: -------------------------------------------------------------------------------- 1 | # cloud-pubsub-samples-python 2 | 3 | ## appengine-push 4 | 5 | Note: The push endpoints don't work with the App Engine's local 6 | devserver. The push notifications will go to an HTTP URL on the App 7 | Engine server even when you run this sample locally. So we recommend 8 | you deploy and run the app on App Engine. 9 | TODO(tmatsuo): Better implementation for devserver. 10 | 11 | ## Register your application 12 | 13 | - Go to 14 | [Google Developers Console](https://console.developers.google.com/project) 15 | and create a new project. This will automatically enable an App 16 | Engine application with the same ID as the project. 17 | 18 | - Enable the "Google Cloud Pub/Sub" API under "APIs & auth > APIs." 19 | 20 | - For local development also follow the instructions below. 21 | 22 | - Go to "Credentials" and create a new Service Account. 23 | 24 | - Select "Generate new JSON key", then download a new JSON file. 25 | 26 | - Set the following environment variable. 27 | 28 | GOOGLE_APPLICATION_CREDENTIALS: the file path to the downloaded JSON file. 29 | 30 | ## Prerequisites 31 | 32 | - Install Python-2.7, pip-6.0.0 or higher and App Engine Python SDK. 33 | We recommend you install 34 | [Cloud SDK](https://developers.google.com/cloud/sdk/) rather than 35 | just installing App Engine SDK. 36 | 37 | - Install Google API client library for python into 'lib' directory by: 38 | 39 | ``` 40 | $ pip install -t lib -r requirements.txt 41 | ``` 42 | 43 | ## Configuration 44 | 45 | - Edit constants.py 46 | - Replace '{AN_UNIQUE_TOKEN}' with an arbitrary secret string of 47 | your choice to protect the endpoint from abuse. 48 | 49 | ## Deploy the application to App Engine 50 | 51 | ``` 52 | $ appcfg.py --oauth2 update -A your-application-id . 53 | ``` 54 | 55 | or you can use gcloud SDK 56 | 57 | ``` 58 | $ gcloud app deploy 59 | ``` 60 | 61 | Then access the following URL: 62 | https://{your-application-id}.appspot.com/ 63 | 64 | ## Run the application locally 65 | 66 | ``` 67 | $ dev_appserver.py -A your-application-id . 68 | ``` 69 | -------------------------------------------------------------------------------- /appengine-push/app.yaml: -------------------------------------------------------------------------------- 1 | runtime: python27 2 | api_version: 1 3 | threadsafe: true 4 | 5 | handlers: 6 | - url: /js 7 | static_dir: js 8 | - url: /_ah/push-handlers/.* 9 | script: main.APPLICATION 10 | login: admin 11 | - url: /.* 12 | script: main.APPLICATION 13 | 14 | libraries: 15 | - name: jinja2 16 | version: latest 17 | - name: webapp2 18 | version: latest 19 | - name: pycrypto 20 | version: latest 21 | - name: ssl 22 | version: latest 23 | -------------------------------------------------------------------------------- /appengine-push/appengine_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Cloud Pub/Sub sample application config.""" 18 | 19 | from google.appengine.ext import vendor 20 | 21 | vendor.add('lib') 22 | -------------------------------------------------------------------------------- /appengine-push/constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Cloud Pub/Sub sample application constants.""" 18 | 19 | SUBSCRIPTION_UNIQUE_TOKEN = '{AN-UNIQUE-TOKEN}' 20 | -------------------------------------------------------------------------------- /appengine-push/js/pubsub.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | var pubsub = pubsub || angular.module('pubsub', []); 5 | 6 | /** 7 | * PubsubController. 8 | * 9 | * @NgInject 10 | */ 11 | pubsub.PubsubController = function($http, $log, $timeout) { 12 | this.promise = null; 13 | this.logger = $log; 14 | this.http = $http; 15 | this.timeout = $timeout; 16 | this.interval = 1; 17 | this.isAutoUpdating = true; 18 | this.failCount = 0; 19 | this.fetchMessages(); 20 | }; 21 | 22 | pubsub.PubsubController.MAX_FAILURE_COUNT = 3; 23 | 24 | pubsub.PubsubController.TIMEOUT_MULTIPLIER = 1000; 25 | 26 | /** 27 | * Toggles the auto update flag. 28 | */ 29 | pubsub.PubsubController.prototype.toggleAutoUpdate = function() { 30 | this.isAutoUpdating = !this.isAutoUpdating; 31 | if (this.isAutoUpdating) { 32 | this.logger.info('Start fetching.'); 33 | this.fetchMessages(); 34 | } else if (this.promise !== null) { 35 | this.logger.info('Cancel the promise.'); 36 | this.timeout.cancel(this.promise); 37 | this.promise = null; 38 | } 39 | }; 40 | 41 | /** 42 | * Sends a message 43 | * 44 | * @param {string} message 45 | */ 46 | pubsub.PubsubController.prototype.sendMessage = function(message) { 47 | var self = this; 48 | self.http({ 49 | method: 'POST', 50 | url: '/send_message', 51 | data: 'message=' + encodeURIComponent(message), 52 | headers: {'Content-Type': 'application/x-www-form-urlencoded'} 53 | }).success(function(data, status) { 54 | self.message = null; 55 | }).error(function(data, status) { 56 | self.logger.error('Failed to send the message. Status: ' + status + '.'); 57 | }); 58 | }; 59 | 60 | /** 61 | * Continuously fetches messages from the server. 62 | */ 63 | pubsub.PubsubController.prototype.fetchMessages = function() { 64 | var self = this; 65 | self.http.get('/fetch_messages') 66 | .success(function(data, status) { 67 | self.messages = data; 68 | self.failCount = 0; 69 | }) 70 | .error(function(data, status) { 71 | self.logger.error('Failed to receive the messages. Status: ' + 72 | status + '.'); 73 | self.failCount += 1; 74 | }); 75 | if (self.failCount < pubsub.PubsubController.MAX_FAILURE_COUNT) { 76 | if (self.isAutoUpdating) { 77 | self.promise = self.timeout( 78 | function() { self.fetchMessages(); }, 79 | self.interval * pubsub.PubsubController.TIMEOUT_MULTIPLIER); 80 | } 81 | } else { 82 | self.errorNotice = 'Maximum failure count reached, ' + 83 | 'so stopped fetching messages.'; 84 | self.logger.error(self.errorNotice); 85 | self.isAutoUpdating = false; 86 | self.failCount = 0; 87 | } 88 | }; 89 | -------------------------------------------------------------------------------- /appengine-push/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Cloud Pub/Sub sample application.""" 18 | 19 | 20 | import base64 21 | import json 22 | import logging 23 | import re 24 | import urllib 25 | 26 | from apiclient import errors 27 | from google.appengine.api import memcache 28 | from google.appengine.ext import ndb 29 | 30 | import jinja2 31 | 32 | import webapp2 33 | 34 | import constants 35 | import pubsub_utils 36 | 37 | 38 | JINJA2 = jinja2.Environment(loader=jinja2.FileSystemLoader('templates'), 39 | extensions=['jinja2.ext.autoescape'], 40 | variable_start_string='((', 41 | variable_end_string='))', 42 | autoescape=True) 43 | 44 | MAX_ITEM = 20 45 | 46 | MESSAGE_CACHE_KEY = 'messages_key' 47 | 48 | 49 | class PubSubMessage(ndb.Model): 50 | """A model stores pubsub message and the time when it arrived.""" 51 | message = ndb.StringProperty() 52 | created_at = ndb.DateTimeProperty(auto_now_add=True) 53 | 54 | 55 | class InitHandler(webapp2.RequestHandler): 56 | """Initializes the Pub/Sub resources.""" 57 | def __init__(self, request=None, response=None): 58 | """Calls the constructor of the super and does the local setup.""" 59 | super(InitHandler, self).__init__(request, response) 60 | self.client = pubsub_utils.get_client() 61 | self._setup_topic() 62 | self._setup_subscription() 63 | 64 | def _setup_topic(self): 65 | """Creates a topic if it does not exist.""" 66 | topic_name = pubsub_utils.get_full_topic_name() 67 | try: 68 | self.client.projects().topics().get( 69 | topic=topic_name).execute() 70 | except errors.HttpError as e: 71 | if e.resp.status == 404: 72 | self.client.projects().topics().create( 73 | name=topic_name, body={}).execute() 74 | else: 75 | logging.exception(e) 76 | raise 77 | 78 | def _setup_subscription(self): 79 | """Creates a subscription if it does not exist.""" 80 | subscription_name = pubsub_utils.get_full_subscription_name() 81 | try: 82 | self.client.projects().subscriptions().get( 83 | subscription=subscription_name).execute() 84 | except errors.HttpError as e: 85 | if e.resp.status == 404: 86 | body = { 87 | 'topic': pubsub_utils.get_full_topic_name(), 88 | 'pushConfig': { 89 | 'pushEndpoint': pubsub_utils.get_app_endpoint_url() 90 | } 91 | } 92 | self.client.projects().subscriptions().create( 93 | name=subscription_name, body=body).execute() 94 | else: 95 | logging.exception(e) 96 | raise 97 | 98 | def get(self): 99 | """Shows an HTML form.""" 100 | template = JINJA2.get_template('pubsub.html') 101 | endpoint_url = re.sub('token=[^&]*', 'token=REDACTED', 102 | pubsub_utils.get_app_endpoint_url()) 103 | context = { 104 | 'project': pubsub_utils.get_project_id(), 105 | 'topic': pubsub_utils.get_app_topic_name(), 106 | 'subscription': pubsub_utils.get_app_subscription_name(), 107 | 'subscriptionEndpoint': endpoint_url 108 | } 109 | self.response.write(template.render(context)) 110 | 111 | 112 | class FetchMessages(webapp2.RequestHandler): 113 | """A handler returns messages.""" 114 | def get(self): 115 | """Returns recent messages as a json.""" 116 | messages = memcache.get(MESSAGE_CACHE_KEY) 117 | if not messages: 118 | messages = PubSubMessage.query().order( 119 | -PubSubMessage.created_at).fetch(MAX_ITEM) 120 | memcache.add(MESSAGE_CACHE_KEY, messages) 121 | self.response.headers['Content-Type'] = ('application/json;' 122 | ' charset=UTF-8') 123 | self.response.write( 124 | json.dumps( 125 | [message.message for message in messages])) 126 | 127 | 128 | class SendMessage(webapp2.RequestHandler): 129 | """A handler publishes the given message.""" 130 | def post(self): 131 | """Publishes the message via the Pub/Sub API.""" 132 | client = pubsub_utils.get_client() 133 | message = self.request.get('message') 134 | if message: 135 | topic_name = pubsub_utils.get_full_topic_name() 136 | body = { 137 | 'messages': [{ 138 | 'data': base64.b64encode(message.encode('utf-8')) 139 | }] 140 | } 141 | client.projects().topics().publish( 142 | topic=topic_name, body=body).execute() 143 | self.response.status = 204 144 | 145 | 146 | class ReceiveMessage(webapp2.RequestHandler): 147 | """A handler for push subscription endpoint..""" 148 | def post(self): 149 | if constants.SUBSCRIPTION_UNIQUE_TOKEN != self.request.get('token'): 150 | self.response.status = 404 151 | return 152 | 153 | # Store the message in the datastore. 154 | logging.debug('Post body: {}'.format(self.request.body)) 155 | message = json.loads(urllib.unquote(self.request.body).rstrip('=')) 156 | message_body = base64.b64decode(str(message['message']['data'])) 157 | pubsub_message = PubSubMessage(message=message_body) 158 | pubsub_message.put() 159 | 160 | # Invalidate the cache 161 | memcache.delete(MESSAGE_CACHE_KEY) 162 | self.response.status = 200 163 | 164 | 165 | APPLICATION = webapp2.WSGIApplication( 166 | [ 167 | ('/', InitHandler), 168 | ('/fetch_messages', FetchMessages), 169 | ('/send_message', SendMessage), 170 | ('/_ah/push-handlers/receive_message', ReceiveMessage), 171 | ], debug=True) 172 | -------------------------------------------------------------------------------- /appengine-push/pubsub_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Utility module for this Pub/Sub sample.""" 18 | 19 | import os 20 | import threading 21 | 22 | from google.appengine.api import app_identity 23 | from google.appengine.api import memcache 24 | from google.appengine.api import modules 25 | 26 | from googleapiclient import discovery 27 | import httplib2 28 | from oauth2client.client import GoogleCredentials 29 | 30 | import constants 31 | 32 | 33 | APPLICATION_NAME = "google-cloud-pubsub-appengine-sample/1.0" 34 | 35 | PUBSUB_SCOPES = ["https://www.googleapis.com/auth/pubsub"] 36 | 37 | 38 | client_store = threading.local() 39 | 40 | 41 | def is_devserver(): 42 | """Check if the app is running on devserver or not.""" 43 | return os.getenv('SERVER_SOFTWARE', '').startswith('Dev') 44 | 45 | 46 | def get_client(): 47 | """Creates Pub/Sub client and returns it.""" 48 | if not hasattr(client_store, 'client'): 49 | client_store.client = get_client_from_credentials( 50 | GoogleCredentials.get_application_default()) 51 | return client_store.client 52 | 53 | 54 | def get_client_from_credentials(credentials): 55 | """Creates Pub/Sub client from a given credentials and returns it.""" 56 | if credentials.create_scoped_required(): 57 | credentials = credentials.create_scoped(PUBSUB_SCOPES) 58 | 59 | http = httplib2.Http(memcache) 60 | credentials.authorize(http) 61 | 62 | return discovery.build('pubsub', 'v1', http=http) 63 | 64 | 65 | def get_full_topic_name(): 66 | return 'projects/{}/topics/{}'.format( 67 | get_project_id(), get_app_topic_name()) 68 | 69 | 70 | def get_full_subscription_name(): 71 | return 'projects/{}/subscriptions/{}'.format( 72 | get_project_id(), get_app_subscription_name()) 73 | 74 | 75 | def get_app_topic_name(): 76 | return 'topic-pubsub-api-appengine-sample-python' 77 | 78 | 79 | def get_app_subscription_name(): 80 | return 'subscription-python-{}'.format(get_project_id()) 81 | 82 | 83 | def get_app_endpoint_url(): 84 | return ('https://{}-dot-{}.appspot.com/_ah/push-handlers' 85 | '/receive_message?token={}').format( 86 | get_current_version(), get_project_id(), 87 | constants.SUBSCRIPTION_UNIQUE_TOKEN) 88 | 89 | 90 | def get_project_id(): 91 | return app_identity.get_application_id() 92 | 93 | 94 | def get_current_version(): 95 | return modules.get_current_version_name() 96 | -------------------------------------------------------------------------------- /appengine-push/requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client 2 | -------------------------------------------------------------------------------- /appengine-push/templates/pubsub.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Hello Cloud Pub/Sub 6 | 7 | 8 | 9 |

Hello Cloud Pub/Sub!

10 | 11 | Project: (( project ))
12 | Publishing to Topic: (( topic ))
13 | Listening on Subscription: (( subscription ))
14 | Endpoint URL: (( subscriptionEndpoint ))
15 | 16 |
17 | 18 |
19 | 20 | 23 | 26 | per seconds. 27 | {{ PubsubController.errorNotice }} 28 | 29 |

Messages:

30 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /appengine-push/test_deploy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Test classes for the command line Cloud Pub/Sub sample.""" 18 | 19 | import os 20 | import time 21 | import unittest 22 | import urllib 23 | import uuid 24 | 25 | import httplib2 26 | 27 | TEST_GAE_HOST_ENV = 'TEST_GAE_HOST' 28 | DEFAULT_TEST_GAE_HOST = "py-dot-cloud-pubsub-sample-test.appspot.com" 29 | MAX_RETRY = 3 30 | SLEEP_TIME = 1 31 | GAE_HOST = os.getenv(TEST_GAE_HOST_ENV, DEFAULT_TEST_GAE_HOST) 32 | 33 | 34 | def url_for(path): 35 | """Returns the URL of the endpoint for the given path.""" 36 | return 'https://%s%s' % (GAE_HOST, path) 37 | 38 | 39 | class IntegrationTestCase(unittest.TestCase): 40 | """A test case for the Pubsub App Engine sample.""" 41 | 42 | def setUp(self): 43 | random_id = uuid.uuid4() 44 | # The first three character will be encoded as a string 45 | # containing '+', in order to test the consistency on which 46 | # base64 variant to use on the server side and the client side. 47 | self.message = '=@~message-%s' % random_id 48 | self.http = httplib2.Http() 49 | 50 | def test_get(self): 51 | """Test accessing the top page.""" 52 | (resp, content) = self.http.request(url_for('/'), 'GET') 53 | # This ensures that our App Engine service account is working 54 | # correctly. 55 | self.assertEquals(200, resp.status) 56 | 57 | def fetch_messages(self): 58 | """Fetch messages""" 59 | (_, content) = self.http.request(url_for('/fetch_messages'), 'GET') 60 | return content 61 | 62 | def test_send_message(self): 63 | """Test submitting a message.""" 64 | headers = {'Content-Type': 'application/x-www-form-urlencoded'} 65 | params = urllib.urlencode({'message': self.message}) 66 | (resp, content) = self.http.request( 67 | url_for('/send_message'), 'POST', body=params, headers=headers) 68 | self.assertEquals(204, resp.status) 69 | found = False 70 | for i in range(MAX_RETRY): 71 | time.sleep(SLEEP_TIME) 72 | content = self.fetch_messages() 73 | if self.message in content: 74 | found = True 75 | break 76 | self.assertTrue(found) 77 | 78 | def test_receive_message(self): 79 | """Test that the /_ah/push-handlers/ is protected.""" 80 | (resp, _) = self.http.request( 81 | url_for('/_ah/push-handlers/receive_message'), 'POST', 82 | headers={'Content-Length': '0'}) 83 | self.assertEquals( 84 | 302, resp.status, 85 | 'status for /_ah/push-handlers/receive_message should be 302, ' 86 | 'actual: {}'.format(resp.status)) 87 | -------------------------------------------------------------------------------- /client-secret.json.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-pubsub-samples-python/8f59af0fb234d29d32459def1581e93467fabf01/client-secret.json.enc -------------------------------------------------------------------------------- /cmdline-pull/.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | include 3 | lib 4 | local 5 | share 6 | *.bak 7 | discovery-doc-* 8 | -------------------------------------------------------------------------------- /cmdline-pull/README.md: -------------------------------------------------------------------------------- 1 | # cloud-pubsub-samples-python 2 | 3 | 4 | ## cmdline-pull 5 | 6 | This is a command line sample application using the Cloud Pub/Sub 7 | API. You can do the following things with this command line tool: 8 | 9 | - List/Create/Delete topics/subscriptions 10 | - Connect to an IRC channel and publish IRC massages. 11 | - Pull messages from a subscription and print those messages. 12 | 13 | ## Prerequisites 14 | 15 | - Install Python-2.7 and google-api-python-client. Here are the 16 | instructions with virtualenv and pip. 17 | 18 | ``` 19 | $ virtualenv -p python2.7 --no-site-packages . 20 | $ source bin/activate 21 | $ pip install -r requirements.txt 22 | ``` 23 | ## Register your application 24 | 25 | - If you don't have a project, go to [Google Developers Console][1] 26 | and create a new project. 27 | 28 | - Enable the "Google Cloud Pub/Sub" API under "APIs & auth > APIs." 29 | 30 | - Go to "Credentials" and create a new Service Account. 31 | 32 | - Select "Generate new JSON key", then download a new JSON file. 33 | 34 | - Set the following environment variable. 35 | 36 | GOOGLE_APPLICATION_CREDENTIALS: the file path to the downloaded JSON file. 37 | 38 | ## Run the application 39 | 40 | ``` 41 | $ python pubsub_sample.py 42 | ``` 43 | 44 | This will give you a help message. Here is an example session with 45 | this command. 46 | 47 | ``` 48 | # create a new topic "test" on MYPROJ 49 | $ python pubsub_sample.py MYPROJ create_topic test 50 | 51 | # list the current topics 52 | $ python pubsub_sample.py MYPROJ list_topics 53 | 54 | # create a new subscription "sub" on the "test" topic 55 | $ python pubsub_sample.py MYPROJ create_subscription sub test 56 | 57 | # publish a message "hello" to the "test" topic 58 | $ python pubsub_sample.py MYPROJ publish_message test hello 59 | 60 | # connect to the Wikipedia recent change channel 61 | $ python pubsub_sample.py \ 62 | MYPROJ \ 63 | connect_irc \ 64 | test \ 65 | irc.wikimedia.org \ 66 | "#en.wikipedia" 67 | 68 | # fetch messages from the subscription "sub" 69 | $ python pubsub_sample.py MYPROJ pull_messages sub 70 | ``` 71 | 72 | Enjoy! 73 | 74 | [1]: https://console.developers.google.com/project 75 | -------------------------------------------------------------------------------- /cmdline-pull/pubsub_sample.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2014 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Cloud Pub/Sub sample application.""" 18 | 19 | 20 | import argparse 21 | import base64 22 | import json 23 | import re 24 | import socket 25 | import sys 26 | import time 27 | 28 | from googleapiclient import discovery 29 | from oauth2client.client import GoogleCredentials 30 | 31 | 32 | PUBSUB_SCOPES = ["https://www.googleapis.com/auth/pubsub"] 33 | 34 | BOTNAME = 'pubsub-irc-bot/1.0' 35 | 36 | PORT = 6667 37 | 38 | NUM_RETRIES = 3 39 | 40 | BATCH_SIZE = 10 41 | 42 | 43 | def fqrn(resource_type, project, resource): 44 | """Return a fully qualified resource name for Cloud Pub/Sub.""" 45 | return "projects/{}/{}/{}".format(project, resource_type, resource) 46 | 47 | 48 | def get_full_topic_name(project, topic): 49 | """Return a fully qualified topic name.""" 50 | return fqrn('topics', project, topic) 51 | 52 | 53 | def get_full_subscription_name(project, subscription): 54 | """Return a fully qualified subscription name.""" 55 | return fqrn('subscriptions', project, subscription) 56 | 57 | 58 | def list_topics(client, args): 59 | """Show the list of current topics.""" 60 | next_page_token = None 61 | while True: 62 | resp = client.projects().topics().list( 63 | project='projects/{}'.format(args.project_name), 64 | pageToken=next_page_token).execute(num_retries=NUM_RETRIES) 65 | if 'topics' in resp: 66 | for topic in resp['topics']: 67 | print topic['name'] 68 | next_page_token = resp.get('nextPageToken') 69 | if not next_page_token: 70 | break 71 | 72 | 73 | def list_subscriptions(client, args): 74 | """Show the list of current subscriptions. 75 | 76 | If a topic is specified, only subscriptions associated with the topic will 77 | be listed. 78 | """ 79 | next_page_token = None 80 | while True: 81 | if args.topic is None: 82 | resp = client.projects().subscriptions().list( 83 | project='projects/{}'.format(args.project_name), 84 | pageToken=next_page_token).execute(num_retries=NUM_RETRIES) 85 | else: 86 | topic = get_full_topic_name(args.project_name, args.topic) 87 | resp = client.projects().topics().subscriptions().list( 88 | topic=topic, 89 | pageToken=next_page_token).execute(num_retries=NUM_RETRIES) 90 | for subscription in resp['subscriptions']: 91 | print json.dumps(subscription, indent=1) 92 | next_page_token = resp.get('nextPageToken') 93 | if not next_page_token: 94 | break 95 | 96 | 97 | def create_topic(client, args): 98 | """Create a new topic.""" 99 | topic = client.projects().topics().create( 100 | name=get_full_topic_name(args.project_name, args.topic), 101 | body={}).execute(num_retries=NUM_RETRIES) 102 | print 'Topic {} was created.'.format(topic['name']) 103 | 104 | 105 | def delete_topic(client, args): 106 | """Delete a topic.""" 107 | topic = get_full_topic_name(args.project_name, args.topic) 108 | client.projects().topics().delete( 109 | topic=topic).execute(num_retries=NUM_RETRIES) 110 | print 'Topic {} was deleted.'.format(topic) 111 | 112 | 113 | def create_subscription(client, args): 114 | """Create a new subscription to a given topic. 115 | 116 | If an endpoint is specified, this function will attach to that 117 | endpoint. 118 | """ 119 | name = get_full_subscription_name(args.project_name, args.subscription) 120 | if '/' in args.topic: 121 | topic_name = args.topic 122 | else: 123 | topic_name = get_full_topic_name(args.project_name, args.topic) 124 | body = {'topic': topic_name} 125 | if args.push_endpoint is not None: 126 | body['pushConfig'] = {'pushEndpoint': args.push_endpoint} 127 | subscription = client.projects().subscriptions().create( 128 | name=name, body=body).execute(num_retries=NUM_RETRIES) 129 | print 'Subscription {} was created.'.format(subscription['name']) 130 | 131 | 132 | def delete_subscription(client, args): 133 | """Delete a subscription.""" 134 | subscription = get_full_subscription_name(args.project_name, 135 | args.subscription) 136 | client.projects().subscriptions().delete( 137 | subscription=subscription).execute(num_retries=NUM_RETRIES) 138 | print 'Subscription {} was deleted.'.format(subscription) 139 | 140 | 141 | def _check_connection(irc): 142 | """Check a connection to an IRC channel.""" 143 | readbuffer = '' 144 | while True: 145 | readbuffer = readbuffer + irc.recv(1024) 146 | temp = readbuffer.split('\n') 147 | readbuffer = temp.pop() 148 | for line in temp: 149 | if "004" in line: 150 | return 151 | elif "433" in line: 152 | sys.err.write('Nickname is already in use.') 153 | sys.exit(1) 154 | 155 | 156 | def connect_irc(client, args): 157 | """Connect to an IRC channel and publishe messages.""" 158 | server = args.server 159 | channel = args.channel 160 | topic = get_full_topic_name(args.project_name, args.topic) 161 | nick = 'bot-{}'.format(args.project_name) 162 | irc = socket.socket() 163 | print 'Connecting to {}'.format(server) 164 | irc.connect((server, PORT)) 165 | 166 | irc.send("NICK {}\r\n".format(nick)) 167 | irc.send("USER {} 8 * : {}\r\n".format(nick, BOTNAME)) 168 | readbuffer = '' 169 | _check_connection(irc) 170 | print 'Connected to {}.'.format(server) 171 | 172 | irc.send("JOIN {}\r\n".format(channel)) 173 | priv_mark = "PRIVMSG {} :".format(channel) 174 | p = re.compile( 175 | r'\x0314\[\[\x0307(.*)\x0314\]\]\x03.*\x0302(http://[^\x03]*)\x03') 176 | while True: 177 | readbuffer = readbuffer + irc.recv(1024) 178 | temp = readbuffer.split('\n') 179 | readbuffer = temp.pop() 180 | for line in temp: 181 | line = line.rstrip() 182 | parts = line.split() 183 | if parts[0] == "PING": 184 | irc.send("PONG {}\r\n".format(parts[1])) 185 | else: 186 | i = line.find(priv_mark) 187 | if i == -1: 188 | continue 189 | line = line[i + len(priv_mark):] 190 | m = p.match(line) 191 | if m: 192 | line = "Title: {}, Diff: {}".format(m.group(1), m.group(2)) 193 | body = { 194 | 'messages': [{'data': base64.b64encode(str(line))}] 195 | } 196 | client.projects().topics().publish( 197 | topic=topic, body=body).execute(num_retries=NUM_RETRIES) 198 | 199 | 200 | def publish_message(client, args): 201 | """Publish a message to a given topic.""" 202 | topic = get_full_topic_name(args.project_name, args.topic) 203 | message = base64.b64encode(str(args.message)) 204 | body = {'messages': [{'data': message}]} 205 | resp = client.projects().topics().publish( 206 | topic=topic, body=body).execute(num_retries=NUM_RETRIES) 207 | print ('Published a message "{}" to a topic {}. The message_id was {}.' 208 | .format(args.message, topic, resp.get('messageIds')[0])) 209 | 210 | 211 | def pull_messages(client, args): 212 | """Pull messages from a given subscription.""" 213 | subscription = get_full_subscription_name( 214 | args.project_name, 215 | args.subscription) 216 | body = { 217 | 'returnImmediately': False, 218 | 'maxMessages': BATCH_SIZE 219 | } 220 | while True: 221 | try: 222 | resp = client.projects().subscriptions().pull( 223 | subscription=subscription, body=body).execute( 224 | num_retries=NUM_RETRIES) 225 | except Exception as e: 226 | time.sleep(0.5) 227 | print e 228 | continue 229 | receivedMessages = resp.get('receivedMessages') 230 | if receivedMessages: 231 | ack_ids = [] 232 | for receivedMessage in receivedMessages: 233 | message = receivedMessage.get('message') 234 | if message: 235 | print base64.b64decode(str(message.get('data'))) 236 | ack_ids.append(receivedMessage.get('ackId')) 237 | ack_body = {'ackIds': ack_ids} 238 | client.projects().subscriptions().acknowledge( 239 | subscription=subscription, body=ack_body).execute( 240 | num_retries=NUM_RETRIES) 241 | if args.no_loop: 242 | break 243 | 244 | 245 | def main(argv): 246 | """Invoke a subcommand.""" 247 | # Main parser setup 248 | parser = argparse.ArgumentParser( 249 | description='A sample command line interface for Pub/Sub') 250 | parser.add_argument('project_name', help='Project name in console') 251 | 252 | topic_parser = argparse.ArgumentParser(add_help=False) 253 | topic_parser.add_argument('topic', help='Topic name') 254 | subscription_parser = argparse.ArgumentParser(add_help=False) 255 | subscription_parser.add_argument('subscription', help='Subscription name') 256 | 257 | # Sub command parsers 258 | sub_parsers = parser.add_subparsers( 259 | title='List of possible commands', metavar='') 260 | 261 | list_topics_str = 'List topics in project' 262 | parser_list_topics = sub_parsers.add_parser( 263 | 'list_topics', description=list_topics_str, help=list_topics_str) 264 | parser_list_topics.set_defaults(func=list_topics) 265 | 266 | create_topic_str = 'Create a topic with specified name' 267 | parser_create_topic = sub_parsers.add_parser( 268 | 'create_topic', parents=[topic_parser], 269 | description=create_topic_str, help=create_topic_str) 270 | parser_create_topic.set_defaults(func=create_topic) 271 | 272 | delete_topic_str = 'Delete a topic with specified name' 273 | parser_delete_topic = sub_parsers.add_parser( 274 | 'delete_topic', parents=[topic_parser], 275 | description=delete_topic_str, help=delete_topic_str) 276 | parser_delete_topic.set_defaults(func=delete_topic) 277 | 278 | list_subscriptions_str = 'List subscriptions in project' 279 | parser_list_subscriptions = sub_parsers.add_parser( 280 | 'list_subscriptions', 281 | description=list_subscriptions_str, help=list_subscriptions_str) 282 | parser_list_subscriptions.set_defaults(func=list_subscriptions) 283 | parser_list_subscriptions.add_argument( 284 | '-t', '--topic', help='Show only subscriptions for given topic') 285 | 286 | create_subscription_str = 'Create a subscription to the specified topic' 287 | parser_create_subscription = sub_parsers.add_parser( 288 | 'create_subscription', parents=[subscription_parser, topic_parser], 289 | description=create_subscription_str, help=create_subscription_str) 290 | parser_create_subscription.set_defaults(func=create_subscription) 291 | parser_create_subscription.add_argument( 292 | '-p', '--push_endpoint', 293 | help='Push endpoint to which this method attaches') 294 | 295 | delete_subscription_str = 'Delete the specified subscription' 296 | parser_delete_subscription = sub_parsers.add_parser( 297 | 'delete_subscription', parents=[subscription_parser], 298 | description=delete_subscription_str, help=delete_subscription_str) 299 | parser_delete_subscription.set_defaults(func=delete_subscription) 300 | 301 | connect_irc_str = 'Connect to the topic IRC channel' 302 | parser_connect_irc = sub_parsers.add_parser( 303 | 'connect_irc', parents=[topic_parser], 304 | description=connect_irc_str, help=connect_irc_str) 305 | parser_connect_irc.set_defaults(func=connect_irc) 306 | parser_connect_irc.add_argument('server', help='Server name') 307 | parser_connect_irc.add_argument('channel', help='Channel name') 308 | 309 | publish_message_str = 'Publish a message to specified topic' 310 | parser_publish_message = sub_parsers.add_parser( 311 | 'publish_message', parents=[topic_parser], 312 | description=publish_message_str, help=publish_message_str) 313 | parser_publish_message.set_defaults(func=publish_message) 314 | parser_publish_message.add_argument('message', help='Message to publish') 315 | 316 | pull_messages_str = ('Pull messages for given subscription. ' 317 | 'Loops continuously unless otherwise specified') 318 | parser_pull_messages = sub_parsers.add_parser( 319 | 'pull_messages', parents=[subscription_parser], 320 | description=pull_messages_str, help=pull_messages_str) 321 | parser_pull_messages.set_defaults(func=pull_messages) 322 | parser_pull_messages.add_argument( 323 | '-n', '--no_loop', action='store_true', 324 | help='Execute only once and do not loop') 325 | 326 | # Google API setup 327 | credentials = GoogleCredentials.get_application_default() 328 | if credentials.create_scoped_required(): 329 | credentials = credentials.create_scoped(PUBSUB_SCOPES) 330 | client = discovery.build('pubsub', 'v1', credentials=credentials) 331 | 332 | args = parser.parse_args(argv[1:]) 333 | args.func(client, args) 334 | 335 | 336 | if __name__ == '__main__': 337 | main(sys.argv) 338 | -------------------------------------------------------------------------------- /cmdline-pull/requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client 2 | pycrypto 3 | -------------------------------------------------------------------------------- /cmdline-pull/test_pubsub_sample.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Test classes for the command line Cloud Pub/Sub sample.""" 18 | 19 | 20 | import contextlib 21 | import os 22 | import StringIO 23 | import sys 24 | import unittest 25 | import uuid 26 | 27 | 28 | from pubsub_sample import main 29 | 30 | 31 | TEST_PROJECT_ID_ENV = 'TEST_PROJECT_ID' 32 | DEFAULT_TEST_PROJECT_ID = "cloud-pubsub-sample-test" 33 | 34 | 35 | @contextlib.contextmanager 36 | def captured_output(): 37 | """Capture output and redirect to sys.""" 38 | new_out, new_err = StringIO.StringIO(), StringIO.StringIO() 39 | old_out, old_err = sys.stdout, sys.stderr 40 | try: 41 | sys.stdout, sys.stderr = new_out, new_err 42 | yield sys.stdout, sys.stderr 43 | finally: 44 | sys.stdout, sys.stderr = old_out, old_err 45 | 46 | 47 | def get_project_id(): 48 | """Return the project id to use in the tests.""" 49 | return os.getenv(TEST_PROJECT_ID_ENV, DEFAULT_TEST_PROJECT_ID) 50 | 51 | 52 | class PubsubSampleTestCase(unittest.TestCase): 53 | """A test case for the Pubsub sample. 54 | 55 | Define a test case that creates and lists topics and subscriptions. 56 | Also tests publishing and pulling messages 57 | """ 58 | 59 | @classmethod 60 | def setUpClass(cls): 61 | """Create a new topic and subscription with a random name.""" 62 | random_id = uuid.uuid4() 63 | cls.topic = 'topic-%s' % random_id 64 | cls.sub = 'sub-%s' % random_id 65 | main(['pubsub_sample.py', get_project_id(), 'create_topic', 66 | cls.topic]) 67 | main(['pubsub_sample.py', get_project_id(), 'create_subscription', 68 | cls.sub, cls.topic]) 69 | # The third message is to check the consistency between base64 70 | # variants used on the server side and the client side. 71 | cls.messages = ['message-1-%s' % uuid.uuid4(), 72 | 'message-2-%s' % uuid.uuid4(), 73 | '=@~'] 74 | 75 | @classmethod 76 | def tearDownClass(cls): 77 | """Delete resources used in the tests.""" 78 | main(['pubsub_sample.py', get_project_id(), 'delete_topic', cls.topic]) 79 | main(['pubsub_sample.py', get_project_id(), 'delete_subscription', 80 | cls.sub]) 81 | 82 | def test_list_topics(self): 83 | """Test the list_topics action.""" 84 | expected_topic = ('projects/%s/topics/%s' 85 | % (get_project_id(), self.topic)) 86 | with captured_output() as (out, _): 87 | main(['pubsub_sample.py', get_project_id(), 'list_topics']) 88 | output = out.getvalue().strip() 89 | self.assertTrue(expected_topic in output) 90 | 91 | def test_list_subscriptions(self): 92 | """Test the list_subscriptions action.""" 93 | expected_sub = ('projects/%s/subscriptions/%s' 94 | % (get_project_id(), self.sub)) 95 | with captured_output() as (out, _): 96 | main(['pubsub_sample.py', get_project_id(), 'list_subscriptions']) 97 | output = out.getvalue().strip() 98 | self.assertTrue(expected_sub in output) 99 | 100 | def test_publish_message(self): 101 | """Try to publish a message and check the output for the message.""" 102 | for message in self.messages: 103 | with captured_output() as (out, _): 104 | main(['pubsub_sample.py', get_project_id(), 'publish_message', 105 | self.topic, message]) 106 | output = out.getvalue().strip() 107 | self.assertTrue(message in output) 108 | 109 | def test_pull_message(self): 110 | """Try to pull messages from a subscription.""" 111 | with captured_output() as (out, _): 112 | main(['pubsub_sample.py', get_project_id(), 'pull_messages', 113 | self.sub, '-n']) 114 | output = out.getvalue().strip() 115 | for message in self.messages: 116 | self.assertTrue(message in output) 117 | -------------------------------------------------------------------------------- /gce-cmdline-publisher/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## 'Traffic Sensor' data generator script 3 | 4 | The `traffic_pubsub_generator.py` script reads traffic sensor data from a file and publishes that data 5 | to a PubSub topic. 6 | The script uses the [ Google APIs Client Library for Python](https://developers.google.com/api-client-library/python/?_ga=1.268664177.1432014927.1424389293). 7 | You can install this library via: 8 | 9 | ``` 10 | pip install --upgrade google-api-python-client 11 | ``` 12 | 13 | See [this page](https://developers.google.com/accounts/docs/application-default-credentials) 14 | for more information about the `GoogleCredentials` library used by the script. 15 | 16 | See the documentation in `traffic_pubsub_generator.py` for more detail. 17 | -------------------------------------------------------------------------------- /gce-cmdline-publisher/traffic_pubsub_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ 17 | This script reads traffic sensor data from a file and publishes that data 18 | to PubSub. If you run it on a GCE instance, this instance must be 19 | created with the "Cloud Platform" Project Access enabled. Click on 20 | "Show advanced options" when creating the image to find this setting. 21 | 22 | Before you run the script, create two PubSub topics, in the same project as the 23 | GCE instance you will run on, to publish to. Edit the TRAFFIC_TOPIC and 24 | INCIDENT_TOPIC variables in the script to point to these topics, or you can 25 | pass in their names as command-line arguments. 26 | 27 | Before you run this script, download an input data file (~2GB): 28 | curl -O \ 29 | http://storage.googleapis.com/aju-sd-traffic/unzipped/Freeways-5Minaa2010-01-01_to_2010-02-15.csv 30 | Or, for a smaller test file, you can use: 31 | http://storage.googleapis.com/aju-sd-traffic/unzipped/Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv 32 | These files contain real traffic sensor data from San Diego freeways. 33 | See this file for copyright info: 34 | http://storage.googleapis.com/aju-sd-traffic/freeway_detector_config/Freeways-Metadata-2010_01_01/copyright(san%20diego).txt 35 | 36 | Usage: 37 | 38 | Run the script like this to 'replay', with pauses in data publication 39 | consistent with pauses in the series of data timestamps, which arrive every 5 40 | minutes: 41 | % python traffic_pubsub_generator.py --filename 'yourdatafile.csv' --replay 42 | 43 | To restrict to N lines, do something like this: 44 | % python traffic_pubsub_generator.py --filename 'yourdatafile.csv' \ 45 | --num_lines 10 --replay 46 | 47 | To alter the data timestamps to start from the script time, add 48 | the --current flag. 49 | If you want to set the topics from the command line, use 50 | the --topic and --incident_topic flags. 51 | Run 'python traffic_pubsub_generator.py -h' for more information. 52 | """ 53 | import argparse 54 | import base64 55 | import csv 56 | import datetime 57 | import random 58 | import sys 59 | import time 60 | 61 | from apiclient import discovery 62 | from dateutil.parser import parse 63 | from oauth2client.client import GoogleCredentials 64 | 65 | # default; set to your traffic topic. Can override on command line. 66 | TRAFFIC_TOPIC = 'projects/your-project/topics/your-topic' 67 | # default; set to your incident topic. Can override on command line. 68 | INCIDENT_TOPIC = 'projects/your-project/topics/your-incident-topic' 69 | LINE_BATCHES = 100 # report periodic progress 70 | 71 | PUBSUB_SCOPES = ['https://www.googleapis.com/auth/pubsub'] 72 | NUM_RETRIES = 3 73 | INCIDENT_TYPES = ['Traffic Hazard - Vehicle', 'Traffic Collision - No Details', 74 | 'Traffic Collision - No Injuries', 75 | 'Traffic Collision - Ambulance Responding', 76 | 'Hit and Run - No Injuries', 77 | 'Vehicle Fire', 'Pedestrian on a Highway', 'Animal on Road'] 78 | INCIDENT_DURATION_RANGE = 60 # max in minutes of a randomly generated duration 79 | # The following value is used to determine whether an 80 | # 'incident' is generated for a given reading. Increase/decrease this value 81 | # to increase/decrease the likelihood that an incident is generated for a given 82 | # reading. 83 | INCIDENT_THRESH = 0.005 84 | 85 | 86 | def create_pubsub_client(): 87 | """Build the pubsub client.""" 88 | credentials = GoogleCredentials.get_application_default() 89 | if credentials.create_scoped_required(): 90 | credentials = credentials.create_scoped(PUBSUB_SCOPES) 91 | return discovery.build('pubsub', 'v1beta2', credentials=credentials) 92 | 93 | 94 | def publish(client, pubsub_topic, data_line, msg_attributes=None): 95 | """Publish to the given pubsub topic.""" 96 | data = base64.b64encode(data_line) 97 | msg_payload = {'data': data} 98 | if msg_attributes: 99 | msg_payload['attributes'] = msg_attributes 100 | body = {'messages': [msg_payload]} 101 | resp = client.projects().topics().publish( 102 | topic=pubsub_topic, body=body).execute(num_retries=NUM_RETRIES) 103 | return resp 104 | 105 | 106 | def maybe_add_delay(line, ts_int): 107 | """Randomly determine whether to simulate a publishing delay with this 108 | data element.""" 109 | # 10 mins in ms. Edit this value to change the amount of delay. 110 | ms_delay = 600000 111 | threshold = .005 112 | if random.random() < threshold: 113 | ts_int -= ms_delay # generate 10-min apparent delay 114 | print line 115 | line[0] = "%s" % datetime.datetime.utcfromtimestamp(ts_int/1000) 116 | print ("Delaying ts attr %s, %s, %s" % 117 | (ts_int, datetime.datetime.utcfromtimestamp(ts_int/1000), 118 | line)) 119 | return (line, ts_int) 120 | 121 | 122 | def process_current_mode(orig_date, diff, line, replay, random_delays): 123 | """When using --current flag, modify original data to generate updated time 124 | information.""" 125 | epoch = datetime.datetime(1970, 1, 1) 126 | if replay: # use adjusted date from line of data 127 | new_date = orig_date + diff 128 | line[0] = new_date.strftime("%Y-%m-%d %H:%M:%S") 129 | ts_int = int( 130 | ((new_date - epoch).total_seconds() * 1000) + 131 | new_date.microsecond/1000) 132 | # 'random_delays' indicates whether to include random apparent delays 133 | # in published data 134 | if random_delays: 135 | (line, ts_int) = maybe_add_delay(line, ts_int) 136 | return (line, str(ts_int)) 137 | else: # simply using current time 138 | currtime = datetime.datetime.utcnow() 139 | ts_int = int( 140 | ((currtime - epoch).total_seconds() * 1000) + 141 | currtime.microsecond/1000) 142 | line[0] = currtime.strftime("%Y-%m-%d %H:%M:%S") 143 | # 'random_delays' indicates whether to include random apparent delays 144 | # in published data 145 | if random_delays: 146 | (line, ts_int) = maybe_add_delay(line, ts_int) 147 | return (line, str(ts_int)) 148 | 149 | 150 | def process_noncurrent_mode(orig_date, line, random_delays): 151 | """Called when not using --current flag; retaining original time 152 | information in data.""" 153 | epoch = datetime.datetime(1970, 1, 1) 154 | ts_int = int( 155 | ((orig_date - epoch).total_seconds() * 1000) + 156 | orig_date.microsecond/1000) 157 | # 'random_delays' indicates whether to include random apparent delays 158 | # in published data 159 | if random_delays: 160 | (line, ts_int) = maybe_add_delay(line, ts_int) 161 | return (line, str(ts_int)) 162 | 163 | 164 | def publish_random_incident(client, incident_topic, incident_id, 165 | timestamp, station_id, freeway, travel_direction, 166 | msg_attributes=None): 167 | """Generate a random traffic 'incident' based on information from the 168 | given traffic reading, and publish it to the specified 'incidents' pubsub 169 | topic.""" 170 | duration = random.randrange(INCIDENT_DURATION_RANGE) # minutes 171 | cause = INCIDENT_TYPES[random.randrange(len(INCIDENT_TYPES))] 172 | data_line = '%s,%s,%s,%s,%s,%s,%s' % (incident_id, timestamp, duration, 173 | station_id, freeway, 174 | travel_direction, cause) 175 | print "incident data: %s" % data_line 176 | publish(client, incident_topic, data_line, msg_attributes) 177 | 178 | 179 | def main(argv): 180 | parser = argparse.ArgumentParser() 181 | parser.add_argument("--replay", help="Replay in 'real time'", 182 | action="store_true") 183 | parser.add_argument("--current", 184 | help="Use date adjusted from script start time.", 185 | action="store_true") 186 | parser.add_argument("--incidents", 187 | help="Whether to generate and publish (fake) " + 188 | "traffic incidents. Requires a second PubSub topic " + 189 | "to be specified.", 190 | action="store_true") 191 | parser.add_argument("--random_delays", 192 | help="Whether to randomly alter the data to " + 193 | "sometimes introduce delays between log date and " + 194 | "publish timestamp.", 195 | action="store_true") 196 | parser.add_argument("--filename", help="input filename") 197 | parser.add_argument("--num_lines", type=int, default=0, 198 | help="The number of lines to process. " + 199 | "0 indicates all.") 200 | parser.add_argument("--topic", default=TRAFFIC_TOPIC, 201 | help="The pubsub 'traffic' topic to publish to. " + 202 | "Should already exist.") 203 | parser.add_argument("--incident_topic", default=INCIDENT_TOPIC, 204 | help="The pubsub 'incident' topic to publish to. " + 205 | "Only used if the --incidents flag is set. " + 206 | "If so, should already exist.") 207 | args = parser.parse_args() 208 | 209 | pubsub_topic = args.topic 210 | print "Publishing to pubsub 'traffic' topic: %s" % pubsub_topic 211 | incidents = args.incidents 212 | random_delays = args.random_delays 213 | if incidents: 214 | incident_topic = args.incident_topic 215 | print "Publishing to pubsub 'incident' topic: %s" % incident_topic 216 | filename = args.filename 217 | print "filename: %s" % filename 218 | replay = args.replay 219 | print "replay mode: %s" % replay 220 | current = args.current 221 | print "current date mode: %s" % current 222 | num_lines = args.num_lines 223 | if num_lines: 224 | print "processing %s lines" % num_lines 225 | 226 | client = create_pubsub_client() 227 | dt = parse('01/01/2010 00:00:00') # earliest date in the traffic files 228 | now = datetime.datetime.utcnow() 229 | # used if altering date to replay from start time 230 | diff = now - dt 231 | # used if running in 'replay' mode, reflecting pauses in the data 232 | prev_date = dt 233 | restart_time = now 234 | line_count = 0 235 | incident_count = 0 236 | 237 | print "processing %s" % filename # process the traffic data file 238 | with open(filename) as data_file: 239 | reader = csv.reader(data_file) 240 | for line in reader: 241 | line_count += 1 242 | if num_lines: # if terminating after num_lines processed 243 | if line_count >= num_lines: 244 | print "Have processed %s lines" % num_lines 245 | break 246 | if (line_count % LINE_BATCHES) == 0: 247 | print "%s lines processed" % line_count 248 | ts = "" 249 | try: 250 | timestring = line[0] 251 | orig_date = parse(timestring) 252 | if current: # if using --current flag 253 | (line, ts) = process_current_mode( 254 | orig_date, diff, line, replay, random_delays) 255 | else: # not using --current flag 256 | (line, ts) = process_noncurrent_mode( 257 | orig_date, line, random_delays) 258 | 259 | if replay and orig_date != prev_date: 260 | date_delta = orig_date - prev_date 261 | print "date delta: %s" % date_delta.total_seconds() 262 | current_time = datetime.datetime.utcnow() 263 | timelapse = current_time - restart_time 264 | print "timelapse: %s" % timelapse.total_seconds() 265 | d2 = date_delta - timelapse 266 | sleeptime = d2.total_seconds() 267 | print "sleeping %s" % sleeptime 268 | time.sleep(sleeptime) 269 | restart_time = datetime.datetime.utcnow() 270 | print "restart_time is set to: %s" % restart_time 271 | prev_date = orig_date 272 | msg_attributes = {'timestamp': ts} 273 | publish(client, pubsub_topic, ",".join(line), msg_attributes) 274 | if incidents: # if generating traffic 'incidents' as well 275 | # randomly determine whether we'll generate an incident 276 | # associated with this reading. 277 | if random.random() < INCIDENT_THRESH: 278 | print "Generating a traffic incident for %s." % line 279 | # grab the timestring, station id, freeway, and 280 | # direction of travel. 281 | # Then generate some 'incident' data and publish it to 282 | # the incident topic. Use the incident count as a 283 | # simplistic id. 284 | incident_count += 1 285 | publish_random_incident(client, incident_topic, 286 | incident_count, 287 | line[0], line[1], line[2], 288 | line[3], msg_attributes) 289 | except ValueError, e: 290 | sys.stderr.write("---Error: %s for %s\n" % (e, line)) 291 | 292 | 293 | if __name__ == '__main__': 294 | main(sys.argv) 295 | -------------------------------------------------------------------------------- /grpc/README.md: -------------------------------------------------------------------------------- 1 | # cloud-pubsub-samples-python 2 | 3 | 4 | ## grpc 5 | 6 | This is a command line sample application using the Cloud Pub/Sub API 7 | via gRPC client library. The sample only lists topics in a given 8 | project. 9 | 10 | ## Prerequisites 11 | 12 | - Install grpc client library for Cloud Pub/Sub. Here is an example 13 | for installing it on your virtualenv environment. 14 | 15 | ``` 16 | $ virtualenv -p python2.7 --no-site-packages /some/dir 17 | $ source /some/dir/bin/activate 18 | $ pip install -r requirements.txt 19 | ``` 20 | 21 | ## Register your application 22 | 23 | - If you don't have a project, go to [Google Developers Console][1] 24 | and create a new project. 25 | 26 | - Enable the "Google Cloud Pub/Sub" API under "APIs & auth > APIs." 27 | 28 | - Go to "Credentials" and create a new Service Account. 29 | 30 | - Select "Generate new JSON key", then download a new JSON file. 31 | 32 | - Set the following environment variable. 33 | 34 | GOOGLE_APPLICATION_CREDENTIALS: the file path to the downloaded JSON file. 35 | 36 | ## Run the application 37 | 38 | ``` 39 | $ python pubsub_sample.py PROJECT_NAME 40 | ``` 41 | 42 | This will give you a list of topics in the given project. 43 | 44 | Enjoy! 45 | 46 | [1]: https://console.developers.google.com/project 47 | -------------------------------------------------------------------------------- /grpc/pubsub_sample.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | """Cloud Pub/Sub gRPC sample application.""" 18 | 19 | from __future__ import print_function 20 | 21 | 22 | import logging 23 | import sys 24 | 25 | from google.pubsub.v1 import pubsub_pb2 26 | from grpc.beta import implementations 27 | from grpc.framework.interfaces.face.face import NetworkError 28 | 29 | from oauth2client import client 30 | 31 | 32 | logging.basicConfig(level=logging.INFO) 33 | log = logging.getLogger(__name__) 34 | log.setLevel(logging.INFO) 35 | 36 | PUBSUB_ENDPOINT = "pubsub.googleapis.com" 37 | SSL_PORT = 443 38 | OAUTH_SCOPE = "https://www.googleapis.com/auth/pubsub", 39 | GOOGLE_CREDS = client.GoogleCredentials.get_application_default() 40 | SCOPED_CREDS = GOOGLE_CREDS.create_scoped(OAUTH_SCOPE) 41 | TIMEOUT = 30 42 | 43 | 44 | def auth_func(scoped_creds=SCOPED_CREDS): 45 | """Returns a token obtained from Google Creds.""" 46 | authn = scoped_creds.get_access_token().access_token 47 | return [('authorization', 'Bearer %s' % authn)] 48 | 49 | 50 | def make_channel_creds(ssl_creds, auth_func=auth_func): 51 | """Returns a channel with credentials callback.""" 52 | call_creds = implementations.metadata_call_credentials( 53 | lambda ctx, callback: callback(auth_func(), None)) 54 | return implementations.composite_channel_credentials(ssl_creds, call_creds) 55 | 56 | 57 | def create_pubsub_stub(host=PUBSUB_ENDPOINT, port=SSL_PORT): 58 | """Creates a secure pubsub channel.""" 59 | ssl_creds = implementations.ssl_channel_credentials(None, None, None) 60 | channel_creds = make_channel_creds(ssl_creds, auth_func) 61 | channel = implementations.secure_channel(host, port, channel_creds) 62 | return pubsub_pb2.beta_create_Publisher_stub(channel) 63 | 64 | 65 | def list_topics(stub, project): 66 | """Lists topics in the given project.""" 67 | req = pubsub_pb2.ListTopicsRequest(project=project) 68 | try: 69 | resp = stub.ListTopics(req, TIMEOUT) 70 | for t in resp.topics: 71 | print("Topic is: {}".format(t.name)) 72 | except NetworkError, e: 73 | logging.warning('Failed to list topics: {}'.format(e)) 74 | sys.exit(1) 75 | 76 | 77 | def usage(): 78 | """Prints usage to the stderr.""" 79 | print('{} project_id'.format(sys.argv[0]), file=sys.stderr) 80 | 81 | 82 | def main(): 83 | if len(sys.argv) < 2: 84 | usage() 85 | exit(1) 86 | stub = create_pubsub_stub() 87 | list_topics(stub, 'projects/{}'.format(sys.argv[1])) 88 | 89 | 90 | if __name__ == '__main__': 91 | main() 92 | -------------------------------------------------------------------------------- /grpc/requirements.txt: -------------------------------------------------------------------------------- 1 | grpc-google-pubsub-v1 2 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | envlist = {py27}-{nosetest,pep8,grpc} 4 | 5 | [testenv] 6 | passenv = PYTHONPATH GCLOUD_* TEST_* TRAVIS* 7 | setenv = 8 | GOOGLE_APPLICATION_CREDENTIALS = {toxinidir}/client-secret.json 9 | PYTHONPATH = {toxinidir}/cmdline-pull 10 | basepython = 11 | py27: python2.7 12 | deps = 13 | google-api-python-client 14 | pep8: flake8 15 | pep8: flake8-import-order 16 | nosetest: mock 17 | nosetest: nose 18 | nosetest: httplib2 19 | changedir = 20 | grpc: grpc 21 | commands = 22 | # TOOD: decrease the max allowed complexity to 10 after adding tests 23 | pep8: flake8 --max-complexity=13 --exclude=lib,bin,local \ 24 | pep8: --import-order-style=google \ 25 | pep8: --application-import-names=constants,pubsub_utils 26 | nosetest: nosetests cmdline-pull 27 | nosetest: nosetests appengine-push/test_deploy.py 28 | grpc: pip install -r requirements.txt 29 | grpc: python pubsub_sample.py cloud-pubsub-sample-test 30 | 31 | [flake8] 32 | application-import-names = constants,pubsub_utils 33 | --------------------------------------------------------------------------------