├── .coveragerc ├── .github └── workflows │ ├── deploy.yml │ └── tests.yml ├── .gitignore ├── AUTHORS ├── LICENSE ├── MANIFEST.in ├── README ├── README.md ├── RELEASING.md ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── oauth_creds ├── test.png ├── test_cmdline.py ├── test_internals.py ├── test_sanity.py └── test_util.py ├── twitter ├── __init__.py ├── ansi.py ├── api.py ├── archiver.py ├── auth.py ├── cmdline.py ├── follow.py ├── ircbot.py ├── logger.py ├── oauth.py ├── oauth2.py ├── oauth_dance.py ├── stream.py ├── stream_example.py ├── timezones.py ├── twitter_globals.py └── util.py └── utils └── update.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | relative_files = True 3 | 4 | [report] 5 | omit = 6 | */python?.?/* 7 | */pypy/* 8 | */site-packages/nose/* 9 | tests/* 10 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | release: 8 | types: 9 | - published 10 | 11 | jobs: 12 | build: 13 | if: github.repository == 'python-twitter-tools/twitter' 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v3 18 | with: 19 | fetch-depth: 0 20 | 21 | - name: Set up Python 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: "3.x" 25 | cache: pip 26 | cache-dependency-path: setup.py 27 | 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install -U pip 31 | python -m pip install -U setuptools twine wheel 32 | python -m pip install -r requirements.txt 33 | 34 | - name: Build package 35 | run: | 36 | python setup.py --version 37 | python setup.py sdist --format=gztar bdist_wheel 38 | twine check dist/* 39 | 40 | - name: Publish package to PyPI 41 | if: github.event.action == 'published' 42 | uses: pypa/gh-action-pypi-publish@master 43 | with: 44 | user: __token__ 45 | password: ${{ secrets.pypi_password }} 46 | 47 | - name: Publish package to TestPyPI 48 | uses: pypa/gh-action-pypi-publish@release/v1 49 | with: 50 | user: __token__ 51 | password: ${{ secrets.test_pypi_password }} 52 | repository_url: https://test.pypi.org/legacy/ 53 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push, pull_request, workflow_dispatch] 4 | 5 | env: 6 | FORCE_COLOR: 1 7 | 8 | jobs: 9 | tests: 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | fail-fast: false 13 | max-parallel: 1 14 | matrix: 15 | python-version: [ 16 | "2.7", 17 | "3.7", 18 | "3.8", 19 | "3.9", 20 | "3.10", 21 | "3.11", 22 | "pypy2.7", 23 | ] 24 | os: [ubuntu-latest] 25 | 26 | steps: 27 | - uses: actions/checkout@v3 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v4 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | cache: pip 34 | cache-dependency-path: setup.py 35 | 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install -U pip 39 | python -m pip install -U pytest pytest-cov mock 40 | python -m pip install -r requirements.txt 41 | 42 | - name: Tests 43 | shell: bash 44 | run: | 45 | pytest --cov twitter --cov tests --cov-report=xml 46 | 47 | - name: Coveralls 48 | uses: AndreMiras/coveralls-python-action@develop 49 | with: 50 | parallel: true 51 | 52 | coveralls_finish: 53 | needs: tests 54 | runs-on: ubuntu-latest 55 | steps: 56 | - name: Coveralls finished 57 | uses: AndreMiras/coveralls-python-action@develop 58 | with: 59 | parallel-finished: true 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Python 2 | twitter.egg-info 3 | twitter3.egg-info 4 | *.bak 5 | *.orig 6 | *.rej 7 | *~ 8 | .idea 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | env/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | 59 | # Sphinx documentation 60 | docs/_build/ 61 | 62 | # PyBuilder 63 | target/ 64 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Developers: 2 | Mike Verdone 3 | Hatem Nassrat 4 | Wes Devauld 5 | 6 | Contributors: 7 | Horacio Duran (utf-8 patch for IRC bot) 8 | Rainer Michael Schmid (bugfix: crash when redirecting output to a file in 1.1) 9 | Anders Sandvig (cmdline -l, -d, and -t flags) 10 | Mark Hammond (OAuth support in API) 11 | Prashant Pawar (IRC bot multi-channel support) 12 | David Bittencourt (python 2.3 support) 13 | Bryan Clark (HTTP encoding bugfix, improved exception logging) 14 | Irfan Ahmad (Fixed #91 rate limit headers and #99 twitter API timeouts) 15 | StalkR (archiver, follow) 16 | Matthew Cengia (DM support, ISO timezone support, more API 1.1 support) 17 | Andrew (Fixed streams support for HTTP1.1 chunked answers) 18 | Benjamin Ooghe-Tabanou (Helped fix streams support for HTTP1.1 chunked answers, added image support and more API 1.1 support) 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008 Mike Verdone 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | prune .github 2 | include tests/*.py 3 | include tests/*.png 4 | prune utils 5 | exclude .coveragerc 6 | exclude .gitignore 7 | exclude README.md 8 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Python Twitter Tools 2 | ==================== 3 | 4 | > [!IMPORTANT] 5 | > This project is no longer maintained and has been archived. 6 | > The existing releases and code remain available in their current state. 7 | > Thank you to everyone who used and contributed to the success of this project. 8 | 9 | The Minimalist Twitter API for Python is a Python API for Twitter, 10 | everyone's favorite Web 2.0 Facebook-style status updater for people 11 | on the go. 12 | 13 | Also included is a Twitter command-line tool for getting your friends' 14 | tweets and setting your own tweet from the safety and security of your 15 | favorite shell and an IRC bot that can announce Twitter updates to an 16 | IRC channel. 17 | 18 | For more information: 19 | 20 | * install the [package](https://pypi.org/project/twitter/) `pip install twitter` 21 | * import the `twitter` package and run `help()` on it 22 | * run `twitter -h` for command-line tool help 23 | 24 | twitter - The Command-Line Tool 25 | ------------------------------- 26 | 27 | The command-line tool lets you do some awesome things: 28 | 29 | * view your tweets, recent replies, and tweets in lists 30 | * view the public timeline 31 | * follow and unfollow (leave) friends 32 | * various output formats for tweet information 33 | 34 | The bottom line: type `twitter`, receive tweets. 35 | 36 | twitterbot - The IRC Bot 37 | ------------------------ 38 | 39 | The IRC bot is associated with a Twitter account (either your own account or an 40 | account you create for the bot). The bot announces all tweets from friends 41 | it is following. It can be made to follow or leave friends through IRC /msg 42 | commands. 43 | 44 | 45 | `twitter-log` 46 | ------------- 47 | 48 | `twitter-log` is a simple command-line tool that dumps all public 49 | tweets from a given user in a simple text format. It is useful to get 50 | a complete offsite backup of all your tweets. Run `twitter-log` and 51 | read the instructions. 52 | 53 | `twitter-archiver` and `twitter-follow` 54 | --------------------------------------- 55 | 56 | twitter-archiver will log all the tweets posted by any user since they 57 | started posting. twitter-follow will print a list of all of all the 58 | followers of a user (or all the users that user follows). 59 | 60 | 61 | Programming with the Twitter API classes 62 | ======================================== 63 | 64 | The `Twitter` and `TwitterStream` classes are the key to building your own 65 | Twitter-enabled applications. 66 | 67 | 68 | The `Twitter` class 69 | ------------------- 70 | 71 | The minimalist yet fully featured Twitter API class. 72 | 73 | Get RESTful data by accessing members of this class. The result 74 | is decoded python objects (lists and dicts). 75 | 76 | The Twitter API is documented at: 77 | 78 | **[https://developer.twitter.com/en/docs](https://developer.twitter.com/en/docs)** 79 | 80 | The list of most accessible functions is listed at: 81 | 82 | **[https://developer.twitter.com/en/docs/api-reference-index](https://developer.twitter.com/en/docs/api-reference-index)** 83 | 84 | Examples: 85 | 86 | ```python 87 | from twitter import * 88 | 89 | t = Twitter( 90 | auth=OAuth(token, token_secret, consumer_key, consumer_secret)) 91 | 92 | # Get your "home" timeline 93 | t.statuses.home_timeline() 94 | 95 | # Get a particular friend's timeline 96 | t.statuses.user_timeline(screen_name="boogheta") 97 | 98 | # to pass in GET/POST parameters, such as `count` 99 | t.statuses.home_timeline(count=5) 100 | 101 | # to pass in the GET/POST parameter `id` you need to use `_id` 102 | t.statuses.show(_id=1234567890) 103 | 104 | # Update your status 105 | t.statuses.update( 106 | status="Using @boogheta's sweet Python Twitter Tools.") 107 | 108 | # Send a direct message 109 | t.direct_messages.events.new( 110 | _json={ 111 | "event": { 112 | "type": "message_create", 113 | "message_create": { 114 | "target": { 115 | "recipient_id": t.users.show(screen_name="boogheta")["id"]}, 116 | "message_data": { 117 | "text": "I think yer swell!"}}}}) 118 | 119 | # Get the members of maxmunnecke's list "network analysis tools" (grab the list_id within the url) https://twitter.com/i/lists/1130857490764091392 120 | t.lists.members(owner_screen_name="maxmunnecke", list_id="1130857490764091392") 121 | 122 | # Favorite/like a status 123 | status = t.statuses.home_timeline()[0] 124 | if not status['favorited']: 125 | t.favorites.create(_id=status['id']) 126 | 127 | # An *optional* `_timeout` parameter can also be used for API 128 | # calls which take much more time than normal or twitter stops 129 | # responding for some reason: 130 | t.users.lookup( 131 | screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), _timeout=1) 132 | 133 | # Overriding Method: GET/POST 134 | # you should not need to use this method as this library properly 135 | # detects whether GET or POST should be used, Nevertheless 136 | # to force a particular method, use `_method` 137 | t.statuses.oembed(_id=1234567890, _method='GET') 138 | 139 | # Send images along with your tweets: 140 | # - first just read images from the web or from files the regular way: 141 | with open("example.png", "rb") as imagefile: 142 | imagedata = imagefile.read() 143 | # - then upload medias one by one on Twitter's dedicated server 144 | # and collect each one's id: 145 | t_upload = Twitter(domain='upload.twitter.com', 146 | auth=OAuth(token, token_secret, consumer_key, consumer_secret)) 147 | id_img1 = t_upload.media.upload(media=imagedata)["media_id_string"] 148 | id_img2 = t_upload.media.upload(media=imagedata)["media_id_string"] 149 | # - finally send your tweet with the list of media ids: 150 | t.statuses.update(status="PTT ★", media_ids=",".join([id_img1, id_img2])) 151 | 152 | # Or send a tweet with an image (or set a logo/banner similarly) 153 | # using the old deprecated method that will probably disappear some day 154 | params = {"media[]": imagedata, "status": "PTT ★"} 155 | # Or for an image encoded as base64: 156 | params = {"media[]": base64_image, "status": "PTT ★", "_base64": True} 157 | t.statuses.update_with_media(**params) 158 | 159 | # Attach text metadata to medias sent, using the upload.twitter.com route 160 | # using the _json workaround to send json arguments as POST body 161 | # (warning: to be done before attaching the media to a tweet) 162 | t_upload.media.metadata.create(_json={ 163 | "media_id": id_img1, 164 | "alt_text": { "text": "metadata generated via PTT!" } 165 | }) 166 | # or with the shortcut arguments ("alt_text" and "text" work): 167 | t_upload.media.metadata.create(media_id=id_img1, text="metadata generated via PTT!") 168 | 169 | # Alternatively, you can reuse the originally instantiated object, 170 | # changing the domain, that is: 171 | t.domain = 'upload.twitter.com' 172 | 173 | # Now you can upload the image (or images). 174 | id_img1 = t.media.upload(media=imagedata)['media_id_string'] 175 | id_img2 = t.media.upload(media=imagedata)["media_id_string"] 176 | 177 | # You now can reset the domain to the original one: 178 | t.domain = 'api.twitter.com' 179 | 180 | # And you can send the update: 181 | t.statuses.update(status="PTT ★", media_ids=",".join([id_img1, id_img2])) 182 | 183 | 184 | ``` 185 | 186 | Searching Twitter: 187 | ```python 188 | # Search for the latest tweets about #pycon 189 | t.search.tweets(q="#pycon") 190 | 191 | # Search for the latest tweets about #pycon, using [extended mode](https://developer.twitter.com/en/docs/tweets/tweet-updates) 192 | t.search.tweets(q="#pycon", tweet_mode='extended') 193 | ``` 194 | 195 | 196 | Retrying after reaching the API rate limit 197 | ------------------------------------------ 198 | 199 | Simply create the `Twitter` instance with the argument `retry=True`, then the 200 | HTTP error codes `429`, `502`, `503`, and `504` will cause a retry of the last 201 | request. 202 | 203 | If `retry` is an integer, it defines the maximum number of retry attempts. 204 | 205 | 206 | Using the data returned 207 | ----------------------- 208 | 209 | Twitter API calls return decoded JSON. This is converted into 210 | a bunch of Python lists, dicts, ints, and strings. For example: 211 | 212 | ```python 213 | x = twitter.statuses.home_timeline() 214 | 215 | # The first 'tweet' in the timeline 216 | x[0] 217 | 218 | # The screen name of the user who wrote the first 'tweet' 219 | x[0]['user']['screen_name'] 220 | ``` 221 | 222 | Getting raw XML data 223 | -------------------- 224 | 225 | If you prefer to get your Twitter data in XML format, pass 226 | `format="xml"` to the `Twitter` object when you instantiate it: 227 | 228 | ```python 229 | twitter = Twitter(format="xml") 230 | ``` 231 | 232 | The output will not be parsed in any way. It will be a raw string 233 | of XML. 234 | 235 | The `TwitterStream` class 236 | ------------------------- 237 | 238 | The `TwitterStream` object is an interface to the Twitter Stream 239 | API. This can be used pretty much the same as the `Twitter` class, 240 | except the result of calling a method will be an iterator that 241 | yields objects decoded from the stream. For example:: 242 | 243 | ```python 244 | twitter_stream = TwitterStream(auth=OAuth(...)) 245 | iterator = twitter_stream.statuses.sample() 246 | 247 | for tweet in iterator: 248 | ...do something with this tweet... 249 | ``` 250 | 251 | Per default the `TwitterStream` object uses 252 | [public streams](https://dev.twitter.com/docs/streaming-apis/streams/public). 253 | If you want to use one of the other 254 | [streaming APIs](https://dev.twitter.com/docs/streaming-apis), specify the URL 255 | manually. 256 | 257 | The iterator will `yield` until the TCP connection breaks. When the 258 | connection breaks, the iterator yields `{'hangup': True}` (and 259 | raises `StopIteration` if iterated again). 260 | 261 | Similarly, if the stream does not produce heartbeats for more than 262 | 90 seconds, the iterator yields `{'hangup': True, 263 | 'heartbeat_timeout': True}` (and raises `StopIteration` if 264 | iterated again). 265 | 266 | The `timeout` parameter controls the maximum time between 267 | yields. If it is nonzero, then the iterator will yield either 268 | stream data or `{'timeout': True}` within the timeout period. This 269 | is useful if you want your program to do other stuff in between 270 | waiting for tweets. 271 | 272 | The `block` parameter sets the stream to be fully non-blocking. 273 | In this mode, the iterator always yields immediately. It returns 274 | stream data, or `None`. 275 | 276 | Note that `timeout` supercedes this argument, so it should also be 277 | set `None` to use this mode, and non-blocking can potentially lead 278 | to 100% CPU usage. 279 | 280 | Twitter `Response` Objects 281 | -------------------------- 282 | 283 | Response from a Twitter request. Behaves like a list or a string 284 | (depending on requested format), but it has a few other interesting 285 | attributes. 286 | 287 | `headers` gives you access to the response headers as an 288 | `httplib.HTTPHeaders` instance. Use `response.headers.get('h')` 289 | to retrieve a header. 290 | 291 | Authentication 292 | -------------- 293 | 294 | You can authenticate with Twitter in three ways: NoAuth, OAuth, or 295 | OAuth2 (app-only). Get `help()` on these classes to learn how to use them. 296 | 297 | OAuth and OAuth2 are probably the most useful. 298 | 299 | 300 | Working with OAuth 301 | ------------------ 302 | 303 | Visit the Twitter developer page and create a new application: 304 | 305 | **[https://dev.twitter.com/apps/new](https://dev.twitter.com/apps/new)** 306 | 307 | This will get you a `CONSUMER_KEY` and `CONSUMER_SECRET`. 308 | 309 | When users run your application they have to authenticate your app 310 | with their Twitter account. A few HTTP calls to Twitter are required 311 | to do this. Please see the `twitter.oauth_dance` module to see how this 312 | is done. If you are making a command-line app, you can use the 313 | `oauth_dance()` function directly. 314 | 315 | Performing the "oauth dance" gets you an oauth token and oauth secret 316 | that authenticate the user with Twitter. You should save these for 317 | later, so that the user doesn't have to do the oauth dance again. 318 | 319 | `read_token_file` and `write_token_file` are utility methods to read and 320 | write OAuth `token` and `secret` key values. The values are stored as 321 | strings in the file. Not terribly exciting. 322 | 323 | Finally, you can use the `OAuth` authenticator to connect to Twitter. In 324 | code it all goes like this: 325 | 326 | ```python 327 | from twitter import * 328 | 329 | MY_TWITTER_CREDS = os.path.expanduser('~/.my_app_credentials') 330 | if not os.path.exists(MY_TWITTER_CREDS): 331 | oauth_dance("My App Name", CONSUMER_KEY, CONSUMER_SECRET, 332 | MY_TWITTER_CREDS) 333 | 334 | oauth_token, oauth_secret = read_token_file(MY_TWITTER_CREDS) 335 | 336 | twitter = Twitter(auth=OAuth( 337 | oauth_token, oauth_secret, CONSUMER_KEY, CONSUMER_SECRET)) 338 | 339 | # Now work with Twitter 340 | twitter.statuses.update(status='Hello, world!') 341 | ``` 342 | 343 | Working with `OAuth2` 344 | --------------------- 345 | 346 | Twitter only supports the application-only flow of OAuth2 for certain 347 | API endpoints. This OAuth2 authenticator only supports the application-only 348 | flow right now. 349 | 350 | To authenticate with OAuth2, visit the Twitter developer page and create a new 351 | application: 352 | 353 | **[https://dev.twitter.com/apps/new](https://dev.twitter.com/apps/new)** 354 | 355 | This will get you a `CONSUMER_KEY` and `CONSUMER_SECRET`. 356 | 357 | Exchange your `CONSUMER_KEY` and `CONSUMER_SECRET` for a bearer token using the 358 | `oauth2_dance` function. 359 | 360 | Finally, you can use the `OAuth2` authenticator and your bearer token to connect 361 | to Twitter. In code it goes like this:: 362 | 363 | ```python 364 | twitter = Twitter(auth=OAuth2(bearer_token=BEARER_TOKEN)) 365 | 366 | # Now work with Twitter 367 | twitter.search.tweets(q='keyword') 368 | ``` 369 | 370 | License 371 | ======= 372 | 373 | Python Twitter Tools are released under an MIT License. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | README -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Release Checklist 2 | 3 | - [ ] Get master to the appropriate code release state. 4 | 5 | - [ ] Publish release with new tag like `twitter-x.y.z`: 6 | https://github.com/python-twitter-tools/twitter/releases/new 7 | 8 | - [ ] Check the tagged 9 | [GitHub Actions build](https://github.com/python-twitter-tools/twitter/actions?query=workflow%3ADeploy) 10 | has deployed to [PyPI](https://pypi.org/project/twitter/#history) 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("README") as f: 4 | long_description = f.read() 5 | 6 | 7 | def local_scheme(version): 8 | """Skip the local version (eg. +xyz of 0.6.1.dev4+gdf99fe2) 9 | to be able to upload to Test PyPI""" 10 | return "" 11 | 12 | setup(name='twitter', 13 | description="An API and command-line toolset for Twitter (twitter.com)", 14 | long_description=long_description, 15 | long_description_content_type="text/markdown", 16 | python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", 17 | # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers 18 | classifiers=[ 19 | "Development Status :: 5 - Production/Stable", 20 | "Environment :: Console", 21 | "Intended Audience :: End Users/Desktop", 22 | "Natural Language :: English", 23 | "Operating System :: OS Independent", 24 | "Programming Language :: Python :: 2", 25 | "Programming Language :: Python :: 2.7", 26 | "Programming Language :: Python :: 3", 27 | "Programming Language :: Python :: 3.7", 28 | "Programming Language :: Python :: 3.8", 29 | "Programming Language :: Python :: 3.9", 30 | "Programming Language :: Python :: 3.10", 31 | "Programming Language :: Python :: 3.11", 32 | "Programming Language :: Python :: Implementation :: CPython", 33 | "Programming Language :: Python :: Implementation :: PyPy", 34 | "Topic :: Communications :: Chat :: Internet Relay Chat", 35 | "Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries", 36 | "Topic :: Utilities", 37 | "License :: OSI Approved :: MIT License", 38 | ], 39 | keywords='twitter, IRC, command-line tools, web 2.0', 40 | author='Mike Verdone', 41 | author_email='mike.verdone+twitterapi@gmail.com', 42 | url='https://mike.verdone.ca/twitter/', 43 | license='MIT License', 44 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 45 | include_package_data=True, 46 | zip_safe=True, 47 | use_scm_version={"local_scheme": local_scheme}, 48 | setup_requires=["setuptools_scm"], 49 | install_requires=["certifi"], 50 | entry_points=""" 51 | # -*- Entry points: -*- 52 | [console_scripts] 53 | twitter=twitter.cmdline:main 54 | twitterbot=twitter.ircbot:main 55 | twitter-log=twitter.logger:main 56 | twitter-archiver=twitter.archiver:main 57 | twitter-follow=twitter.follow:main 58 | twitter-stream-example=twitter.stream_example:main 59 | """, 60 | ) 61 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-twitter-tools/twitter/353cb7426939822331d385aaa92fc04c8b422643/tests/__init__.py -------------------------------------------------------------------------------- /tests/oauth_creds: -------------------------------------------------------------------------------- 1 | 262332119-7bclllngLikvfVFAqgSe1cRNUE0cnVSYD2YOX7Ju 2 | aRTJHmcY6szoRLCfHZccTkCqX7yrL3B4fYjwB5ZrI 3 | -------------------------------------------------------------------------------- /tests/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/python-twitter-tools/twitter/353cb7426939822331d385aaa92fc04c8b422643/tests/test.png -------------------------------------------------------------------------------- /tests/test_cmdline.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | import json 4 | import re 5 | import unittest 6 | try: 7 | from mock import patch 8 | except ImportError: 9 | from unittest.mock import patch 10 | 11 | try: 12 | import HTMLParser 13 | except ImportError: 14 | import html.parser as HTMLParser 15 | 16 | from twitter.cmdline import ( 17 | replaceInStatus, 18 | StatusFormatter, 19 | VerboseStatusFormatter, 20 | JSONStatusFormatter, 21 | ) 22 | 23 | 24 | class TestCmdLine(unittest.TestCase): 25 | 26 | status = { 27 | "created_at": "sun dec 20 18:33:30 +0000 2020", 28 | "user": {"screen_name": "myusername", "location": "Paris, France"}, 29 | "text": "test & test", 30 | } 31 | 32 | def test_replaceInStatus(self): 33 | status = "my&status @twitter #tag" 34 | assert replaceInStatus(status) == "my&status @twitter #tag" 35 | 36 | @patch("twitter.cmdline.get_time_string", return_value="18:33:30 ") 37 | def test_StatusFormatter(self, mock_get_time_string): 38 | test_status = StatusFormatter() 39 | options = {"timestamp": True, "datestamp": False} 40 | assert test_status(self.status, options) == "18:33:30 @myusername test & test" 41 | 42 | def test_VerboseStatusFormatter(self): 43 | test_status = VerboseStatusFormatter() 44 | options = {"timestamp": True, "datestamp": False} 45 | assert ( 46 | test_status(self.status, options) 47 | == "-- myusername (Paris, France) on sun dec 20 18:33:30 +0000 2020\ntest & test\n" 48 | ) 49 | 50 | def test_JSONStatusFormatter(self): 51 | test_status = JSONStatusFormatter() 52 | options = {"timestamp": True, "datestamp": False} 53 | assert test_status(self.status, options) == json.dumps( 54 | { 55 | "created_at": "sun dec 20 18:33:30 +0000 2020", 56 | "user": {"screen_name": "myusername", "location": "Paris, France"}, 57 | "text": "test & test", 58 | } 59 | ) 60 | -------------------------------------------------------------------------------- /tests/test_internals.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from twitter.api import method_for_uri, build_uri 5 | from twitter.util import PY_3_OR_HIGHER, actually_bytes 6 | 7 | def test_method_for_uri__lookup(): 8 | assert "POST" == method_for_uri("/1.1/users/lookup") 9 | assert "POST" == method_for_uri("/1.1/statuses/lookup") 10 | assert "POST" == method_for_uri("/1.1/users/lookup/12345") 11 | assert "GET" == method_for_uri("/1.1/friendships/lookup") 12 | 13 | def test_build_uri(): 14 | uri = build_uri(["1.1", "foo", "bar"], {}) 15 | assert uri == "1.1/foo/bar" 16 | 17 | # Interpolation works 18 | uri = build_uri(["1.1", "_foo", "bar"], {"_foo": "asdf"}) 19 | assert uri == "1.1/asdf/bar" 20 | 21 | # But only for strings beginning with _. 22 | uri = build_uri(["1.1", "foo", "bar"], {"foo": "asdf"}) 23 | assert uri == "1.1/foo/bar" 24 | 25 | def test_actually_bytes(): 26 | out_type = str 27 | if PY_3_OR_HIGHER: 28 | out_type = bytes 29 | for inp in [b"asdf", "asdf", "asdfüü", 1234]: 30 | assert type(actually_bytes(inp)) == out_type 31 | -------------------------------------------------------------------------------- /tests/test_sanity.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | import os 5 | from random import choice 6 | import time 7 | import pickle 8 | import json 9 | 10 | from twitter import Twitter, NoAuth, OAuth, read_token_file, TwitterHTTPError 11 | from twitter.api import TwitterDictResponse, TwitterListResponse, POST_ACTIONS, method_for_uri 12 | from twitter.cmdline import CONSUMER_KEY, CONSUMER_SECRET 13 | 14 | noauth = NoAuth() 15 | oauth = OAuth(*read_token_file('tests/oauth_creds') 16 | + (CONSUMER_KEY, CONSUMER_SECRET)) 17 | 18 | twitter11 = Twitter(domain='api.twitter.com', 19 | auth=oauth, 20 | api_version='1.1') 21 | 22 | twitter_upl = Twitter(domain='upload.twitter.com', 23 | auth=oauth, 24 | api_version='1.1') 25 | 26 | twitter11_na = Twitter(domain='api.twitter.com', 27 | auth=noauth, 28 | api_version='1.1') 29 | 30 | AZaz = "abcdefghijklmnopqrstuvwxyz1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ" 31 | 32 | b64_image_data = b"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAAXNSR0IArs4c6QAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB94JFhMBAJv5kaUAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAA4UlEQVQoz7WSIZLGIAxG6c5OFZjianBcIOfgPkju1DsEBWfAUEcNGGpY8Xe7dDoVFRvHfO8NJGRorZE39UVe1nd/WNfVObcsi3OOEAIASikAmOf5D2q/FWPUWgshKKWfiFIqhNBaxxhPjPQ05/z+Bs557xw9hBC89ymlu5BS8t6HEC5NW2sR8alRRLTWXoRSSinlSejT12M9BAAAgCeoTw9BSimlfBIu6WdYtVZEVErdaaUUItZaL/9wOsaY83YAMMb0dGtt6Jdv3/ec87ZtOWdCCGNsmibG2DiOJzP8+7b+AAOmsiPxyHWCAAAAAElFTkSuQmCC" 33 | 34 | def get_random_str(): 35 | return ''.join(choice(AZaz) for _ in range(10)) 36 | 37 | 38 | def test_API_set_tweet(unicod=False): 39 | random_tweet = "A random tweet %s" % \ 40 | ("with unicode üøπ" if unicod else "") + get_random_str() 41 | twitter11.statuses.update(status=random_tweet) 42 | time.sleep(5) 43 | recent = twitter11.statuses.user_timeline() 44 | assert recent 45 | assert isinstance(recent.rate_limit_remaining, int) 46 | assert isinstance(recent.rate_limit_reset, int) 47 | texts = [tweet['text'] for tweet in recent] 48 | assert random_tweet in texts 49 | 50 | def test_API_set_unicode_tweet(): 51 | test_API_set_tweet(unicod=True) 52 | 53 | 54 | def clean_link(text): 55 | pos = text.find(" https://t.co") 56 | if pos != -1: 57 | return text[:pos] 58 | return text 59 | 60 | __location__ = os.path.realpath( 61 | os.path.join(os.getcwd(), os.path.dirname(__file__))) 62 | 63 | def _img_data(): 64 | return open(os.path.join(__location__, "test.png"), "rb").read() 65 | 66 | def _test_API_old_media(img, _base64): 67 | random_tweet = ( 68 | "A random twitpic with unicode üøπ" 69 | + get_random_str()) 70 | params = {"status": random_tweet, "media[]": img, "_base64": _base64} 71 | twitter11.statuses.update_with_media(**params) 72 | time.sleep(5) 73 | recent = twitter11.statuses.user_timeline() 74 | assert recent 75 | texts = [clean_link(tweet['text']) for tweet in recent] 76 | assert random_tweet in texts 77 | 78 | def _test_API_set_unicode_twitpic_base64(): 79 | _test_API_old_media(b64_image_data, True) 80 | 81 | def _test_API_set_unicode_twitpic_base64_string(): 82 | _test_API_old_media(b64_image_data.decode('utf-8'), True) 83 | 84 | def _test_API_set_unicode_twitpic_auto_base64_convert(): 85 | _test_API_old_media(_img_data(), False) 86 | 87 | def _test_upload_media(): 88 | res = twitter_upl.media.upload(media=_img_data()) 89 | assert res 90 | assert res["media_id"] 91 | return str(res["media_id"]) 92 | 93 | def test_metadata_multipic(): 94 | pics = [_test_upload_media(), _test_upload_media(), _test_upload_media()] 95 | metadata = "metadata generated via PTT! ★" + get_random_str() 96 | res = twitter_upl.media.metadata.create(media_id=pics[0], text=metadata) 97 | random_tweet = ("I can even tweet multiple pictures at once and attach metadata onto some! ★ " 98 | + get_random_str()) 99 | res = twitter11.statuses.update(status=random_tweet, media_ids=",".join(pics)) 100 | assert res 101 | assert res["extended_entities"] 102 | assert len(res["extended_entities"]["media"]) == len(pics) 103 | recent = twitter11.statuses.user_timeline(include_ext_alt_text=True, include_entities=True) 104 | assert recent 105 | texts = [clean_link(t['text']) for t in recent] 106 | assert random_tweet in texts 107 | meta = recent[0].get("extended_entities", {}).get("media") 108 | assert meta 109 | assert metadata == meta[0].get("ext_alt_text", "") 110 | 111 | def test_search(): 112 | # In 1.1, search works on api.twitter.com not search.twitter.com 113 | # and requires authorisation 114 | results = twitter11.search.tweets(q='foo') 115 | assert results 116 | 117 | 118 | def test_get_trends(): 119 | # This is one method of inserting parameters, using named 120 | # underscore params. 121 | world_trends = twitter11.trends.available(_woeid=1) 122 | assert world_trends 123 | 124 | 125 | def test_get_trends_2(): 126 | # This is a nicer variation of the same call as above. 127 | world_trends = twitter11.trends._(1) 128 | assert world_trends 129 | 130 | 131 | def test_get_trends_3(): 132 | # Of course they broke it all again in 1.1... 133 | assert twitter11.trends.place(_id=1) 134 | 135 | 136 | def test_TwitterHTTPError_raised_for_invalid_oauth(): 137 | test_passed = False 138 | try: 139 | twitter11_na.statuses.mentions_timeline() 140 | except TwitterHTTPError: 141 | # this is the error we are looking for :) 142 | test_passed = True 143 | assert test_passed 144 | 145 | 146 | def test_picklability(): 147 | res = TwitterDictResponse({'a': 'b'}) 148 | p = pickle.dumps(res) 149 | res2 = pickle.loads(p) 150 | assert res == res2 151 | assert res2['a'] == 'b' 152 | 153 | res = TwitterListResponse([1, 2, 3]) 154 | p = pickle.dumps(res) 155 | res2 = pickle.loads(p) 156 | assert res == res2 157 | assert res2[2] == 3 158 | 159 | p = pickle.dumps(twitter11) 160 | s = pickle.loads(p) 161 | assert twitter11.domain == s.domain 162 | 163 | 164 | def test_jsonifability(): 165 | res = TwitterDictResponse({'a': 'b'}) 166 | p = json.dumps(res) 167 | res2 = json.loads(p) 168 | assert res == res2 169 | assert res2['a'] == 'b' 170 | 171 | res = TwitterListResponse([1, 2, 3]) 172 | p = json.dumps(res) 173 | res2 = json.loads(p) 174 | assert res == res2 175 | assert res2[2] == 3 176 | 177 | 178 | def test_method_for_uri(): 179 | for action in POST_ACTIONS: 180 | assert method_for_uri(get_random_str() + '/' + action) == 'POST' 181 | assert method_for_uri('statuses/timeline') == 'GET' 182 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from collections import namedtuple 5 | import contextlib 6 | import functools 7 | import socket 8 | import threading 9 | from twitter.util import find_links, follow_redirects, expand_line, parse_host_list 10 | 11 | try: 12 | import http.server as BaseHTTPServer 13 | import socketserver as SocketServer 14 | except ImportError: 15 | import BaseHTTPServer 16 | import SocketServer 17 | 18 | 19 | def test_find_links(): 20 | assert find_links("nix") == ("nix", []) 21 | assert find_links("http://abc") == ("%s", ["http://abc"]) 22 | assert find_links("t http://abc") == ("t %s", ["http://abc"]) 23 | assert find_links("http://abc t") == ("%s t", ["http://abc"]) 24 | assert find_links("1 http://a 2 http://b 3") == ("1 %s 2 %s 3", 25 | ["http://a", "http://b"]) 26 | assert find_links("%") == ("%%", []) 27 | assert find_links("(http://abc)") == ("(%s)", ["http://abc"]) 28 | 29 | 30 | Response = namedtuple('Response', 'path code headers') 31 | 32 | @contextlib.contextmanager 33 | def start_server(*resp): 34 | """HTTP server replying with the given responses to the expected 35 | requests.""" 36 | def url(port, path): 37 | return 'http://%s:%s%s' % (socket.gethostname().lower(), port, path) 38 | 39 | responses = list(reversed(resp)) 40 | 41 | class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler): 42 | def do_HEAD(self): 43 | response = responses.pop() 44 | assert response.path == self.path 45 | self.send_response(response.code) 46 | for header, value in list(response.headers.items()): 47 | self.send_header(header, value) 48 | self.end_headers() 49 | 50 | httpd = SocketServer.TCPServer(("", 0), MyHandler) 51 | t = threading.Thread(target=httpd.serve_forever) 52 | t.daemon = True 53 | t.start() 54 | port = httpd.server_address[1] 55 | yield functools.partial(url, port) 56 | httpd.shutdown() 57 | 58 | def test_follow_redirects_direct_link(): 59 | link = "/resource" 60 | with start_server(Response(link, 200, {})) as url: 61 | assert url(link) == follow_redirects(url(link)) 62 | 63 | def test_follow_redirects_redirected_link(): 64 | redirected = "/redirected" 65 | link = "/resource" 66 | with start_server( 67 | Response(link, 301, {"Location": redirected}), 68 | Response(redirected, 200, {})) as url: 69 | assert url(redirected) == follow_redirects(url(link)) 70 | 71 | def test_follow_redirects_unavailable(): 72 | link = "/resource" 73 | with start_server(Response(link, 404, {})) as url: 74 | assert url(link) == follow_redirects(url(link)) 75 | 76 | def test_follow_redirects_link_to_last_available(): 77 | unavailable = "/unavailable" 78 | link = "/resource" 79 | with start_server( 80 | Response(link, 301, {"Location": unavailable}), 81 | Response(unavailable, 404, {})) as url: 82 | assert url(unavailable) == follow_redirects(url(link)) 83 | 84 | 85 | def test_follow_redirects_no_where(): 86 | link = "http://links.nowhere/" 87 | assert link == follow_redirects(link) 88 | 89 | def test_follow_redirects_link_to_nowhere(): 90 | unavailable = "http://links.nowhere/" 91 | link = "/resource" 92 | with start_server( 93 | Response(link, 301, {"Location": unavailable})) as url: 94 | assert unavailable == follow_redirects(url(link)) 95 | 96 | def test_follow_redirects_filtered_by_site(): 97 | link = "/resource" 98 | with start_server() as url: 99 | assert url(link) == follow_redirects(url(link), ["other_host"]) 100 | 101 | 102 | def test_follow_redirects_filtered_by_site_after_redirect(): 103 | link = "/resource" 104 | redirected = "/redirected" 105 | filtered = "http://dont-follow/" 106 | with start_server( 107 | Response(link, 301, {"Location": redirected}), 108 | Response(redirected, 301, {"Location": filtered})) as url: 109 | hosts = [socket.gethostname().lower()] 110 | assert filtered == follow_redirects(url(link), hosts) 111 | 112 | def test_follow_redirects_filtered_by_site_allowed(): 113 | redirected = "/redirected" 114 | link = "/resource" 115 | with start_server( 116 | Response(link, 301, {"Location": redirected}), 117 | Response(redirected, 200, {})) as url: 118 | hosts = [socket.gethostname().lower()] 119 | assert url(redirected) == follow_redirects(url(link), hosts) 120 | 121 | def test_expand_line(): 122 | redirected = "/redirected" 123 | link = "/resource" 124 | with start_server( 125 | Response(link, 301, {"Location": redirected}), 126 | Response(redirected, 200, {})) as url: 127 | fmt = "before %s after" 128 | line = fmt % url(link) 129 | expected = fmt % url(redirected) 130 | assert expected == expand_line(line, None) 131 | 132 | def test_parse_host_config(): 133 | assert set() == parse_host_list("") 134 | assert set("h") == parse_host_list("h") 135 | assert set(["1", "2"]) == parse_host_list("1,2") 136 | assert set(["1", "2"]) == parse_host_list(" 1 , 2 ") 137 | 138 | -------------------------------------------------------------------------------- /twitter/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The minimalist yet fully featured Twitter API and Python toolset. 3 | 4 | The Twitter and TwitterStream classes are the key to building your own 5 | Twitter-enabled applications. 6 | 7 | """ 8 | 9 | from textwrap import dedent 10 | 11 | from .api import Twitter, TwitterError, TwitterHTTPError, TwitterResponse 12 | from .auth import NoAuth, UserPassAuth 13 | from .oauth import ( 14 | OAuth, read_token_file, write_token_file, 15 | __doc__ as oauth_doc) 16 | from .oauth2 import ( 17 | OAuth2, read_bearer_token_file, write_bearer_token_file, 18 | __doc__ as oauth2_doc) 19 | from .stream import TwitterStream 20 | from .oauth_dance import oauth_dance, oauth2_dance 21 | 22 | __doc__ = __doc__ or "" 23 | 24 | __doc__ += """ 25 | The Twitter class 26 | ----------------- 27 | """ 28 | __doc__ += dedent(Twitter.__doc__ or "") 29 | 30 | __doc__ += """ 31 | The TwitterStream class 32 | ----------------------- 33 | """ 34 | __doc__ += dedent(TwitterStream.__doc__ or "") 35 | 36 | 37 | __doc__ += """ 38 | Twitter Response Objects 39 | ------------------------ 40 | """ 41 | __doc__ += dedent(TwitterResponse.__doc__ or "") 42 | 43 | 44 | __doc__ += """ 45 | Authentication 46 | -------------- 47 | 48 | You can authenticate with Twitter in three ways: NoAuth, OAuth, or 49 | OAuth2 (app-only). Get help() on these classes to learn how to use them. 50 | 51 | OAuth and OAuth2 are probably the most useful. 52 | 53 | 54 | Working with OAuth 55 | ------------------ 56 | """ 57 | 58 | __doc__ += dedent(oauth_doc or "") 59 | 60 | __doc__ += """ 61 | Working with OAuth2 62 | ------------------- 63 | """ 64 | 65 | __doc__ += dedent(oauth2_doc or "") 66 | 67 | __all__ = [ 68 | "NoAuth", 69 | "OAuth", 70 | "OAuth2", 71 | "oauth2_dance", 72 | "oauth_dance", 73 | "read_bearer_token_file", 74 | "read_token_file", 75 | "Twitter", 76 | "TwitterError", 77 | "TwitterHTTPError", 78 | "TwitterResponse", 79 | "TwitterStream", 80 | "UserPassAuth", 81 | "write_bearer_token_file", 82 | "write_token_file", 83 | ] 84 | -------------------------------------------------------------------------------- /twitter/ansi.py: -------------------------------------------------------------------------------- 1 | """ 2 | Support for ANSI colours in command-line client. 3 | 4 | .. data:: ESC 5 | ansi escape character 6 | 7 | .. data:: RESET 8 | ansi reset colour (ansi value) 9 | 10 | .. data:: COLOURS_NAMED 11 | dict of colour names mapped to their ansi value 12 | 13 | .. data:: COLOURS_MIDS 14 | A list of ansi values for Mid Spectrum Colours 15 | """ 16 | 17 | import itertools 18 | import sys 19 | 20 | ESC = chr(0x1B) 21 | RESET = "0" 22 | 23 | COLOURS_NAMED = dict(list(zip( 24 | ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'], 25 | [str(x) for x in range(30, 38)] 26 | ))) 27 | COLOURS_MIDS = [ 28 | colour for name, colour in list(COLOURS_NAMED.items()) 29 | if name not in ('black', 'white') 30 | ] 31 | 32 | class AnsiColourException(Exception): 33 | ''' Exception while processing ansi colours ''' 34 | pass 35 | 36 | class ColourMap(object): 37 | ''' 38 | Object that allows for mapping strings to ansi colour values. 39 | ''' 40 | def __init__(self, colors=COLOURS_MIDS): 41 | ''' uses the list of ansi `colors` values to initialize the map ''' 42 | self._cmap = {} 43 | self._colourIter = itertools.cycle(colors) 44 | 45 | def colourFor(self, string): 46 | ''' 47 | Returns an ansi colour value given a `string`. 48 | The same ansi colour value is always returned for the same string 49 | ''' 50 | if string not in self._cmap: 51 | self._cmap[string] = next(self._colourIter) 52 | return self._cmap[string] 53 | 54 | class AnsiCmd(object): 55 | def __init__(self, forceAnsi): 56 | self.forceAnsi = forceAnsi 57 | 58 | def cmdReset(self): 59 | ''' Returns the ansi cmd colour for a RESET ''' 60 | if sys.stdout.isatty() or self.forceAnsi: 61 | return ESC + "[0m" 62 | else: 63 | return "" 64 | 65 | def cmdColour(self, colour): 66 | ''' 67 | Return the ansi cmd colour (i.e. escape sequence) 68 | for the ansi `colour` value 69 | ''' 70 | if sys.stdout.isatty() or self.forceAnsi: 71 | return ESC + "[" + colour + "m" 72 | else: 73 | return "" 74 | 75 | def cmdColourNamed(self, colour): 76 | ''' Return the ansi cmdColour for a given named `colour` ''' 77 | try: 78 | return self.cmdColour(COLOURS_NAMED[colour]) 79 | except KeyError: 80 | raise AnsiColourException('Unknown Colour %s' % (colour)) 81 | 82 | def cmdBold(self): 83 | if sys.stdout.isatty() or self.forceAnsi: 84 | return ESC + "[1m" 85 | else: 86 | return "" 87 | 88 | def cmdUnderline(self): 89 | if sys.stdout.isatty() or self.forceAnsi: 90 | return ESC + "[4m" 91 | else: 92 | return "" 93 | 94 | """These exist to maintain compatibility with users of version<=1.9.0""" 95 | def cmdReset(): 96 | return AnsiCmd(False).cmdReset() 97 | 98 | def cmdColour(colour): 99 | return AnsiCmd(False).cmdColour(colour) 100 | 101 | def cmdColourNamed(colour): 102 | return AnsiCmd(False).cmdColourNamed(colour) 103 | -------------------------------------------------------------------------------- /twitter/api.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals, print_function 3 | 4 | from .util import PY_3_OR_HIGHER, actually_bytes 5 | 6 | try: 7 | import ssl 8 | except ImportError: 9 | _HAVE_SSL = False 10 | else: 11 | _HAVE_SSL = True 12 | 13 | try: 14 | import urllib.request as urllib_request 15 | import urllib.error as urllib_error 16 | except ImportError: 17 | import urllib2 as urllib_request 18 | import urllib2 as urllib_error 19 | 20 | import certifi 21 | 22 | try: 23 | from cStringIO import StringIO 24 | except ImportError: 25 | from io import BytesIO as StringIO 26 | 27 | from .twitter_globals import POST_ACTIONS 28 | from .auth import NoAuth 29 | 30 | import re 31 | import sys 32 | import gzip 33 | from time import sleep, time 34 | 35 | try: 36 | import http.client as http_client 37 | except ImportError: 38 | import httplib as http_client 39 | 40 | try: 41 | import json 42 | except ImportError: 43 | import simplejson as json 44 | 45 | try: 46 | from json.decoder import JSONDecodeError 47 | except ImportError: 48 | JSONDecodeError = ValueError 49 | 50 | 51 | class _DEFAULT(object): 52 | pass 53 | 54 | 55 | class TwitterError(Exception): 56 | """ 57 | Base Exception thrown by the Twitter object when there is a 58 | general error interacting with the API. 59 | """ 60 | pass 61 | 62 | 63 | class TwitterHTTPError(TwitterError): 64 | """ 65 | Exception thrown by the Twitter object when there is an 66 | HTTP error interacting with twitter.com. 67 | """ 68 | 69 | def __init__(self, e, uri, format, uriparts): 70 | self.e = e 71 | self.uri = uri 72 | self.format = format 73 | self.uriparts = uriparts 74 | try: 75 | data = self.e.fp.read() 76 | except http_client.IncompleteRead as e: 77 | # can't read the error text 78 | # let's try some of it 79 | data = e.partial 80 | if self.e.headers.get('Content-Encoding') == 'gzip': 81 | buf = StringIO(data) 82 | f = gzip.GzipFile(fileobj=buf) 83 | data = f.read() 84 | if len(data) == 0: 85 | data = {} 86 | else: 87 | data = data.decode('utf8') 88 | if "json" == self.format: 89 | try: 90 | data = json.loads(data) 91 | except ValueError: 92 | # We try to load the response as json as a nicety; if it fails, carry on. 93 | pass 94 | self.response_data = data 95 | super(TwitterHTTPError, self).__init__(str(self)) 96 | 97 | def __str__(self): 98 | fmt = ("." + self.format) if self.format else "" 99 | return ( 100 | "Twitter sent status %i for URL: %s%s using parameters: " 101 | "(%s)\ndetails: %s" % ( 102 | self.e.code, self.uri, fmt, self.uriparts, 103 | self.response_data)) 104 | 105 | 106 | class TwitterResponse(object): 107 | """ 108 | Response from a twitter request. Behaves like a list or a string 109 | (depending on requested format) but it has a few other interesting 110 | attributes. 111 | 112 | `headers` gives you access to the response headers as an 113 | httplib.HTTPHeaders instance. You can do 114 | `response.headers.get('h')` to retrieve a header. 115 | """ 116 | 117 | @property 118 | def rate_limit_remaining(self): 119 | """ 120 | Remaining requests in the current rate-limit. 121 | """ 122 | return int(self.headers.get('X-Rate-Limit-Remaining', "0")) 123 | 124 | @property 125 | def rate_limit_limit(self): 126 | """ 127 | The rate limit ceiling for that given request. 128 | """ 129 | return int(self.headers.get('X-Rate-Limit-Limit', "0")) 130 | 131 | @property 132 | def rate_limit_reset(self): 133 | """ 134 | Time in UTC epoch seconds when the rate limit will reset. 135 | """ 136 | return int(self.headers.get('X-Rate-Limit-Reset', "0")) 137 | 138 | 139 | class TwitterDictResponse(dict, TwitterResponse): 140 | pass 141 | 142 | 143 | class TwitterListResponse(list, TwitterResponse): 144 | pass 145 | 146 | 147 | def wrap_response(response, headers): 148 | response_typ = type(response) 149 | if response_typ is dict: 150 | res = TwitterDictResponse(response) 151 | res.headers = headers 152 | elif response_typ is list: 153 | res = TwitterListResponse(response) 154 | res.headers = headers 155 | else: 156 | res = response 157 | return res 158 | 159 | 160 | POST_ACTIONS_RE = re.compile('(' + '|'.join(POST_ACTIONS) + r')(/\d+)?$') 161 | 162 | 163 | def method_for_uri(uri): 164 | if POST_ACTIONS_RE.search(uri): 165 | return "POST" 166 | return "GET" 167 | 168 | 169 | def build_uri(orig_uriparts, kwargs): 170 | """ 171 | Build the URI from the original uriparts and kwargs. Modifies kwargs. 172 | """ 173 | uriparts = [] 174 | for uripart in orig_uriparts: 175 | # If this part matches a keyword argument (starting with _), use 176 | # the supplied value. Otherwise, just use the part. 177 | if uripart.startswith("_"): 178 | part = (str(kwargs.pop(uripart, uripart))) 179 | else: 180 | part = uripart 181 | uriparts.append(part) 182 | uri = '/'.join(uriparts) 183 | 184 | # If an id kwarg is present and there is no id to fill in in 185 | # the list of uriparts, assume the id goes at the end. 186 | id = kwargs.pop('id', None) 187 | if id: 188 | uri += "/%s" % (id) 189 | 190 | return uri 191 | 192 | 193 | class TwitterCall(object): 194 | TWITTER_UNAVAILABLE_WAIT = 30 # delay after HTTP codes 502, 503 or 504 195 | 196 | def __init__( 197 | self, auth, format, domain, callable_cls, uri="", 198 | uriparts=None, secure=True, timeout=None, gzip=False, retry=False, verify_context=True): 199 | self.auth = auth 200 | self.format = format 201 | self.domain = domain 202 | self.callable_cls = callable_cls 203 | self.uri = uri 204 | self.uriparts = uriparts 205 | self.secure = secure 206 | self.timeout = timeout 207 | self.gzip = gzip 208 | self.retry = retry 209 | self.verify_context = verify_context 210 | 211 | def __getattr__(self, k): 212 | 213 | # NOTE: we test this to avoid doing dangerous things when actually 214 | # attempting to get magic methods this object does not have (such as 215 | # __getstate__, for instance, which is important to pickling). 216 | # NOTE: when this code is run, the desired magic method cannot exist 217 | # on this object because we are using __getattr__ and not 218 | # __getattribute__, hence if it existed, it would be accessed normally. 219 | if k.startswith('__'): 220 | raise AttributeError 221 | 222 | def extend_call(arg): 223 | return self.callable_cls( 224 | auth=self.auth, format=self.format, domain=self.domain, 225 | callable_cls=self.callable_cls, timeout=self.timeout, 226 | secure=self.secure, gzip=self.gzip, retry=self.retry, 227 | uriparts=self.uriparts + (arg,), verify_context=self.verify_context) 228 | 229 | if k == "_": 230 | return extend_call 231 | else: 232 | return extend_call(k) 233 | 234 | def __call__(self, **kwargs): 235 | kwargs = dict(kwargs) 236 | uri = build_uri(self.uriparts, kwargs) 237 | 238 | # Shortcut call arguments for special json arguments case 239 | if "media/metadata/create" in uri: 240 | media_id = kwargs.pop('media_id', None) 241 | alt_text = kwargs.pop('alt_text', kwargs.pop('text', None)) 242 | if media_id and alt_text: 243 | jsondata = { 244 | "media_id": media_id, 245 | "alt_text": {"text": alt_text} 246 | } 247 | return self.__call__(_json=jsondata, **kwargs) 248 | 249 | method = kwargs.pop('_method', None) or method_for_uri(uri) 250 | domain = self.domain 251 | 252 | # If an _id kwarg is present, this is treated as id as a CGI 253 | # param. 254 | _id = kwargs.pop('_id', None) 255 | if _id: 256 | kwargs['id'] = _id 257 | 258 | # If an _timeout is specified in kwargs, use it 259 | _timeout = kwargs.pop('_timeout', None) 260 | 261 | secure_str = '' 262 | if self.secure: 263 | secure_str = 's' 264 | dot = "" 265 | if self.format: 266 | dot = "." 267 | url_base = "http%s://%s/%s%s%s" % ( 268 | secure_str, domain, uri, dot, self.format) 269 | 270 | # Check if argument tells whether img is already base64 encoded 271 | b64_convert = not kwargs.pop("_base64", False) 272 | if b64_convert: 273 | import base64 274 | 275 | # Catch media arguments to handle oauth query differently for multipart 276 | media = None 277 | if 'media' in kwargs: 278 | mediafield = 'media' 279 | media = kwargs.pop('media') 280 | media_raw = True 281 | elif 'media[]' in kwargs: 282 | mediafield = 'media[]' 283 | media = kwargs.pop('media[]') 284 | if b64_convert: 285 | media = base64.b64encode(media) 286 | media_raw = False 287 | 288 | # Catch media arguments that are not accepted through multipart 289 | # and are not yet base64 encoded 290 | if b64_convert: 291 | for arg in ['banner', 'image']: 292 | if arg in kwargs: 293 | kwargs[arg] = base64.b64encode(kwargs[arg]) 294 | 295 | headers = {'Accept-Encoding': 'gzip'} if self.gzip else dict() 296 | body = None 297 | arg_data = None 298 | 299 | # Catch _json special argument to handle special endpoints which 300 | # require args as a json string within the request's body 301 | # for instance media/metadata/create on upload.twitter.com 302 | # https://dev.twitter.com/rest/reference/post/media/metadata/create 303 | jsondata = kwargs.pop('_json', None) 304 | if jsondata: 305 | body = actually_bytes(json.dumps(jsondata)) 306 | headers['Content-Type'] = 'application/json; charset=UTF-8' 307 | 308 | if self.auth: 309 | headers.update(self.auth.generate_headers()) 310 | # Use urlencoded oauth args with no params when sending media 311 | # via multipart and send it directly via uri even for post 312 | url_args = {} if media or jsondata else kwargs 313 | if method == "PUT" and _id: 314 | # the two PUT method APIs both require uri id parameter 315 | url_args['id'] = _id 316 | arg_data = self.auth.encode_params( 317 | url_base, method, url_args) 318 | if method == 'GET' or media or jsondata: 319 | url_base += '?' + arg_data 320 | else: 321 | body = arg_data.encode('utf-8') 322 | 323 | # Handle query as multipart when sending media 324 | if media: 325 | BOUNDARY = b"###Python-Twitter###" 326 | bod = [] 327 | bod.append(b'--' + BOUNDARY) 328 | bod.append( 329 | b'Content-Disposition: form-data; name="' 330 | + actually_bytes(mediafield) 331 | + b'"') 332 | bod.append(b'Content-Type: application/octet-stream') 333 | if not media_raw: 334 | bod.append(b'Content-Transfer-Encoding: base64') 335 | bod.append(b'') 336 | bod.append(actually_bytes(media)) 337 | for k, v in kwargs.items(): 338 | k = actually_bytes(k) 339 | v = actually_bytes(v) 340 | bod.append(b'--' + BOUNDARY) 341 | bod.append(b'Content-Disposition: form-data; name="' + k + b'"') 342 | bod.append(b'Content-Type: text/plain;charset=utf-8') 343 | bod.append(b'') 344 | bod.append(v) 345 | bod.append(b'--' + BOUNDARY + b'--') 346 | bod.append(b'') 347 | bod.append(b'') 348 | body = b'\r\n'.join(bod) 349 | # print(body.decode('utf-8', errors='ignore')) 350 | headers['Content-Type'] = \ 351 | b'multipart/form-data; boundary=' + BOUNDARY 352 | 353 | if not PY_3_OR_HIGHER: 354 | url_base = url_base.encode("utf-8") 355 | for k in headers: 356 | headers[actually_bytes(k)] = actually_bytes(headers.pop(k)) 357 | 358 | req = urllib_request.Request(url_base, data=body, headers=headers) 359 | # Horrible hack, the python2/urllib2 version of request doesn't 360 | # take a method parameter, but we can overwrite the class 361 | # get_method() function to a lambda function that always returns 362 | # the method we want... 363 | # https://stackoverflow.com/questions/111945/is-there-any-way-to-do-http-put-in-python/111988#111988 364 | if not PY_3_OR_HIGHER: 365 | method = method.encode('utf-8') 366 | req.get_method = lambda: method 367 | 368 | if self.retry: 369 | return self._handle_response_with_retry(req, uri, arg_data, _timeout) 370 | else: 371 | return self._handle_response(req, uri, arg_data, _timeout) 372 | 373 | def _handle_response(self, req, uri, arg_data, _timeout=None): 374 | kwargs = {} 375 | if _timeout: 376 | kwargs['timeout'] = _timeout 377 | try: 378 | context = None 379 | if _HAVE_SSL: 380 | if not self.verify_context: 381 | context = ssl._create_unverified_context() 382 | else: 383 | context = ssl.create_default_context() 384 | context.load_verify_locations(cafile=certifi.where()) 385 | kwargs['context'] = context 386 | handle = urllib_request.urlopen(req, **kwargs) 387 | if handle.headers['Content-Type'] in ['image/jpeg', 'image/png']: 388 | return handle 389 | try: 390 | data = handle.read() 391 | except http_client.IncompleteRead as e: 392 | # Even if we don't get all the bytes we should have there 393 | # may be a complete response in e.partial 394 | data = e.partial 395 | if handle.info().get('Content-Encoding') == 'gzip': 396 | # Handle gzip decompression 397 | buf = StringIO(data) 398 | f = gzip.GzipFile(fileobj=buf) 399 | data = f.read() 400 | if len(data) == 0: 401 | return wrap_response({}, handle.headers) 402 | elif "json" == self.format: 403 | try: 404 | res = json.loads(data.decode('utf8')) 405 | except JSONDecodeError as e: 406 | # it seems like the data received was incomplete 407 | # and we should catch it to allow retries 408 | raise TwitterError("Incomplete JSON data collected for %s (%s): %s)" % (uri, arg_data, e)) 409 | return wrap_response(res, handle.headers) 410 | else: 411 | return wrap_response( 412 | data.decode('utf8'), handle.headers) 413 | except urllib_error.HTTPError as e: 414 | if (e.code == 304): 415 | return [] 416 | else: 417 | raise TwitterHTTPError(e, uri, self.format, arg_data) 418 | 419 | def _handle_response_with_retry(self, req, uri, arg_data, _timeout=None): 420 | delay = 1 421 | retry = self.retry 422 | while retry: 423 | try: 424 | return self._handle_response(req, uri, arg_data, _timeout) 425 | except TwitterHTTPError as e: 426 | if e.e.code == 429: 427 | # API rate limit reached 428 | reset = int(e.e.headers.get('X-Rate-Limit-Reset', time() + 30)) 429 | delay = int(reset - time() + 2) # add some extra margin 430 | if delay <= 0: 431 | delay = self.TWITTER_UNAVAILABLE_WAIT 432 | print("API rate limit reached; waiting for %ds..." % delay, file=sys.stderr) 433 | elif e.e.code in (502, 503, 504): 434 | delay = self.TWITTER_UNAVAILABLE_WAIT 435 | print("Service unavailable; waiting for %ds..." % delay, file=sys.stderr) 436 | else: 437 | raise 438 | if isinstance(retry, int) and not isinstance(retry, bool): 439 | if retry <= 0: 440 | raise 441 | retry -= 1 442 | sleep(delay) 443 | except TwitterError as e: 444 | if isinstance(retry, int) and not isinstance(retry, bool): 445 | if retry <= 0: 446 | raise 447 | retry -= 1 448 | print("There was a problem dialoguing with the API; waiting for %ds..." % delay, file=sys.stderr) 449 | sleep(delay) 450 | delay *= 2 451 | 452 | 453 | class Twitter(TwitterCall): 454 | """ 455 | The minimalist yet fully featured Twitter API class. 456 | 457 | Get RESTful data by accessing members of this class. The result 458 | is decoded python objects (lists and dicts). 459 | 460 | The Twitter API is documented at: 461 | 462 | https://dev.twitter.com/overview/documentation 463 | 464 | The list of most accessible functions is listed at: 465 | 466 | https://dev.twitter.com/rest/public 467 | 468 | 469 | Examples:: 470 | 471 | from twitter import * 472 | 473 | t = Twitter( 474 | auth=OAuth(token, token_secret, consumer_key, consumer_secret)) 475 | 476 | # Get your "home" timeline 477 | t.statuses.home_timeline() 478 | 479 | # Get a particular friend's timeline 480 | t.statuses.user_timeline(screen_name="billybob") 481 | 482 | # to pass in GET/POST parameters, such as `count` 483 | t.statuses.home_timeline(count=5) 484 | 485 | # to pass in the GET/POST parameter `id` you need to use `_id` 486 | t.statuses.oembed(_id=1234567890) 487 | 488 | # Update your status 489 | t.statuses.update( 490 | status="Using @sixohsix's sweet Python Twitter Tools.") 491 | 492 | # Send a direct message 493 | t.direct_messages.new( 494 | user="billybob", 495 | text="I think yer swell!") 496 | 497 | # Get the members of tamtar's list "Things That Are Rad" 498 | t.lists.members(owner_screen_name="tamtar", slug="things-that-are-rad") 499 | 500 | # An *optional* `_timeout` parameter can also be used for API 501 | # calls which take much more time than normal or twitter stops 502 | # responding for some reason: 503 | t.users.lookup( 504 | screen_name=','.join(A_LIST_OF_100_SCREEN_NAMES), \ 505 | _timeout=1) 506 | 507 | # Overriding Method: GET/POST 508 | # you should not need to use this method as this library properly 509 | # detects whether GET or POST should be used, Nevertheless 510 | # to force a particular method, use `_method` 511 | t.statuses.oembed(_id=1234567890, _method='GET') 512 | 513 | # Send images along with your tweets: 514 | # - first just read images from the web or from files the regular way: 515 | with open("example.png", "rb") as imagefile: 516 | imagedata = imagefile.read() 517 | # - then upload medias one by one on Twitter's dedicated server 518 | # and collect each one's id: 519 | t_upload = Twitter(domain='upload.twitter.com', 520 | auth=OAuth(token, token_secret, consumer_key, consumer_secret)) 521 | id_img1 = t_upload.media.upload(media=imagedata)["media_id_string"] 522 | id_img2 = t_upload.media.upload(media=imagedata)["media_id_string"] 523 | 524 | # - finally send your tweet with the list of media ids: 525 | t.statuses.update(status="PTT ★", media_ids=",".join([id_img1, id_img2])) 526 | 527 | # Or send a tweet with an image (or set a logo/banner similarly) 528 | # using the old deprecated method that will probably disappear some day 529 | params = {"media[]": imagedata, "status": "PTT ★"} 530 | # Or for an image encoded as base64: 531 | params = {"media[]": base64_image, "status": "PTT ★", "_base64": True} 532 | t.statuses.update_with_media(**params) 533 | 534 | # Attach text metadata to medias sent, using the upload.twitter.com route 535 | # using the _json workaround to send json arguments as POST body 536 | # (warning: to be done before attaching the media to a tweet) 537 | t_upload.media.metadata.create(_json={ 538 | "media_id": id_img1, 539 | "alt_text": { "text": "metadata generated via PTT!" } 540 | }) 541 | # or with the shortcut arguments ("alt_text" and "text" work): 542 | t_upload.media.metadata.create(media_id=id_img1, text="metadata generated via PTT!") 543 | 544 | Searching Twitter:: 545 | 546 | # Search for the latest tweets about #pycon 547 | t.search.tweets(q="#pycon") 548 | 549 | 550 | Using the data returned 551 | ----------------------- 552 | 553 | Twitter API calls return decoded JSON. This is converted into 554 | a bunch of Python lists, dicts, ints, and strings. For example:: 555 | 556 | x = twitter.statuses.home_timeline() 557 | 558 | # The first 'tweet' in the timeline 559 | x[0] 560 | 561 | # The screen name of the user who wrote the first 'tweet' 562 | x[0]['user']['screen_name'] 563 | 564 | 565 | Getting raw XML data 566 | -------------------- 567 | 568 | If you prefer to get your Twitter data in XML format, pass 569 | format="xml" to the Twitter object when you instantiate it:: 570 | 571 | twitter = Twitter(format="xml") 572 | 573 | The output will not be parsed in any way. It will be a raw string 574 | of XML. 575 | 576 | """ 577 | 578 | def __init__( 579 | self, format="json", 580 | domain="api.twitter.com", secure=True, auth=None, 581 | api_version=_DEFAULT, retry=False, verify_context=True): 582 | """ 583 | Create a new twitter API connector. 584 | 585 | Pass an `auth` parameter to use the credentials of a specific 586 | user. Generally you'll want to pass an `OAuth` 587 | instance:: 588 | 589 | twitter = Twitter(auth=OAuth( 590 | token, token_secret, consumer_key, consumer_secret)) 591 | 592 | 593 | `domain` lets you change the domain you are connecting. By 594 | default it's `api.twitter.com`. 595 | 596 | If `secure` is False you will connect with HTTP instead of 597 | HTTPS. 598 | 599 | `api_version` is used to set the base uri. By default it's 600 | '1.1'. 601 | 602 | If `retry` is True, API rate limits will automatically be 603 | handled by waiting until the next reset, as indicated by 604 | the X-Rate-Limit-Reset HTTP header. If retry is an integer, 605 | it defines the number of retries attempted. 606 | """ 607 | if not auth: 608 | auth = NoAuth() 609 | 610 | if (format not in ("json", "xml", "")): 611 | raise ValueError("Unknown data format '%s'" % (format)) 612 | 613 | if api_version is _DEFAULT: 614 | api_version = '1.1' 615 | 616 | uriparts = () 617 | if api_version: 618 | uriparts += (str(api_version),) 619 | 620 | TwitterCall.__init__( 621 | self, auth=auth, format=format, domain=domain, 622 | callable_cls=TwitterCall, 623 | secure=secure, uriparts=uriparts, retry=retry, 624 | verify_context=verify_context) 625 | 626 | 627 | __all__ = ["Twitter", "TwitterError", "TwitterHTTPError", "TwitterResponse"] 628 | -------------------------------------------------------------------------------- /twitter/archiver.py: -------------------------------------------------------------------------------- 1 | """USAGE 2 | twitter-archiver [options] <-|user> [ ...] 3 | 4 | DESCRIPTION 5 | Archive tweets of users, sorted by date from oldest to newest, in 6 | the following format: <> 7 | Date format is: YYYY-MM-DD HH:MM:SS TZ. Tweet is used to 8 | resume archiving on next run. Archive file name is the user name. 9 | Provide "-" instead of users to read users from standard input. 10 | 11 | OPTIONS 12 | -o --oauth authenticate to Twitter using OAuth (default: no) 13 | -s --save-dir directory to save archives (default: current dir) 14 | -a --api-rate see current API rate limit status 15 | -t --timeline archive own timeline into given file name (requires 16 | OAuth, max 800 statuses) 17 | -m --mentions archive own mentions instead of timeline into 18 | given file name (requires OAuth, max 800 statuses) 19 | -v --favorites archive user's favorites instead of timeline 20 | -f --follow-redirects follow redirects of urls 21 | -r --redirect-sites follow redirects for this comma separated list of hosts 22 | -d --dms archive own direct messages (both received and 23 | sent) into given file name. 24 | -i --isoformat store dates in ISO format (specifically RFC 3339) 25 | 26 | AUTHENTICATION 27 | Authenticate to Twitter using OAuth to archive tweets of private profiles 28 | and have higher API rate limits. OAuth authentication tokens are stored 29 | in ~/.twitter-archiver_oauth. 30 | """ 31 | 32 | from __future__ import print_function 33 | 34 | import functools 35 | import os 36 | import sys 37 | import time as _time 38 | from datetime import datetime, time 39 | from getopt import gnu_getopt as getopt 40 | from getopt import GetoptError 41 | 42 | from .api import Twitter, TwitterError 43 | from .auth import NoAuth 44 | from .follow import lookup 45 | from .oauth import OAuth, read_token_file 46 | from .oauth_dance import oauth_dance 47 | from .timezones import utc as UTC 48 | from .timezones import Local 49 | from .util import Fail, err, expand_line, parse_host_list 50 | 51 | try: 52 | import urllib.request as urllib2 53 | import http.client as httplib 54 | except ImportError: 55 | import urllib2 56 | import httplib 57 | 58 | 59 | # T-Archiver (Twitter-Archiver) application registered by @stalkr_ 60 | CONSUMER_KEY='d8hIyfzs7ievqeeZLjZrqQ' 61 | CONSUMER_SECRET='AnZmK0rnvaX7BoJ75l6XlilnbyMv7FoiDXWVmPD8' 62 | 63 | 64 | def parse_args(args, options): 65 | """Parse arguments from command-line to set options.""" 66 | long_opts = ['help', 'oauth', 'save-dir=', 'api-rate', 'timeline=', 'mentions=', 'favorites', 'follow-redirects',"redirect-sites=", 'dms=', 'isoformat'] 67 | short_opts = "hos:at:m:vfr:d:i" 68 | opts, extra_args = getopt(args, short_opts, long_opts) 69 | 70 | for opt, arg in opts: 71 | if opt in ('-h', '--help'): 72 | print(__doc__) 73 | raise SystemExit(0) 74 | elif opt in ('-o', '--oauth'): 75 | options['oauth'] = True 76 | elif opt in ('-s', '--save-dir'): 77 | options['save-dir'] = arg 78 | elif opt in ('-a', '--api-rate'): 79 | options['api-rate' ] = True 80 | elif opt in ('-t', '--timeline'): 81 | options['timeline'] = arg 82 | elif opt in ('-m', '--mentions'): 83 | options['mentions'] = arg 84 | elif opt in ('-v', '--favorites'): 85 | options['favorites'] = True 86 | elif opt in ('-f', '--follow-redirects'): 87 | options['follow-redirects'] = True 88 | elif opt in ('-r', '--redirect-sites'): 89 | options['redirect-sites'] = arg 90 | elif opt in ('-d', '--dms'): 91 | options['dms'] = arg 92 | elif opt in ('-i', '--isoformat'): 93 | options['isoformat'] = True 94 | 95 | options['extra_args'] = extra_args 96 | 97 | def load_tweets(filename): 98 | """Load tweets from file into dict, see save_tweets().""" 99 | try: 100 | archive = open(filename,"r") 101 | except IOError: # no archive (yet) 102 | return {} 103 | 104 | tweets = {} 105 | for line in archive.readlines(): 106 | try: 107 | tid, text = line.strip().split(" ", 1) 108 | tweets[int(tid)] = text.decode("utf-8") 109 | except Exception as e: 110 | err("loading tweet %s failed due to %s" % (line, unicode(e))) 111 | 112 | archive.close() 113 | return tweets 114 | 115 | def save_tweets(filename, tweets): 116 | """Save tweets from dict to file. 117 | 118 | Save tweets from dict to UTF-8 encoded file, one per line: 119 | 120 | Tweet text is: 121 | <> [RT @: ] 122 | 123 | Args: 124 | filename: A string representing the file name to save tweets to. 125 | tweets: A dict mapping tweet-ids (int) to tweet text (str). 126 | """ 127 | if len(tweets) == 0: 128 | return 129 | 130 | try: 131 | archive = open(filename,"w") 132 | except IOError as e: 133 | err("Cannot save tweets: %s" % str(e)) 134 | return 135 | 136 | for k in sorted(tweets.keys()): 137 | try: 138 | archive.write("%i %s\n" % (k, tweets[k].encode('utf-8'))) 139 | except Exception as ex: 140 | err("archiving tweet %s failed due to %s" % (k, unicode(ex))) 141 | 142 | archive.close() 143 | 144 | def format_date(utc, isoformat=False): 145 | """Parse Twitter's UTC date into UTC or local time.""" 146 | u = datetime.strptime(utc.replace('+0000','UTC'), '%a %b %d %H:%M:%S %Z %Y') 147 | # This is the least painful way I could find to create a non-naive 148 | # datetime including a UTC timezone. Alternative suggestions 149 | # welcome. 150 | unew = datetime.combine(u.date(), time(u.time().hour, 151 | u.time().minute, u.time().second, tzinfo=UTC)) 152 | 153 | # Convert to localtime 154 | unew = unew.astimezone(Local) 155 | 156 | if isoformat: 157 | return unew.isoformat() 158 | else: 159 | return unew.strftime('%Y-%m-%d %H:%M:%S %Z') 160 | 161 | def expand_format_text(hosts, text): 162 | """Following redirects in links.""" 163 | return direct_format_text(expand_line(text, hosts)) 164 | 165 | def direct_format_text(text): 166 | """Transform special chars in text to have only one line.""" 167 | return text.replace('\n','\\n').replace('\r','\\r') 168 | 169 | def statuses_resolve_uids(twitter, tl): 170 | """Resolve user ids to screen names from statuses.""" 171 | # get all user ids that needs a lookup (no screen_name key) 172 | user_ids = [] 173 | for t in tl: 174 | rt = t.get('retweeted_status') 175 | if rt and not rt['user'].get('screen_name'): 176 | user_ids.append(rt['user']['id']) 177 | if not t['user'].get('screen_name'): 178 | user_ids.append(t['user']['id']) 179 | 180 | # resolve all of them at once 181 | names = lookup(twitter, list(set(user_ids))) 182 | 183 | # build new statuses with resolved uids 184 | new_tl = [] 185 | for t in tl: 186 | rt = t.get('retweeted_status') 187 | if rt and not rt['user'].get('screen_name'): 188 | name = names[rt['user']['id']] 189 | t['retweeted_status']['user']['screen_name'] = name 190 | if not t['user'].get('screen_name'): 191 | name = names[t['user']['id']] 192 | t['user']['screen_name'] = name 193 | new_tl.append(t) 194 | 195 | return new_tl 196 | 197 | 198 | def statuses_portion(twitter, screen_name, max_id=None, mentions=False, 199 | favorites=False, received_dms=None, isoformat=False): 200 | """Get a portion of the statuses of a screen name.""" 201 | kwargs = dict(count=200, include_rts=1, screen_name=screen_name, 202 | tweet_mode='extended') 203 | if max_id: 204 | kwargs['max_id'] = max_id 205 | 206 | tweets = {} 207 | if mentions: 208 | tl = twitter.statuses.mentions_timeline(**kwargs) 209 | elif favorites: 210 | tl = twitter.favorites.list(**kwargs) 211 | elif received_dms is not None: 212 | if received_dms: 213 | tl = twitter.direct_messages(**kwargs) 214 | else: # sent DMs 215 | tl = twitter.direct_messages.sent(**kwargs) 216 | else: # timeline 217 | if screen_name: 218 | tl = twitter.statuses.user_timeline(**kwargs) 219 | else: # self 220 | tl = twitter.statuses.home_timeline(**kwargs) 221 | 222 | # some tweets do not provide screen name but user id, resolve those 223 | # this isn't a valid operation for DMs, so special-case them 224 | if received_dms is None: 225 | newtl = statuses_resolve_uids(twitter, tl) 226 | else: 227 | newtl = tl 228 | for t in newtl: 229 | text = t['full_text'] 230 | rt = t.get('retweeted_status') 231 | if rt: 232 | text = "RT @%s: %s" % (rt['user']['screen_name'], rt['full_text']) 233 | # DMs don't include mentions by default, so in order to show who 234 | # the recipient was, we synthesise a mention. If we're not 235 | # operating on DMs, behave as normal 236 | if received_dms is None: 237 | tweets[t['id']] = "%s <%s> %s" % ( 238 | format_date(t['created_at'], isoformat=isoformat), 239 | t['user']['screen_name'], 240 | format_text(text)) 241 | else: 242 | tweets[t['id']] = "%s <%s> @%s %s" % ( 243 | format_date(t['created_at'], isoformat=isoformat), 244 | t['sender_screen_name'], 245 | t['recipient']['screen_name'], 246 | format_text(text)) 247 | return tweets 248 | 249 | 250 | def statuses(twitter, screen_name, tweets, mentions=False, favorites=False, received_dms=None, isoformat=False): 251 | """Get all the statuses for a screen name.""" 252 | max_id = None 253 | fail = Fail() 254 | # get portions of statuses, incrementing max id until no new tweets appear 255 | while True: 256 | try: 257 | portion = statuses_portion(twitter, screen_name, max_id, mentions, favorites, received_dms, isoformat) 258 | except TwitterError as e: 259 | if e.e.code == 401: 260 | err("Fail: %i Unauthorized (tweets of that user are protected)" 261 | % e.e.code) 262 | break 263 | elif e.e.code == 429: 264 | err("Fail: %i API rate limit exceeded" % e.e.code) 265 | rls = twitter.application.rate_limit_status() 266 | reset = rls.rate_limit_reset 267 | reset = _time.asctime(_time.localtime(reset)) 268 | delay = int(rls.rate_limit_reset 269 | - _time.time()) + 5 # avoid race 270 | err("Interval limit of %i requests reached, next reset on %s: " 271 | "going to sleep for %i secs" % (rls.rate_limit_limit, 272 | reset, delay)) 273 | fail.wait(delay) 274 | continue 275 | elif e.e.code == 404: 276 | err("Fail: %i This profile does not exist" % e.e.code) 277 | break 278 | elif e.e.code == 502: 279 | err("Fail: %i Service currently unavailable, retrying..." 280 | % e.e.code) 281 | else: 282 | err("Fail: %s\nRetrying..." % str(e)[:500]) 283 | fail.wait(3) 284 | except urllib2.URLError as e: 285 | err("Fail: urllib2.URLError %s - Retrying..." % str(e)) 286 | fail.wait(3) 287 | except httplib.error as e: 288 | err("Fail: httplib.error %s - Retrying..." % str(e)) 289 | fail.wait(3) 290 | except KeyError as e: 291 | err("Fail: KeyError %s - Retrying..." % str(e)) 292 | fail.wait(3) 293 | else: 294 | new = -len(tweets) 295 | tweets.update(portion) 296 | new += len(tweets) 297 | err("Browsing %s statuses, new tweets: %i" 298 | % (screen_name if screen_name else "home", new)) 299 | if new < 190: 300 | break 301 | max_id = min(portion.keys())-1 # browse backwards 302 | fail = Fail() 303 | 304 | def rate_limit_status(twitter): 305 | """Print current Twitter API rate limit status.""" 306 | rls = twitter.application.rate_limit_status() 307 | print("Remaining API requests: %i/%i (interval limit)" 308 | % (rls.rate_limit_remaining, rls.rate_limit_limit)) 309 | print("Next reset in %is (%s)" 310 | % (int(rls.rate_limit_reset - _time.time()), 311 | _time.asctime(_time.localtime(rls.rate_limit_reset)))) 312 | 313 | def main(args=sys.argv[1:]): 314 | options = { 315 | 'oauth': False, 316 | 'save-dir': ".", 317 | 'api-rate': False, 318 | 'timeline': "", 319 | 'mentions': "", 320 | 'dms': "", 321 | 'favorites': False, 322 | 'follow-redirects': False, 323 | 'redirect-sites': None, 324 | 'isoformat': False, 325 | } 326 | try: 327 | parse_args(args, options) 328 | except GetoptError as e: 329 | err("I can't do that, %s." % e) 330 | raise SystemExit(1) 331 | 332 | # exit if no user given 333 | # except if asking for API rate, or archive of timeline or mentions 334 | if not options['extra_args'] and not (options['api-rate'] or 335 | options['timeline'] or 336 | options['mentions'] or 337 | options['dms']): 338 | print(__doc__) 339 | return 340 | 341 | # authenticate using OAuth, asking for token if necessary 342 | if options['oauth']: 343 | oauth_filename = (os.environ.get('HOME', 344 | os.environ.get('USERPROFILE', '')) 345 | + os.sep 346 | + '.twitter-archiver_oauth') 347 | 348 | if not os.path.exists(oauth_filename): 349 | oauth_dance("Twitter-Archiver", CONSUMER_KEY, CONSUMER_SECRET, 350 | oauth_filename) 351 | oauth_token, oauth_token_secret = read_token_file(oauth_filename) 352 | auth = OAuth(oauth_token, oauth_token_secret, CONSUMER_KEY, 353 | CONSUMER_SECRET) 354 | else: 355 | auth = NoAuth() 356 | 357 | twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com') 358 | 359 | if options['api-rate']: 360 | rate_limit_status(twitter) 361 | return 362 | 363 | global format_text 364 | if options['follow-redirects'] or options['redirect-sites'] : 365 | if options['redirect-sites']: 366 | hosts = parse_host_list(options['redirect-sites']) 367 | else: 368 | hosts = None 369 | format_text = functools.partial(expand_format_text, hosts) 370 | else: 371 | format_text = direct_format_text 372 | 373 | # save own timeline or mentions (the user used in OAuth) 374 | if options['timeline'] or options['mentions']: 375 | if isinstance(auth, NoAuth): 376 | err("You must be authenticated to save timeline or mentions.") 377 | raise SystemExit(1) 378 | 379 | if options['timeline']: 380 | filename = options['save-dir'] + os.sep + options['timeline'] 381 | print("* Archiving own timeline in %s" % filename) 382 | elif options['mentions']: 383 | filename = options['save-dir'] + os.sep + options['mentions'] 384 | print("* Archiving own mentions in %s" % filename) 385 | 386 | tweets = {} 387 | try: 388 | tweets = load_tweets(filename) 389 | except Exception as e: 390 | err("Error when loading saved tweets: %s - continuing without" 391 | % str(e)) 392 | 393 | try: 394 | statuses(twitter, "", tweets, options['mentions'], options['favorites'], isoformat=options['isoformat']) 395 | except KeyboardInterrupt: 396 | err() 397 | err("Interrupted") 398 | raise SystemExit(1) 399 | 400 | save_tweets(filename, tweets) 401 | if options['timeline']: 402 | print("Total tweets in own timeline: %i" % len(tweets)) 403 | elif options['mentions']: 404 | print("Total mentions: %i" % len(tweets)) 405 | 406 | if options['dms']: 407 | if isinstance(auth, NoAuth): 408 | err("You must be authenticated to save DMs.") 409 | raise SystemExit(1) 410 | 411 | filename = options['save-dir'] + os.sep + options['dms'] 412 | print("* Archiving own DMs in %s" % filename) 413 | 414 | dms = {} 415 | try: 416 | dms = load_tweets(filename) 417 | except Exception as e: 418 | err("Error when loading saved DMs: %s - continuing without" 419 | % str(e)) 420 | 421 | try: 422 | statuses(twitter, "", dms, received_dms=True, isoformat=options['isoformat']) 423 | statuses(twitter, "", dms, received_dms=False, isoformat=options['isoformat']) 424 | except KeyboardInterrupt: 425 | err() 426 | err("Interrupted") 427 | raise SystemExit(1) 428 | 429 | save_tweets(filename, dms) 430 | print("Total DMs sent and received: %i" % len(dms)) 431 | 432 | 433 | # read users from command-line or stdin 434 | users = options['extra_args'] 435 | if len(users) == 1 and users[0] == "-": 436 | users = [line.strip() for line in sys.stdin.readlines()] 437 | 438 | # save tweets for every user 439 | total, total_new = 0, 0 440 | for user in users: 441 | filename = options['save-dir'] + os.sep + user 442 | if options['favorites']: 443 | filename = filename + "-favorites" 444 | print("* Archiving %s tweets in %s" % (user, filename)) 445 | 446 | tweets = {} 447 | try: 448 | tweets = load_tweets(filename) 449 | except Exception as e: 450 | err("Error when loading saved tweets: %s - continuing without" 451 | % str(e)) 452 | 453 | new = 0 454 | before = len(tweets) 455 | try: 456 | statuses(twitter, user, tweets, options['mentions'], options['favorites'], isoformat=options['isoformat']) 457 | except KeyboardInterrupt: 458 | err() 459 | err("Interrupted") 460 | raise SystemExit(1) 461 | 462 | save_tweets(filename, tweets) 463 | total += len(tweets) 464 | new = len(tweets) - before 465 | total_new += new 466 | print("Total tweets for %s: %i (%i new)" % (user, len(tweets), new)) 467 | 468 | print("Total: %i tweets (%i new) for %i users" 469 | % (total, total_new, len(users))) 470 | -------------------------------------------------------------------------------- /twitter/auth.py: -------------------------------------------------------------------------------- 1 | try: 2 | import urllib.parse as urllib_parse 3 | from base64 import encodebytes 4 | except ImportError: 5 | import urllib as urllib_parse 6 | from base64 import encodestring as encodebytes 7 | 8 | 9 | class Auth(object): 10 | """ 11 | ABC for Authenticator objects. 12 | """ 13 | 14 | def encode_params(self, base_url, method, params): 15 | """Encodes parameters for a request suitable for including in a URL 16 | or POST body. This method may also add new params to the request 17 | if required by the authentication scheme in use.""" 18 | raise NotImplementedError() 19 | 20 | def generate_headers(self): 21 | """Generates headers which should be added to the request if required 22 | by the authentication scheme in use.""" 23 | raise NotImplementedError() 24 | 25 | 26 | class UserPassAuth(Auth): 27 | """ 28 | Basic auth authentication using email/username and 29 | password. Deprecated. 30 | """ 31 | def __init__(self, username, password): 32 | self.username = username 33 | self.password = password 34 | 35 | def encode_params(self, base_url, method, params): 36 | # We could consider automatically converting unicode to utf8 strings 37 | # before encoding... 38 | return urllib_parse.urlencode(params) 39 | 40 | def generate_headers(self): 41 | return {b"Authorization": b"Basic " + encodebytes( 42 | ("%s:%s" %(self.username, self.password)) 43 | .encode('utf8')).strip(b'\n') 44 | } 45 | 46 | 47 | class NoAuth(Auth): 48 | """ 49 | No authentication authenticator. 50 | """ 51 | def __init__(self): 52 | pass 53 | 54 | def encode_params(self, base_url, method, params): 55 | return urllib_parse.urlencode(params) 56 | 57 | def generate_headers(self): 58 | return {} 59 | 60 | 61 | class MissingCredentialsError(Exception): 62 | pass 63 | -------------------------------------------------------------------------------- /twitter/cmdline.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | USAGE: 4 | 5 | twitter [action] [options] 6 | 7 | 8 | ACTIONS: 9 | authorize authorize the command-line tool to interact with Twitter 10 | follow follow a user 11 | friends get latest tweets from your friends (default action) 12 | user get latest tweets from a specific user 13 | help print this help text that you are currently reading 14 | leave stop following a user 15 | list get list of a user's lists; give a list name to get 16 | tweets from that list 17 | mylist get list of your lists; give a list name to get tweets 18 | from that list 19 | pyprompt start a Python prompt for interacting with the twitter 20 | object directly 21 | replies get latest replies to you 22 | search search twitter (Beware: octothorpe, escape it) 23 | set set your twitter status 24 | shell login to the twitter shell 25 | rate get your current rate limit status (remaining API reqs) 26 | repl begin a Read-Eval-Print-Loop with a configured twitter 27 | object 28 | 29 | OPTIONS: 30 | 31 | -r --refresh run this command forever, polling every once 32 | in a while (default: every 5 minutes) 33 | -R --refresh-rate set the refresh rate (in seconds) 34 | -f --format specify the output format for status updates 35 | -c --config read username and password from given config 36 | file (default ~/.twitter) 37 | -l --length specify number of status updates shown 38 | (default: 20, max: 200) 39 | -t --timestamp show time before status lines 40 | -d --datestamp show date before status lines 41 | --no-ssl use less-secure HTTP instead of HTTPS 42 | --oauth filename to read/store oauth credentials to 43 | 44 | FORMATS for the --format option 45 | 46 | default one line per status 47 | verbose multiple lines per status, more verbose status info 48 | json raw json data from the api on each line 49 | urls nothing but URLs 50 | ansi ansi colour (rainbow mode) 51 | 52 | 53 | CONFIG FILES 54 | 55 | The config file should be placed in your home directory and be named .twitter. 56 | It must contain a [twitter] header, and all the desired options you wish to 57 | set, like so: 58 | 59 | [twitter] 60 | format: 61 | prompt: '> 62 | 63 | OAuth authentication tokens are stored in the file .twitter_oauth in your 64 | home directory. 65 | """ 66 | 67 | from __future__ import print_function 68 | 69 | try: 70 | input = __builtins__.raw_input 71 | except (AttributeError, KeyError): 72 | pass 73 | 74 | 75 | CONSUMER_KEY = 'uS6hO2sV6tDKIOeVjhnFnQ' 76 | CONSUMER_SECRET = 'MEYTOS97VvlHX7K1rwHPEqVpTSqZ71HtvoK4sVuYk' 77 | 78 | import code 79 | from getopt import gnu_getopt as getopt, GetoptError 80 | import json 81 | import locale 82 | import os.path 83 | import re 84 | import sys 85 | import time 86 | 87 | try: 88 | from ConfigParser import SafeConfigParser 89 | except ImportError: 90 | from configparser import ConfigParser as SafeConfigParser 91 | import datetime 92 | try: 93 | from urllib.parse import quote 94 | except ImportError: 95 | from urllib2 import quote 96 | try: 97 | import HTMLParser 98 | except ImportError: 99 | import html.parser as HTMLParser 100 | 101 | from .api import Twitter, TwitterError 102 | from .oauth import OAuth, read_token_file 103 | from .oauth_dance import oauth_dance 104 | from . import ansi 105 | from .util import smrt_input, printNicely, align_text 106 | 107 | OPTIONS = { 108 | 'action': 'friends', 109 | 'refresh': False, 110 | 'refresh_rate': 600, 111 | 'format': 'default', 112 | 'prompt': '[cyan]twitter[R]> ', 113 | 'config_filename': os.environ.get('HOME', 114 | os.environ.get('USERPROFILE', '')) 115 | + os.sep + '.twitter', 116 | 'oauth_filename': os.environ.get('HOME', 117 | os.environ.get('USERPROFILE', '')) 118 | + os.sep + '.twitter_oauth', 119 | 'length': 20, 120 | 'timestamp': False, 121 | 'datestamp': False, 122 | 'extra_args': [], 123 | 'secure': True, 124 | 'invert_split': False, 125 | 'force-ansi': False, 126 | } 127 | 128 | try: 129 | import html 130 | unescape = html.unescape 131 | except ImportError: 132 | unescape = HTMLParser.HTMLParser().unescape 133 | 134 | hashtagRe = re.compile(r'(?P#\S+)') 135 | profileRe = re.compile(r'(?P\@\S+)') 136 | ansiFormatter = ansi.AnsiCmd(False) 137 | 138 | 139 | def parse_args(args, options): 140 | long_opts = ['help', 'format=', 'refresh', 'oauth=', 141 | 'refresh-rate=', 'config=', 'length=', 'timestamp', 142 | 'datestamp', 'no-ssl', 'force-ansi'] 143 | short_opts = "e:p:f:h?rR:c:l:td" 144 | opts, extra_args = getopt(args, short_opts, long_opts) 145 | if extra_args and hasattr(extra_args[0], 'decode'): 146 | extra_args = [arg.decode(locale.getpreferredencoding()) 147 | for arg in extra_args] 148 | 149 | for opt, arg in opts: 150 | if opt in ('-f', '--format'): 151 | options['format'] = arg 152 | elif opt in ('-r', '--refresh'): 153 | options['refresh'] = True 154 | elif opt in ('-R', '--refresh-rate'): 155 | options['refresh_rate'] = int(arg) 156 | elif opt in ('-l', '--length'): 157 | options["length"] = int(arg) 158 | elif opt in ('-t', '--timestamp'): 159 | options["timestamp"] = True 160 | elif opt in ('-d', '--datestamp'): 161 | options["datestamp"] = True 162 | elif opt in ('-?', '-h', '--help'): 163 | options['action'] = 'help' 164 | elif opt in ('-c', '--config'): 165 | options['config_filename'] = arg 166 | elif opt == '--no-ssl': 167 | options['secure'] = False 168 | elif opt == '--oauth': 169 | options['oauth_filename'] = arg 170 | elif opt == '--force-ansi': 171 | options['force-ansi'] = True 172 | 173 | if extra_args and not ('action' in options and options['action'] == 'help'): 174 | options['action'] = extra_args[0] 175 | options['extra_args'] = extra_args[1:] 176 | 177 | 178 | def get_time_string(status, options, format="%a %b %d %H:%M:%S +0000 %Y"): 179 | timestamp = options["timestamp"] 180 | datestamp = options["datestamp"] 181 | t = time.strptime(status['created_at'], format) 182 | i_hate_timezones = time.timezone 183 | if time.daylight: 184 | i_hate_timezones = time.altzone 185 | dt = datetime.datetime(*t[:-3]) - datetime.timedelta( 186 | seconds=i_hate_timezones) 187 | t = dt.timetuple() 188 | if timestamp and datestamp: 189 | return time.strftime("%Y-%m-%d %H:%M:%S ", t) 190 | elif timestamp: 191 | return time.strftime("%H:%M:%S ", t) 192 | elif datestamp: 193 | return time.strftime("%Y-%m-%d ", t) 194 | return "" 195 | 196 | 197 | def reRepl(m): 198 | ansiTypes = { 199 | 'clear': ansiFormatter.cmdReset(), 200 | 'hashtag': ansiFormatter.cmdBold(), 201 | 'profile': ansiFormatter.cmdUnderline(), 202 | } 203 | 204 | s = None 205 | try: 206 | mkey = m.lastgroup 207 | if m.group(mkey): 208 | s = '%s%s%s' % (ansiTypes[mkey], m.group(mkey), ansiTypes['clear']) 209 | except IndexError: 210 | pass 211 | return s 212 | 213 | 214 | def replaceInStatus(status): 215 | txt = unescape(status) 216 | txt = re.sub(hashtagRe, reRepl, txt) 217 | txt = re.sub(profileRe, reRepl, txt) 218 | return txt 219 | 220 | 221 | def correctRTStatus(status): 222 | if 'retweeted_status' in status: 223 | return ("RT @" + status['retweeted_status']['user']['screen_name'] 224 | + " " + status['retweeted_status']['text']) 225 | else: 226 | return status['text'] 227 | 228 | 229 | class StatusFormatter(object): 230 | def __call__(self, status, options): 231 | return ("%s@%s %s" % ( 232 | get_time_string(status, options), 233 | status['user']['screen_name'], 234 | unescape(correctRTStatus(status)))) 235 | 236 | 237 | class AnsiStatusFormatter(object): 238 | def __init__(self): 239 | self._colourMap = ansi.ColourMap() 240 | 241 | def __call__(self, status, options): 242 | colour = self._colourMap.colourFor(status['user']['screen_name']) 243 | return ("%s%s% 16s%s %s " % ( 244 | get_time_string(status, options), 245 | ansiFormatter.cmdColour(colour), status['user']['screen_name'], 246 | ansiFormatter.cmdReset(), 247 | align_text(replaceInStatus(correctRTStatus(status))))) 248 | 249 | 250 | class VerboseStatusFormatter(object): 251 | def __call__(self, status, options): 252 | return ("-- %s (%s) on %s\n%s\n" % ( 253 | status['user']['screen_name'], 254 | status['user']['location'], 255 | status['created_at'], 256 | unescape(correctRTStatus(status)))) 257 | 258 | 259 | class JSONStatusFormatter(object): 260 | def __call__(self, status, options): 261 | status['text'] = unescape(status['text']) 262 | return json.dumps(status) 263 | 264 | 265 | class URLStatusFormatter(object): 266 | urlmatch = re.compile(r'https?://\S+') 267 | 268 | def __call__(self, status, options): 269 | urls = self.urlmatch.findall(correctRTStatus(status)) 270 | return '\n'.join(urls) if urls else "" 271 | 272 | 273 | class ListsFormatter(object): 274 | def __call__(self, list): 275 | if list['description']: 276 | list_str = "%-30s (%s)" % (list['name'], list['description']) 277 | else: 278 | list_str = "%-30s" % (list['name']) 279 | return "%s\n" % list_str 280 | 281 | 282 | class ListsVerboseFormatter(object): 283 | def __call__(self, list): 284 | list_str = "%-30s\n description: %s\n members: %s\n mode:%s\n" % ( 285 | list['name'], list['description'], 286 | list['member_count'], list['mode']) 287 | return list_str 288 | 289 | 290 | class AnsiListsFormatter(object): 291 | def __init__(self): 292 | self._colourMap = ansi.ColourMap() 293 | 294 | def __call__(self, list): 295 | colour = self._colourMap.colourFor(list['name']) 296 | return ("%s%-15s%s %s" % ( 297 | ansiFormatter.cmdColour(colour), list['name'], 298 | ansiFormatter.cmdReset(), list['description'])) 299 | 300 | 301 | class AdminFormatter(object): 302 | def __call__(self, action, user): 303 | user_str = "%s (%s)" % (user['screen_name'], user['name']) 304 | if action == "follow": 305 | return "You are now following %s.\n" % (user_str) 306 | else: 307 | return "You are no longer following %s.\n" % (user_str) 308 | 309 | 310 | class VerboseAdminFormatter(object): 311 | def __call__(self, action, user): 312 | return("-- %s: %s (%s): %s" % ( 313 | "Following" if action == "follow" else "Leaving", 314 | user['screen_name'], 315 | user['name'], 316 | user['url'])) 317 | 318 | 319 | class SearchFormatter(object): 320 | def __call__(self, result, options): 321 | payload = result['text'].replace('\n', ' ') 322 | return("%s%s %s" % ( 323 | get_time_string(result, options, "%a %b %d %H:%M:%S +0000 %Y"), 324 | result['user']['screen_name'], payload)) 325 | 326 | 327 | class VerboseSearchFormatter(SearchFormatter): 328 | pass # Default to the regular one 329 | 330 | 331 | class URLSearchFormatter(object): 332 | urlmatch = re.compile(r'https?://\S+') 333 | 334 | def __call__(self, result, options): 335 | urls = self.urlmatch.findall(result['text']) 336 | return '\n'.join(urls) if urls else "" 337 | 338 | 339 | class AnsiSearchFormatter(object): 340 | def __init__(self): 341 | self._colourMap = ansi.ColourMap() 342 | 343 | def __call__(self, result, options): 344 | colour = self._colourMap.colourFor(result['from_user']) 345 | return ("%s%s%s%s %s" % ( 346 | get_time_string(result, options, "%a, %d %b %Y %H:%M:%S +0000"), 347 | ansiFormatter.cmdColour(colour), result['from_user'], 348 | ansiFormatter.cmdReset(), result['text'])) 349 | 350 | _term_encoding = None 351 | 352 | 353 | def get_term_encoding(): 354 | global _term_encoding 355 | if not _term_encoding: 356 | lang = os.getenv('LANG', 'unknown.UTF-8').split('.') 357 | if lang[1:]: 358 | _term_encoding = lang[1] 359 | else: 360 | _term_encoding = 'UTF-8' 361 | return _term_encoding 362 | 363 | formatters = {} 364 | status_formatters = { 365 | 'default': StatusFormatter, 366 | 'verbose': VerboseStatusFormatter, 367 | 'json': JSONStatusFormatter, 368 | 'urls': URLStatusFormatter, 369 | 'ansi': AnsiStatusFormatter 370 | } 371 | formatters['status'] = status_formatters 372 | 373 | admin_formatters = { 374 | 'default': AdminFormatter, 375 | 'verbose': VerboseAdminFormatter, 376 | 'urls': AdminFormatter, 377 | 'ansi': AdminFormatter 378 | } 379 | formatters['admin'] = admin_formatters 380 | 381 | search_formatters = { 382 | 'default': SearchFormatter, 383 | 'verbose': VerboseSearchFormatter, 384 | 'urls': URLSearchFormatter, 385 | 'ansi': AnsiSearchFormatter 386 | } 387 | formatters['search'] = search_formatters 388 | 389 | lists_formatters = { 390 | 'default': ListsFormatter, 391 | 'verbose': ListsVerboseFormatter, 392 | 'urls': None, 393 | 'ansi': AnsiListsFormatter 394 | } 395 | formatters['lists'] = lists_formatters 396 | 397 | 398 | def get_formatter(action_type, options): 399 | formatters_dict = formatters.get(action_type) 400 | if not formatters_dict: 401 | raise TwitterError( 402 | "There was an error finding a class of formatters for your type (%s)" 403 | % (action_type)) 404 | f = formatters_dict.get(options['format']) 405 | if not f: 406 | raise TwitterError( 407 | "Unknown formatter '%s' for status actions" % (options['format'])) 408 | return f() 409 | 410 | 411 | class Action(object): 412 | 413 | def ask(self, subject='perform this action', careful=False): 414 | ''' 415 | Requests from the user using `raw_input` if `subject` should be 416 | performed. When `careful`, the default answer is NO, otherwise YES. 417 | Returns the user answer in the form `True` or `False`. 418 | ''' 419 | sample = '(y/N)' 420 | if not careful: 421 | sample = '(Y/n)' 422 | 423 | prompt = 'You really want to %s %s? ' % (subject, sample) 424 | try: 425 | answer = input(prompt).lower() 426 | if careful: 427 | return answer in ('yes', 'y') 428 | else: 429 | return answer not in ('no', 'n') 430 | except EOFError: 431 | print(file=sys.stderr) # Put Newline since Enter was never pressed 432 | # TODO: 433 | # Figure out why on OS X the raw_input keeps raising 434 | # EOFError and is never able to reset and get more input 435 | # Hint: Look at how IPython implements their console 436 | default = True 437 | if careful: 438 | default = False 439 | return default 440 | 441 | def __call__(self, twitter, options): 442 | action = actions.get(options['action'], NoSuchAction)() 443 | try: 444 | doAction = lambda: action(twitter, options) 445 | if options['refresh'] and isinstance(action, StatusAction): 446 | while True: 447 | doAction() 448 | sys.stdout.flush() 449 | time.sleep(options['refresh_rate']) 450 | else: 451 | doAction() 452 | except KeyboardInterrupt: 453 | print('\n[Keyboard Interrupt]', file=sys.stderr) 454 | pass 455 | 456 | 457 | class NoSuchActionError(Exception): 458 | pass 459 | 460 | 461 | class NoSuchAction(Action): 462 | def __call__(self, twitter, options): 463 | raise NoSuchActionError("No such action: %s" % (options['action'])) 464 | 465 | 466 | class StatusAction(Action): 467 | def __call__(self, twitter, options): 468 | statuses = self.getStatuses(twitter, options) 469 | sf = get_formatter('status', options) 470 | for status in statuses: 471 | statusStr = sf(status, options) 472 | if statusStr.strip(): 473 | printNicely(statusStr) 474 | 475 | 476 | class SearchAction(Action): 477 | def __call__(self, twitter, options): 478 | # don't need the "search.twitter.com" domain & keep original uriparts 479 | if not (options['extra_args'] and options['extra_args'][0]): 480 | raise TwitterError("You need to specify something to search enclosed in single/double quotes") 481 | query_string = options['extra_args'][0] 482 | 483 | results = twitter.search.tweets( 484 | q=query_string, count=options['length'])['statuses'] 485 | f = get_formatter('search', options) 486 | for result in results: 487 | resultStr = f(result, options) 488 | if resultStr.strip(): 489 | printNicely(resultStr) 490 | 491 | 492 | class AdminAction(Action): 493 | def __call__(self, twitter, options): 494 | if not (options['extra_args'] and options['extra_args'][0]): 495 | raise TwitterError("You need to specify a user (screen name)") 496 | af = get_formatter('admin', options) 497 | try: 498 | user = self.getUser(twitter, options['extra_args'][0]) 499 | except TwitterError as e: 500 | print("There was a problem following or leaving the specified user.") 501 | print("You may be trying to follow a user you are already following;") 502 | print("Leaving a user you are not currently following;") 503 | print("Or the user may not exist.") 504 | print("Sorry.") 505 | print() 506 | print(e) 507 | else: 508 | printNicely(af(options['action'], user)) 509 | 510 | 511 | class ListsAction(StatusAction): 512 | def getStatuses(self, twitter, options): 513 | if not options['extra_args']: 514 | raise TwitterError("Please provide a user to query for lists") 515 | 516 | screen_name = options['extra_args'][0] 517 | 518 | if not options['extra_args'][1:]: 519 | lists = twitter.lists.list(screen_name=screen_name) 520 | if not lists: 521 | printNicely("This user has no lists.") 522 | for lst in lists: 523 | lf = get_formatter('lists', options) 524 | printNicely(lf(lst)) 525 | return [] 526 | else: 527 | return list(reversed(twitter.lists.statuses( 528 | count=options['length'], 529 | owner_screen_name=screen_name, 530 | slug=options['extra_args'][1]))) 531 | 532 | 533 | class MyListsAction(ListsAction): 534 | def getStatuses(self, twitter, options): 535 | screen_name = twitter.account.verify_credentials()['screen_name'] 536 | options['extra_args'].insert(0, screen_name) 537 | return ListsAction.getStatuses(self, twitter, options) 538 | 539 | 540 | class FriendsAction(StatusAction): 541 | def getStatuses(self, twitter, options): 542 | return list(reversed( 543 | twitter.statuses.home_timeline(count=options["length"]))) 544 | 545 | 546 | class UserAction(StatusAction): 547 | def getStatuses(self, twitter, options): 548 | if not options['extra_args']: 549 | raise TwitterError("You need to specify a user (screen name)") 550 | 551 | screen_name = options['extra_args'][0] 552 | 553 | return list(reversed( 554 | twitter.statuses.user_timeline(screen_name=screen_name, 555 | count=options["length"]))) 556 | 557 | 558 | class RepliesAction(StatusAction): 559 | def getStatuses(self, twitter, options): 560 | return list(reversed( 561 | twitter.statuses.mentions_timeline(count=options["length"]))) 562 | 563 | 564 | class FollowAction(AdminAction): 565 | def getUser(self, twitter, user): 566 | return twitter.friendships.create(screen_name=user) 567 | 568 | 569 | class LeaveAction(AdminAction): 570 | def getUser(self, twitter, user): 571 | return twitter.friendships.destroy(screen_name=user) 572 | 573 | 574 | class SetStatusAction(Action): 575 | def __call__(self, twitter, options): 576 | statusTxt = (" ".join(options['extra_args']) 577 | if options['extra_args'] 578 | else str(input("message: "))) 579 | statusTxt = statusTxt.replace('\\n', '\n') 580 | replies = [] 581 | ptr = re.compile(r"@[\w_]+") 582 | while statusTxt: 583 | s = ptr.match(statusTxt) 584 | if s and s.start() == 0: 585 | replies.append(statusTxt[s.start():s.end()]) 586 | statusTxt = statusTxt[s.end() + 1:] 587 | else: 588 | break 589 | replies = " ".join(replies) 590 | if len(replies) >= 280: 591 | # just go back 592 | statusTxt = replies 593 | replies = "" 594 | 595 | splitted = [] 596 | while statusTxt: 597 | limit = 280 - len(replies) 598 | if len(statusTxt) > limit: 599 | end = str.rfind(statusTxt, ' ', 0, limit) 600 | else: 601 | end = limit 602 | splitted.append(" ".join((replies, statusTxt[:end]))) 603 | statusTxt = statusTxt[end:] 604 | 605 | if options['invert_split']: 606 | splitted.reverse() 607 | for status in splitted: 608 | twitter.statuses.update(status=status) 609 | 610 | 611 | class TwitterShell(Action): 612 | 613 | def render_prompt(self, prompt): 614 | '''Parses the `prompt` string and returns the rendered version''' 615 | prompt = prompt.strip("'").replace("\\'", "'") 616 | for colour in ansi.COLOURS_NAMED: 617 | if '[%s]' % (colour) in prompt: 618 | prompt = prompt.replace( 619 | '[%s]' % (colour), ansiFormatter.cmdColourNamed(colour)) 620 | prompt = prompt.replace('[R]', ansiFormatter.cmdReset()) 621 | return prompt 622 | 623 | def __call__(self, twitter, options): 624 | prompt = self.render_prompt(options.get('prompt', 'twitter> ')) 625 | while True: 626 | options['action'] = "" 627 | try: 628 | args = input(prompt).split() 629 | parse_args(args, options) 630 | if not options['action']: 631 | continue 632 | elif options['action'] == 'exit': 633 | raise SystemExit(0) 634 | elif options['action'] == 'shell': 635 | print('Sorry Xzibit does not work here!', file=sys.stderr) 636 | continue 637 | elif options['action'] == 'help': 638 | print('''\ntwitter> `action`\n 639 | The Shell accepts all the command line actions along with: 640 | 641 | exit Leave the twitter shell (^D may also be used) 642 | 643 | Full CMD Line help is appended below for your convenience.''', 644 | file=sys.stderr) 645 | Action()(twitter, options) 646 | options['action'] = '' 647 | except NoSuchActionError as e: 648 | print(e, file=sys.stderr) 649 | except KeyboardInterrupt: 650 | print('\n[Keyboard Interrupt]', file=sys.stderr) 651 | except EOFError: 652 | print(file=sys.stderr) 653 | leaving = self.ask(subject='Leave') 654 | if not leaving: 655 | print('Excellent!', file=sys.stderr) 656 | else: 657 | raise SystemExit(0) 658 | 659 | 660 | class PythonPromptAction(Action): 661 | def __call__(self, twitter, options): 662 | try: 663 | while True: 664 | smrt_input(globals(), locals()) 665 | except EOFError: 666 | pass 667 | 668 | 669 | class HelpAction(Action): 670 | def __call__(self, twitter, options): 671 | print(__doc__) 672 | 673 | 674 | class DoNothingAction(Action): 675 | def __call__(self, twitter, options): 676 | pass 677 | 678 | 679 | class RateLimitStatus(Action): 680 | def __call__(self, twitter, options): 681 | rate = twitter.application.rate_limit_status() 682 | resources = rate['resources'] 683 | for resource in resources: 684 | for method in resources[resource]: 685 | limit = resources[resource][method]['limit'] 686 | remaining = resources[resource][method]['remaining'] 687 | reset = resources[resource][method]['reset'] 688 | 689 | print("Remaining API requests for %s: %s / %s" % 690 | (method, remaining, limit)) 691 | print("Next reset in %ss (%s)\n" % (int(reset - time.time()), 692 | time.asctime(time.localtime(reset)))) 693 | 694 | 695 | class ReplAction(Action): 696 | def __call__(self, twitter, options): 697 | upload = Twitter( 698 | auth=twitter.auth, 699 | domain="upload.twitter.com") 700 | printNicely( 701 | "\nUse the 'twitter' object to interact with" 702 | " the Twitter REST API.\n" 703 | "Use twitter_upload to interact with " 704 | "upload.twitter.com\n\n") 705 | code.interact(local={ 706 | "twitter": twitter, 707 | "t": twitter, 708 | "twitter_upload": upload, 709 | "u": upload 710 | }) 711 | 712 | 713 | actions = { 714 | 'authorize' : DoNothingAction, 715 | 'follow' : FollowAction, 716 | 'friends' : FriendsAction, 717 | 'user' : UserAction, 718 | 'list' : ListsAction, 719 | 'mylist' : MyListsAction, 720 | 'help' : HelpAction, 721 | 'leave' : LeaveAction, 722 | 'pyprompt' : PythonPromptAction, 723 | 'replies' : RepliesAction, 724 | 'search' : SearchAction, 725 | 'set' : SetStatusAction, 726 | 'shell' : TwitterShell, 727 | 'rate' : RateLimitStatus, 728 | 'repl' : ReplAction, 729 | } 730 | 731 | 732 | def loadConfig(filename): 733 | options = dict(OPTIONS) 734 | if os.path.exists(filename): 735 | cp = SafeConfigParser() 736 | cp.read([filename]) 737 | for option in ('format', 'prompt'): 738 | if cp.has_option('twitter', option): 739 | options[option] = cp.get('twitter', option) 740 | # process booleans 741 | for option in ('invert_split',): 742 | if cp.has_option('twitter', option): 743 | options[option] = cp.getboolean('twitter', option) 744 | return options 745 | 746 | 747 | def main(args=sys.argv[1:]): 748 | arg_options = {} 749 | try: 750 | parse_args(args, arg_options) 751 | except GetoptError as e: 752 | print("I can't do that, %s." % (e), file=sys.stderr) 753 | print(file=sys.stderr) 754 | raise SystemExit(1) 755 | 756 | config_path = os.path.expanduser( 757 | arg_options.get('config_filename') or OPTIONS.get('config_filename')) 758 | config_options = loadConfig(config_path) 759 | 760 | # Apply the various options in order, the most important applied last. 761 | # Defaults first, then what's read from config file, then command-line 762 | # arguments. 763 | options = dict(OPTIONS) 764 | for d in config_options, arg_options: 765 | for k, v in list(d.items()): 766 | if v: 767 | options[k] = v 768 | 769 | if options['refresh'] and options['action'] not in ('friends', 'replies'): 770 | print("You can only refresh the friends or replies actions.", 771 | file=sys.stderr) 772 | print("Use 'twitter -h' for help.", file=sys.stderr) 773 | return 1 774 | 775 | oauth_filename = os.path.expanduser(options['oauth_filename']) 776 | 777 | if options['action'] == 'authorize' or not os.path.exists(oauth_filename): 778 | oauth_dance( 779 | "the Command-Line Tool", CONSUMER_KEY, CONSUMER_SECRET, 780 | options['oauth_filename']) 781 | 782 | global ansiFormatter 783 | ansiFormatter = ansi.AnsiCmd(options["force-ansi"]) 784 | 785 | oauth_token, oauth_token_secret = read_token_file(oauth_filename) 786 | 787 | twitter = Twitter( 788 | auth=OAuth( 789 | oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET), 790 | secure=options['secure'], 791 | api_version='1.1', 792 | domain='api.twitter.com') 793 | 794 | try: 795 | Action()(twitter, options) 796 | except NoSuchActionError as e: 797 | print(e, file=sys.stderr) 798 | raise SystemExit(1) 799 | except TwitterError as e: 800 | print(str(e), file=sys.stderr) 801 | print("Use 'twitter -h' for help.", file=sys.stderr) 802 | raise SystemExit(1) 803 | -------------------------------------------------------------------------------- /twitter/follow.py: -------------------------------------------------------------------------------- 1 | """USAGE 2 | twitter-follow [options] 3 | 4 | DESCRIPTION 5 | Display all following/followers of a user, one user per line. 6 | 7 | OPTIONS 8 | -o --oauth authenticate to Twitter using OAuth (default no) 9 | -r --followers display followers of the given user (default) 10 | -g --following display users the given user is following 11 | -a --api-rate see your current API rate limit status 12 | -i --ids prepend user id to each line. useful to tracking renames 13 | 14 | AUTHENTICATION 15 | Authenticate to Twitter using OAuth to see following/followers of private 16 | profiles and have higher API rate limits. OAuth authentication tokens 17 | are stored in the file .twitter-follow_oauth in your home directory. 18 | """ 19 | 20 | from __future__ import print_function 21 | 22 | import os, sys, time, calendar 23 | from getopt import gnu_getopt as getopt, GetoptError 24 | 25 | try: 26 | import urllib.request as urllib2 27 | import http.client as httplib 28 | except ImportError: 29 | import urllib2 30 | import httplib 31 | 32 | # T-Follow (Twitter-Follow) application registered by @stalkr_ 33 | CONSUMER_KEY='USRZQfvFFjB6UvZIN2Edww' 34 | CONSUMER_SECRET='AwGAaSzZa5r0TDL8RKCDtffnI9H9mooZUdOa95nw8' 35 | 36 | from .api import Twitter, TwitterError 37 | from .oauth import OAuth, read_token_file 38 | from .oauth_dance import oauth_dance 39 | from .auth import NoAuth 40 | from .util import Fail, err 41 | 42 | 43 | def parse_args(args, options): 44 | """Parse arguments from command-line to set options.""" 45 | long_opts = ['help', 'oauth', 'followers', 'following', 'api-rate', 'ids'] 46 | short_opts = "horgai" 47 | opts, extra_args = getopt(args, short_opts, long_opts) 48 | 49 | for opt, arg in opts: 50 | if opt in ('-h', '--help'): 51 | print(__doc__) 52 | raise SystemExit(1) 53 | elif opt in ('-o', '--oauth'): 54 | options['oauth'] = True 55 | elif opt in ('-r', '--followers'): 56 | options['followers'] = True 57 | elif opt in ('-g', '--following'): 58 | options['followers'] = False 59 | elif opt in ('-a', '--api-rate'): 60 | options['api-rate' ] = True 61 | elif opt in ('-i', '--ids'): 62 | options['show_id'] = True 63 | 64 | options['extra_args'] = extra_args 65 | 66 | def lookup_portion(twitter, user_ids): 67 | """Resolve a limited list of user ids to screen names.""" 68 | users = {} 69 | kwargs = dict(user_id=",".join(map(str, user_ids)), skip_status=1) 70 | for u in twitter.users.lookup(**kwargs): 71 | users[int(u['id'])] = u['screen_name'] 72 | return users 73 | 74 | def lookup(twitter, user_ids): 75 | """Resolve an entire list of user ids to screen names.""" 76 | users = {} 77 | api_limit = 100 78 | for i in range(0, len(user_ids), api_limit): 79 | fail = Fail() 80 | while True: 81 | try: 82 | portion = lookup_portion(twitter, user_ids[i:][:api_limit]) 83 | except TwitterError as e: 84 | if e.e.code == 429: 85 | err("Fail: %i API rate limit exceeded" % e.e.code) 86 | rls = twitter.application.rate_limit_status() 87 | reset = rls.rate_limit_reset 88 | reset = time.asctime(time.localtime(reset)) 89 | delay = int(rls.rate_limit_reset 90 | - time.time()) + 5 # avoid race 91 | err("Interval limit of %i requests reached, next reset on " 92 | "%s: going to sleep for %i secs" 93 | % (rls.rate_limit_limit, reset, delay)) 94 | fail.wait(delay) 95 | continue 96 | elif e.e.code == 502: 97 | err("Fail: %i Service currently unavailable, retrying..." 98 | % e.e.code) 99 | else: 100 | err("Fail: %s\nRetrying..." % str(e)[:500]) 101 | fail.wait(3) 102 | except urllib2.URLError as e: 103 | err("Fail: urllib2.URLError %s - Retrying..." % str(e)) 104 | fail.wait(3) 105 | except httplib.error as e: 106 | err("Fail: httplib.error %s - Retrying..." % str(e)) 107 | fail.wait(3) 108 | except KeyError as e: 109 | err("Fail: KeyError %s - Retrying..." % str(e)) 110 | fail.wait(3) 111 | else: 112 | users.update(portion) 113 | err("Resolving user ids to screen names: %i/%i" 114 | % (len(users), len(user_ids))) 115 | break 116 | return users 117 | 118 | def follow_portion(twitter, screen_name, cursor=-1, followers=True): 119 | """Get a portion of followers/following for a user.""" 120 | kwargs = dict(screen_name=screen_name, cursor=cursor) 121 | if followers: 122 | t = twitter.followers.ids(**kwargs) 123 | else: # following 124 | t = twitter.friends.ids(**kwargs) 125 | return t['ids'], t['next_cursor'] 126 | 127 | def follow(twitter, screen_name, followers=True): 128 | """Get the entire list of followers/following for a user.""" 129 | user_ids = [] 130 | cursor = -1 131 | fail = Fail() 132 | while True: 133 | try: 134 | portion, cursor = follow_portion(twitter, screen_name, cursor, 135 | followers) 136 | except TwitterError as e: 137 | if e.e.code == 401: 138 | reason = ("follow%s of that user are protected" 139 | % ("ers" if followers else "ing")) 140 | err("Fail: %i Unauthorized (%s)" % (e.e.code, reason)) 141 | break 142 | elif e.e.code == 429: 143 | err("Fail: %i API rate limit exceeded" % e.e.code) 144 | rls = twitter.application.rate_limit_status() 145 | reset = rls.rate_limit_reset 146 | reset = time.asctime(time.localtime(reset)) 147 | delay = int(rls.rate_limit_reset 148 | - time.time()) + 5 # avoid race 149 | err("Interval limit of %i requests reached, next reset on %s: " 150 | "going to sleep for %i secs" % (rls.rate_limit_limit, 151 | reset, delay)) 152 | fail.wait(delay) 153 | continue 154 | elif e.e.code == 502: 155 | err("Fail: %i Service currently unavailable, retrying..." 156 | % e.e.code) 157 | else: 158 | err("Fail: %s\nRetrying..." % str(e)[:500]) 159 | fail.wait(3) 160 | except urllib2.URLError as e: 161 | err("Fail: urllib2.URLError %s - Retrying..." % str(e)) 162 | fail.wait(3) 163 | except httplib.error as e: 164 | err("Fail: httplib.error %s - Retrying..." % str(e)) 165 | fail.wait(3) 166 | except KeyError as e: 167 | err("Fail: KeyError %s - Retrying..." % str(e)) 168 | fail.wait(3) 169 | else: 170 | new = -len(user_ids) 171 | user_ids = list(set(user_ids + portion)) 172 | new += len(user_ids) 173 | what = "follow%s" % ("ers" if followers else "ing") 174 | err("Browsing %s %s, new: %i" % (screen_name, what, new)) 175 | if cursor == 0: 176 | break 177 | fail = Fail() 178 | return user_ids 179 | 180 | 181 | def rate_limit_status(twitter): 182 | """Print current Twitter API rate limit status.""" 183 | rls = twitter.application.rate_limit_status() 184 | print("Remaining API requests: %i/%i (interval limit)" 185 | % (rls.rate_limit_remaining, rls.rate_limit_limit)) 186 | print("Next reset in %is (%s)" 187 | % (int(rls.rate_limit_reset - time.time()), 188 | time.asctime(time.localtime(rls.rate_limit_reset)))) 189 | 190 | def main(args=sys.argv[1:]): 191 | options = { 192 | 'oauth': False, 193 | 'followers': True, 194 | 'api-rate': False, 195 | 'show_id': False 196 | } 197 | try: 198 | parse_args(args, options) 199 | except GetoptError as e: 200 | err("I can't do that, %s." % e) 201 | raise SystemExit(1) 202 | 203 | # exit if no user or given, except if asking for API rate 204 | if not options['extra_args'] and not options['api-rate']: 205 | print(__doc__) 206 | raise SystemExit(1) 207 | 208 | # authenticate using OAuth, asking for token if necessary 209 | if options['oauth']: 210 | oauth_filename = (os.getenv("HOME", "") + os.sep 211 | + ".twitter-follow_oauth") 212 | if not os.path.exists(oauth_filename): 213 | oauth_dance("Twitter-Follow", CONSUMER_KEY, CONSUMER_SECRET, 214 | oauth_filename) 215 | oauth_token, oauth_token_secret = read_token_file(oauth_filename) 216 | auth = OAuth(oauth_token, oauth_token_secret, CONSUMER_KEY, 217 | CONSUMER_SECRET) 218 | else: 219 | auth = NoAuth() 220 | 221 | twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com') 222 | 223 | if options['api-rate']: 224 | rate_limit_status(twitter) 225 | return 226 | 227 | # obtain list of followers (or following) for every given user 228 | for user in options['extra_args']: 229 | user_ids, users = [], {} 230 | try: 231 | user_ids = follow(twitter, user, options['followers']) 232 | users = lookup(twitter, user_ids) 233 | except KeyboardInterrupt as e: 234 | err() 235 | err("Interrupted.") 236 | raise SystemExit(1) 237 | 238 | for uid in user_ids: 239 | if options['show_id']: 240 | try: 241 | print(str(uid) + "\t" + users[uid].encode("utf-8")) 242 | except KeyError: 243 | pass 244 | 245 | else: 246 | try: 247 | print(users[uid].encode("utf-8")) 248 | except KeyError: 249 | pass 250 | 251 | # print total on stderr to separate from user list on stdout 252 | if options['followers']: 253 | err("Total followers for %s: %i" % (user, len(user_ids))) 254 | else: 255 | err("Total users %s is following: %i" % (user, len(user_ids))) 256 | -------------------------------------------------------------------------------- /twitter/ircbot.py: -------------------------------------------------------------------------------- 1 | """ 2 | twitterbot 3 | 4 | A twitter IRC bot. Twitterbot connected to an IRC server and idles in 5 | a channel, polling a twitter account and broadcasting all updates to 6 | friends. 7 | 8 | USAGE 9 | 10 | twitterbot [config_file] 11 | 12 | CONFIG_FILE 13 | 14 | The config file is an ini-style file that must contain the following: 15 | 16 | [irc] 17 | server: 18 | port: 19 | nick: 20 | channel: 21 | prefixes: 22 | 23 | [twitter] 24 | oauth_token_file: 25 | 26 | 27 | If no config file is given "twitterbot.ini" will be used by default. 28 | 29 | The channel argument can accept multiple channels separated by commas. 30 | 31 | The default token file is ~/.twitterbot_oauth. 32 | 33 | The default prefix type is 'cats'. You can also use 'none'. 34 | 35 | """ 36 | 37 | from __future__ import print_function 38 | 39 | BOT_VERSION = "TwitterBot 1.9.1 (http://mike.verdone.ca/twitter)" 40 | 41 | CONSUMER_KEY = "XryIxN3J2ACaJs50EizfLQ" 42 | CONSUMER_SECRET = "j7IuDCNjftVY8DBauRdqXs4jDl5Fgk1IJRag8iE" 43 | 44 | IRC_BOLD = chr(0x02) 45 | IRC_ITALIC = chr(0x16) 46 | IRC_UNDERLINE = chr(0x1f) 47 | IRC_REGULAR = chr(0x0f) 48 | 49 | import sys 50 | import time 51 | from datetime import datetime, timedelta 52 | from email.utils import parsedate 53 | try: 54 | from configparser import ConfigParser 55 | except ImportError: 56 | from ConfigParser import ConfigParser 57 | from heapq import heappop, heappush 58 | import traceback 59 | import os 60 | import os.path 61 | 62 | from .api import Twitter, TwitterError 63 | from .oauth import OAuth, read_token_file 64 | from .oauth_dance import oauth_dance 65 | from .util import htmlentitydecode 66 | 67 | PREFIXES = dict( 68 | cats=dict( 69 | new_tweet="=^_^= ", 70 | error="=O_o= ", 71 | inform="=o_o= " 72 | ), 73 | none=dict( 74 | new_tweet="" 75 | ), 76 | ) 77 | ACTIVE_PREFIXES=dict() 78 | 79 | def get_prefix(prefix_typ=None): 80 | return ACTIVE_PREFIXES.get(prefix_typ, ACTIVE_PREFIXES.get('new_tweet', '')) 81 | 82 | 83 | try: 84 | import irclib 85 | except ImportError: 86 | raise ImportError( 87 | "This module requires python irclib available from " 88 | + "https://github.com/sixohsix/python-irclib/zipball/python-irclib3-0.4.8") 89 | 90 | OAUTH_FILE = os.environ.get('HOME', os.environ.get('USERPROFILE', '')) + os.sep + '.twitterbot_oauth' 91 | 92 | def debug(msg): 93 | # uncomment this for debug text stuff 94 | # print(msg, file=sys.stdout) 95 | pass 96 | 97 | class SchedTask(object): 98 | def __init__(self, task, delta): 99 | self.task = task 100 | self.delta = delta 101 | self.next = time.time() 102 | 103 | def __repr__(self): 104 | return "" %( 105 | self.task.__name__, self.__next__, self.delta) 106 | 107 | def __lt__(self, other): 108 | return self.next < other.next 109 | 110 | def __call__(self): 111 | return self.task() 112 | 113 | class Scheduler(object): 114 | def __init__(self, tasks): 115 | self.task_heap = [] 116 | for task in tasks: 117 | heappush(self.task_heap, task) 118 | 119 | def next_task(self): 120 | now = time.time() 121 | task = heappop(self.task_heap) 122 | wait = task.next - now 123 | task.next = now + task.delta 124 | heappush(self.task_heap, task) 125 | if (wait > 0): 126 | time.sleep(wait) 127 | task() 128 | #debug("tasks: " + str(self.task_heap)) 129 | 130 | def run_forever(self): 131 | while True: 132 | self.next_task() 133 | 134 | 135 | class TwitterBot(object): 136 | def __init__(self, configFilename): 137 | self.configFilename = configFilename 138 | self.config = load_config(self.configFilename) 139 | 140 | global ACTIVE_PREFIXES 141 | ACTIVE_PREFIXES = PREFIXES[self.config.get('irc', 'prefixes')] 142 | 143 | oauth_file = self.config.get('twitter', 'oauth_token_file') 144 | if not os.path.exists(oauth_file): 145 | oauth_dance("IRC Bot", CONSUMER_KEY, CONSUMER_SECRET, oauth_file) 146 | oauth_token, oauth_secret = read_token_file(oauth_file) 147 | 148 | self.twitter = Twitter( 149 | auth=OAuth( 150 | oauth_token, oauth_secret, CONSUMER_KEY, CONSUMER_SECRET), 151 | domain='api.twitter.com') 152 | 153 | self.irc = irclib.IRC() 154 | self.irc.add_global_handler('privmsg', self.handle_privmsg) 155 | self.irc.add_global_handler('ctcp', self.handle_ctcp) 156 | self.irc.add_global_handler('welcome', self.handle_welcome) 157 | self.ircServer = self.irc.server() 158 | 159 | self.welcome_received = False 160 | self.sched = Scheduler( 161 | (SchedTask(self.process_events, 1), 162 | SchedTask(self.check_statuses, 120))) 163 | self.lastUpdate = (datetime.utcnow() - timedelta(minutes=10)).utctimetuple() 164 | 165 | def check_statuses(self): 166 | if not self.welcome_received: 167 | return 168 | debug("In check_statuses") 169 | try: 170 | updates = reversed(self.twitter.statuses.home_timeline()) 171 | except Exception as e: 172 | print("Exception while querying twitter:", file=sys.stderr) 173 | traceback.print_exc(file=sys.stderr) 174 | return 175 | 176 | nextLastUpdate = self.lastUpdate 177 | for update in updates: 178 | crt = parsedate(update['created_at']) 179 | if (crt > nextLastUpdate): 180 | text = (htmlentitydecode( 181 | update['text'].replace('\n', ' ')) 182 | .encode('utf8', 'replace')) 183 | 184 | # Skip updates beginning with @ 185 | # TODO This would be better if we only ignored messages 186 | # to people who are not on our following list. 187 | if not text.startswith(b"@"): 188 | msg = "%s %s%s%s %s" %( 189 | get_prefix(), 190 | IRC_BOLD, update['user']['screen_name'], 191 | IRC_BOLD, text.decode('utf8')) 192 | self.privmsg_channels(msg) 193 | 194 | nextLastUpdate = crt 195 | 196 | self.lastUpdate = nextLastUpdate 197 | 198 | def process_events(self): 199 | self.irc.process_once() 200 | 201 | def handle_privmsg(self, conn, evt): 202 | debug('got privmsg') 203 | args = evt.arguments()[0].split(' ') 204 | try: 205 | if (not args): 206 | return 207 | if (args[0] == 'follow' and args[1:]): 208 | self.follow(conn, evt, args[1]) 209 | elif (args[0] == 'unfollow' and args[1:]): 210 | self.unfollow(conn, evt, args[1]) 211 | else: 212 | conn.privmsg( 213 | evt.source().split('!')[0], 214 | "%sHi! I'm Twitterbot! you can (follow " 215 | ") to make me follow a user or " 216 | "(unfollow ) to make me stop." % 217 | get_prefix()) 218 | except Exception: 219 | traceback.print_exc(file=sys.stderr) 220 | 221 | def handle_ctcp(self, conn, evt): 222 | args = evt.arguments() 223 | source = evt.source().split('!')[0] 224 | if (args): 225 | if args[0] == 'VERSION': 226 | conn.ctcp_reply(source, "VERSION " + BOT_VERSION) 227 | elif args[0] == 'PING': 228 | conn.ctcp_reply(source, "PING") 229 | elif args[0] == 'CLIENTINFO': 230 | conn.ctcp_reply(source, "CLIENTINFO PING VERSION CLIENTINFO") 231 | 232 | def handle_welcome(self, conn, evt): 233 | """ 234 | Undernet and QuakeNet ignore all your commands until it receives 001. This 235 | handler defers joining until after it sees a magic line. 236 | """ 237 | self.welcome_received = True 238 | channels = self.config.get('irc', 'channel').split(',') 239 | for channel in channels: 240 | self.ircServer.join(channel) 241 | self.check_statuses() 242 | 243 | def privmsg_channels(self, msg): 244 | return_response=True 245 | channels=self.config.get('irc','channel').split(',') 246 | return self.ircServer.privmsg_many(channels, msg.encode('utf8')) 247 | 248 | def follow(self, conn, evt, name): 249 | userNick = evt.source().split('!')[0] 250 | friends = [x['name'] for x in self.twitter.statuses.friends()] 251 | debug("Current friends: %s" %(friends)) 252 | if (name in friends): 253 | conn.privmsg( 254 | userNick, 255 | "%sI'm already following %s." %(get_prefix('error'), name)) 256 | else: 257 | try: 258 | self.twitter.friendships.create(screen_name=name) 259 | except TwitterError: 260 | conn.privmsg( 261 | userNick, 262 | "%sI can't follow that user. Are you sure the name is correct?" %( 263 | get_prefix('error') 264 | )) 265 | return 266 | conn.privmsg( 267 | userNick, 268 | "%sOkay! I'm now following %s." %(get_prefix('followed'), name)) 269 | self.privmsg_channels( 270 | "%s%s has asked me to start following %s" %( 271 | get_prefix('inform'), userNick, name)) 272 | 273 | def unfollow(self, conn, evt, name): 274 | userNick = evt.source().split('!')[0] 275 | friends = [x['name'] for x in self.twitter.statuses.friends()] 276 | debug("Current friends: %s" %(friends)) 277 | if (name not in friends): 278 | conn.privmsg( 279 | userNick, 280 | "%sI'm not following %s." %(get_prefix('error'), name)) 281 | else: 282 | self.twitter.friendships.destroy(screen_name=name) 283 | conn.privmsg( 284 | userNick, 285 | "%sOkay! I've stopped following %s." %( 286 | get_prefix('stop_follow'), name)) 287 | self.privmsg_channels( 288 | "%s%s has asked me to stop following %s" %( 289 | get_prefix('inform'), userNick, name)) 290 | 291 | def _irc_connect(self): 292 | self.welcome_received = False 293 | self.ircServer.connect( 294 | self.config.get('irc', 'server'), 295 | self.config.getint('irc', 'port'), 296 | self.config.get('irc', 'nick')) 297 | 298 | def run(self): 299 | self._irc_connect() 300 | 301 | while True: 302 | try: 303 | self.sched.run_forever() 304 | except KeyboardInterrupt: 305 | break 306 | except TwitterError: 307 | # twitter.com is probably down because it 308 | # sucks. ignore the fault and keep going 309 | pass 310 | except irclib.ServerNotConnectedError: 311 | # Try and reconnect to IRC. 312 | self._irc_connect() 313 | 314 | 315 | def load_config(filename): 316 | # Note: Python ConfigParser module has the worst interface in the 317 | # world. Mega gross. 318 | cp = ConfigParser() 319 | cp.add_section('irc') 320 | cp.set('irc', 'port', '6667') 321 | cp.set('irc', 'nick', 'twitterbot') 322 | cp.set('irc', 'prefixes', 'cats') 323 | cp.add_section('twitter') 324 | cp.set('twitter', 'oauth_token_file', OAUTH_FILE) 325 | 326 | cp.read((filename,)) 327 | 328 | # attempt to read these properties-- they are required 329 | cp.get('twitter', 'oauth_token_file'), 330 | cp.get('irc', 'server') 331 | cp.getint('irc', 'port') 332 | cp.get('irc', 'nick') 333 | cp.get('irc', 'channel') 334 | 335 | return cp 336 | 337 | # So there was a joke here about the twitter business model 338 | # but I got rid of it. Not because I want this codebase to 339 | # be "professional" in any way, but because someone forked 340 | # this and deleted the comment because they couldn't take 341 | # a joke. Hi guy! 342 | # 343 | # Fact: The number one use of Google Code is to look for that 344 | # comment in the Linux kernel that goes "FUCK me gently with 345 | # a chainsaw." Pretty sure Linus himself wrote it. 346 | 347 | def main(): 348 | configFilename = "twitterbot.ini" 349 | if (sys.argv[1:]): 350 | configFilename = sys.argv[1] 351 | 352 | try: 353 | if not os.path.exists(configFilename): 354 | raise Exception() 355 | load_config(configFilename) 356 | except Exception as e: 357 | print("Error while loading ini file %s" %( 358 | configFilename), file=sys.stderr) 359 | print(e, file=sys.stderr) 360 | print(__doc__, file=sys.stderr) 361 | sys.exit(1) 362 | 363 | bot = TwitterBot(configFilename) 364 | return bot.run() 365 | -------------------------------------------------------------------------------- /twitter/logger.py: -------------------------------------------------------------------------------- 1 | """ 2 | twitter-log - Twitter Logger/Archiver 3 | 4 | USAGE: 5 | 6 | twitter-log [max_id] 7 | 8 | DESCRIPTION: 9 | 10 | Produce a complete archive in text form of a user's tweets. The 11 | archive format is: 12 | 13 | screen_name 14 | Date: 15 | [In-Reply-To: a_tweet_id] 16 | 17 | Tweet text possibly spanning multiple lines with 18 | each line indented by four spaces. 19 | 20 | 21 | Each tweet is separated by two blank lines. 22 | 23 | """ 24 | 25 | from __future__ import print_function 26 | 27 | import sys 28 | import os 29 | from time import sleep 30 | 31 | from .api import Twitter, TwitterError 32 | from .cmdline import CONSUMER_KEY, CONSUMER_SECRET 33 | from .auth import NoAuth 34 | from .oauth import OAuth, write_token_file, read_token_file 35 | from .oauth_dance import oauth_dance 36 | from .util import printNicely 37 | 38 | # Registered by @sixohsix 39 | CONSUMER_KEY = "OifqLIQIufeY9znQCkbvg" 40 | CONSUMER_SECRET = "IedFvi0JitR9yaYw9HwcCCEy4KYaLxf4p4rHRqGgX80" 41 | OAUTH_FILENAME = os.environ.get('HOME', os.environ.get('USERPROFILE', '')) + os.sep + '.twitter_log_oauth' 42 | 43 | def log_debug(msg): 44 | print(msg, file=sys.stderr) 45 | 46 | def get_tweets(twitter, screen_name, max_id=None): 47 | kwargs = dict(count=3200, screen_name=screen_name) 48 | if max_id: 49 | kwargs['max_id'] = max_id 50 | 51 | n_tweets = 0 52 | tweets = twitter.statuses.user_timeline(**kwargs) 53 | for tweet in tweets: 54 | if tweet['id'] == max_id: 55 | continue 56 | print("%s %s\nDate: %s" % (tweet['user']['screen_name'], 57 | tweet['id'], 58 | tweet['created_at'])) 59 | if tweet.get('in_reply_to_status_id'): 60 | print("In-Reply-To: %s" % tweet['in_reply_to_status_id']) 61 | print() 62 | for line in tweet['text'].splitlines(): 63 | printNicely(' ' + line + '\n') 64 | print() 65 | print() 66 | max_id = tweet['id'] 67 | n_tweets += 1 68 | return n_tweets, max_id 69 | 70 | def main(args=sys.argv[1:]): 71 | if not args: 72 | print(__doc__) 73 | return 1 74 | 75 | if not os.path.exists(OAUTH_FILENAME): 76 | oauth_dance( 77 | "the Python Twitter Logger", CONSUMER_KEY, CONSUMER_SECRET, 78 | OAUTH_FILENAME) 79 | 80 | oauth_token, oauth_token_secret = read_token_file(OAUTH_FILENAME) 81 | 82 | twitter = Twitter( 83 | auth=OAuth( 84 | oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET), 85 | domain='api.twitter.com') 86 | 87 | screen_name = args[0] 88 | 89 | if args[1:]: 90 | max_id = args[1] 91 | else: 92 | max_id = None 93 | 94 | n_tweets = 0 95 | while True: 96 | try: 97 | tweets_processed, max_id = get_tweets(twitter, screen_name, max_id) 98 | n_tweets += tweets_processed 99 | log_debug("Processed %i tweets (max_id %s)" %(n_tweets, max_id)) 100 | if tweets_processed == 0: 101 | log_debug("That's it, we got all the tweets we could. Done.") 102 | break 103 | except TwitterError as e: 104 | log_debug("Twitter bailed out. I'm going to sleep a bit then try again") 105 | sleep(3) 106 | 107 | return 0 108 | -------------------------------------------------------------------------------- /twitter/oauth.py: -------------------------------------------------------------------------------- 1 | """ 2 | Visit the Twitter developer page and create a new application: 3 | 4 | https://dev.twitter.com/apps/new 5 | 6 | This will get you a CONSUMER_KEY and CONSUMER_SECRET. 7 | 8 | When users run your application they have to authenticate your app 9 | with their Twitter account. A few HTTP calls to twitter are required 10 | to do this. Please see the twitter.oauth_dance module to see how this 11 | is done. If you are making a command-line app, you can use the 12 | oauth_dance() function directly. 13 | 14 | Performing the "oauth dance" gets you an ouath token and oauth secret 15 | that authenticate the user with Twitter. You should save these for 16 | later so that the user doesn't have to do the oauth dance again. 17 | 18 | read_token_file and write_token_file are utility methods to read and 19 | write OAuth token and secret key values. The values are stored as 20 | strings in the file. Not terribly exciting. 21 | 22 | Finally, you can use the OAuth authenticator to connect to Twitter. In 23 | code it all goes like this:: 24 | 25 | from twitter import * 26 | 27 | MY_TWITTER_CREDS = os.path.expanduser('~/.my_app_credentials') 28 | if not os.path.exists(MY_TWITTER_CREDS): 29 | oauth_dance("My App Name", CONSUMER_KEY, CONSUMER_SECRET, 30 | MY_TWITTER_CREDS) 31 | 32 | oauth_token, oauth_secret = read_token_file(MY_TWITTER_CREDS) 33 | 34 | twitter = Twitter(auth=OAuth( 35 | oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET)) 36 | 37 | # Now work with Twitter 38 | twitter.statuses.update(status='Hello, world!') 39 | 40 | """ 41 | 42 | from __future__ import print_function 43 | 44 | from random import getrandbits 45 | from time import time 46 | 47 | from .util import PY_3_OR_HIGHER 48 | 49 | try: 50 | import urllib.parse as urllib_parse 51 | from urllib.parse import urlencode 52 | except ImportError: 53 | import urllib2 as urllib_parse 54 | from urllib import urlencode 55 | 56 | import hashlib 57 | import hmac 58 | import base64 59 | 60 | from .auth import Auth, MissingCredentialsError 61 | 62 | 63 | def write_token_file(filename, oauth_token, oauth_token_secret): 64 | """ 65 | Write a token file to hold the oauth token and oauth token secret. 66 | """ 67 | oauth_file = open(filename, 'w') 68 | print(oauth_token, file=oauth_file) 69 | print(oauth_token_secret, file=oauth_file) 70 | oauth_file.close() 71 | 72 | def read_token_file(filename): 73 | """ 74 | Read a token file and return the oauth token and oauth token secret. 75 | """ 76 | f = open(filename) 77 | return f.readline().strip(), f.readline().strip() 78 | 79 | 80 | class OAuth(Auth): 81 | """ 82 | An OAuth authenticator. 83 | """ 84 | def __init__(self, token, token_secret, consumer_key, consumer_secret): 85 | """ 86 | Create the authenticator. If you are in the initial stages of 87 | the OAuth dance and don't yet have a token or token_secret, 88 | pass empty strings for these params. 89 | """ 90 | self.token = token 91 | self.token_secret = token_secret 92 | self.consumer_key = consumer_key 93 | self.consumer_secret = consumer_secret 94 | 95 | if token_secret is None or consumer_secret is None: 96 | raise MissingCredentialsError( 97 | 'You must supply strings for token_secret and consumer_secret, not None.') 98 | 99 | def encode_params(self, base_url, method, params): 100 | params = params.copy() 101 | 102 | if self.token: 103 | params['oauth_token'] = self.token 104 | 105 | params['oauth_consumer_key'] = self.consumer_key 106 | params['oauth_signature_method'] = 'HMAC-SHA1' 107 | params['oauth_version'] = '1.0' 108 | params['oauth_timestamp'] = str(int(time())) 109 | params['oauth_nonce'] = str(getrandbits(64)) 110 | 111 | enc_params = urlencode_noplus(sorted(params.items())) 112 | 113 | key = self.consumer_secret + "&" + urllib_parse.quote(self.token_secret, safe='~') 114 | 115 | message = '&'.join( 116 | urllib_parse.quote(i, safe='~') for i in [method.upper(), base_url, enc_params]) 117 | 118 | signature = (base64.b64encode(hmac.new( 119 | key.encode('ascii'), message.encode('ascii'), hashlib.sha1) 120 | .digest())) 121 | return enc_params + "&" + "oauth_signature=" + urllib_parse.quote(signature, safe='~') 122 | 123 | def generate_headers(self): 124 | return {} 125 | 126 | # apparently contrary to the HTTP RFCs, spaces in arguments must be encoded as 127 | # %20 rather than '+' when constructing an OAuth signature (and therefore 128 | # also in the request itself.) 129 | # So here is a specialized version which does exactly that. 130 | # In Python2, since there is no safe option for urlencode, we force it by hand 131 | def urlencode_noplus(query): 132 | if not PY_3_OR_HIGHER: 133 | new_query = [] 134 | TILDE = '____TILDE-PYTHON-TWITTER____' 135 | for k,v in query: 136 | if type(k) is unicode: k = k.encode('utf-8') 137 | k = str(k).replace("~", TILDE) 138 | if type(v) is unicode: v = v.encode('utf-8') 139 | v = str(v).replace("~", TILDE) 140 | new_query.append((k, v)) 141 | query = new_query 142 | return urlencode(query).replace(TILDE, "~").replace("+", "%20") 143 | 144 | return urlencode(query, safe='~').replace("+", "%20") 145 | -------------------------------------------------------------------------------- /twitter/oauth2.py: -------------------------------------------------------------------------------- 1 | """ 2 | Twitter only supports the application-only flow of OAuth2 for certain 3 | API endpoints. This OAuth2 authenticator only supports the application-only 4 | flow right now. 5 | 6 | To authenticate with OAuth2, visit the Twitter developer page and create a new 7 | application: 8 | 9 | https://dev.twitter.com/apps/new 10 | 11 | This will get you a CONSUMER_KEY and CONSUMER_SECRET. 12 | 13 | Exchange your CONSUMER_KEY and CONSUMER_SECRET for a bearer token using the 14 | oauth2_dance function. 15 | 16 | Finally, you can use the OAuth2 authenticator and your bearer token to connect 17 | to Twitter. In code it goes like this:: 18 | 19 | twitter = Twitter(auth=OAuth2(bearer_token=BEARER_TOKEN)) 20 | 21 | # Now work with Twitter 22 | twitter.search.tweets(q='keyword') 23 | 24 | """ 25 | 26 | from __future__ import print_function 27 | 28 | try: 29 | from urllib.parse import quote, urlencode 30 | except ImportError: 31 | from urllib import quote, urlencode 32 | 33 | from base64 import b64encode 34 | from .auth import Auth, MissingCredentialsError 35 | 36 | def write_bearer_token_file(filename, oauth2_bearer_token): 37 | """ 38 | Write a token file to hold the oauth2 bearer token. 39 | """ 40 | oauth_file = open(filename, 'w') 41 | print(oauth2_bearer_token, file=oauth_file) 42 | oauth_file.close() 43 | 44 | def read_bearer_token_file(filename): 45 | """ 46 | Read a token file and return the oauth2 bearer token. 47 | """ 48 | f = open(filename) 49 | bearer_token = f.readline().strip() 50 | f.close() 51 | return bearer_token 52 | 53 | class OAuth2(Auth): 54 | """ 55 | An OAuth2 application-only authenticator. 56 | """ 57 | def __init__(self, consumer_key=None, consumer_secret=None, 58 | bearer_token=None): 59 | """ 60 | Create an authenticator. You can supply consumer_key and 61 | consumer_secret if you are requesting a bearer_token. Otherwise 62 | you must supply the bearer_token. 63 | """ 64 | self.bearer_token = bearer_token 65 | self.consumer_key = consumer_key 66 | self.consumer_secret = consumer_secret 67 | 68 | if not (bearer_token or (consumer_key and consumer_secret)): 69 | raise MissingCredentialsError( 70 | 'You must supply either a bearer token, or both a ' 71 | 'consumer_key and a consumer_secret.') 72 | 73 | def encode_params(self, base_url, method, params): 74 | return urlencode(params) 75 | 76 | def generate_headers(self): 77 | if self.bearer_token: 78 | headers = { 79 | 'Authorization': 'Bearer {0}'.format( 80 | self.bearer_token).encode('utf8') 81 | } 82 | else: 83 | headers = { 84 | 'Content-Type': (b'application/x-www-form-urlencoded;' 85 | b'charset=UTF-8'), 86 | 'Authorization': 'Basic {0}'.format( 87 | b64encode('{0}:{1}'.format( 88 | quote(self.consumer_key), 89 | quote(self.consumer_secret)).encode('utf8') 90 | ).decode('utf8') 91 | ).encode('utf8') 92 | } 93 | return headers 94 | -------------------------------------------------------------------------------- /twitter/oauth_dance.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import webbrowser 4 | import time 5 | 6 | from .api import Twitter, json 7 | from .oauth import OAuth, write_token_file 8 | from .oauth2 import OAuth2, write_bearer_token_file 9 | 10 | try: 11 | _input = raw_input 12 | except NameError: 13 | _input = input 14 | 15 | 16 | def oauth2_dance(consumer_key, consumer_secret, token_filename=None): 17 | """ 18 | Perform the OAuth2 dance to transform a consumer key and secret into a 19 | bearer token. 20 | 21 | If a token_filename is given, the bearer token will be written to 22 | the file. 23 | """ 24 | twitter = Twitter( 25 | auth=OAuth2(consumer_key=consumer_key, consumer_secret=consumer_secret), 26 | format="", 27 | api_version="") 28 | token = json.loads(twitter.oauth2.token(grant_type="client_credentials"))["access_token"] 29 | if token_filename: 30 | write_bearer_token_file(token_filename, token) 31 | return token 32 | 33 | def get_oauth_pin(oauth_url, open_browser=True): 34 | """ 35 | Prompt the user for the OAuth PIN. 36 | 37 | By default, a browser will open the authorization page. If `open_browser` 38 | is false, the authorization URL will just be printed instead. 39 | """ 40 | 41 | print("Opening: %s\n" % oauth_url) 42 | 43 | if open_browser: 44 | print(""" 45 | In the web browser window that opens please choose to Allow 46 | access. Copy the PIN number that appears on the next page and paste or 47 | type it here: 48 | """) 49 | 50 | try: 51 | r = webbrowser.open(oauth_url) 52 | time.sleep(2) # Sometimes the last command can print some 53 | # crap. Wait a bit so it doesn't mess up the next 54 | # prompt. 55 | if not r: 56 | raise Exception() 57 | except: 58 | print(""" 59 | Uh, I couldn't open a browser on your computer. Please go here to get 60 | your PIN: 61 | 62 | """ + oauth_url) 63 | 64 | else: # not using a browser 65 | print(""" 66 | Please go to the following URL, authorize the app, and copy the PIN: 67 | 68 | """ + oauth_url) 69 | 70 | return _input("Please enter the PIN: ").strip() 71 | 72 | 73 | def oauth_dance(app_name, consumer_key, consumer_secret, token_filename=None, open_browser=True): 74 | """ 75 | Perform the OAuth dance with some command-line prompts. Return the 76 | oauth_token and oauth_token_secret. 77 | 78 | Provide the name of your app in `app_name`, your consumer_key, and 79 | consumer_secret. This function will let the user allow your app to access 80 | their Twitter account using PIN authentication. 81 | 82 | If a `token_filename` is given, the oauth tokens will be written to 83 | the file. 84 | 85 | By default, this function attempts to open a browser to request access. If 86 | `open_browser` is false it will just print the URL instead. 87 | """ 88 | print("Hi there! We're gonna get you all set up to use %s." % app_name) 89 | twitter = Twitter( 90 | auth=OAuth('', '', consumer_key, consumer_secret), 91 | format='', api_version=None) 92 | oauth_token, oauth_token_secret = parse_oauth_tokens( 93 | twitter.oauth.request_token(oauth_callback="oob")) 94 | oauth_url = ('https://api.twitter.com/oauth/authorize?oauth_token=' + 95 | oauth_token) 96 | oauth_verifier = get_oauth_pin(oauth_url, open_browser) 97 | 98 | twitter = Twitter( 99 | auth=OAuth( 100 | oauth_token, oauth_token_secret, consumer_key, consumer_secret), 101 | format='', api_version=None) 102 | oauth_token, oauth_token_secret = parse_oauth_tokens( 103 | twitter.oauth.access_token(oauth_verifier=oauth_verifier)) 104 | if token_filename: 105 | write_token_file( 106 | token_filename, oauth_token, oauth_token_secret) 107 | print() 108 | print("That's it! Your authorization keys have been written to %s." % ( 109 | token_filename)) 110 | return oauth_token, oauth_token_secret 111 | 112 | def parse_oauth_tokens(result): 113 | for r in result.split('&'): 114 | k, v = r.split('=') 115 | if k == 'oauth_token': 116 | oauth_token = v 117 | elif k == 'oauth_token_secret': 118 | oauth_token_secret = v 119 | return oauth_token, oauth_token_secret 120 | -------------------------------------------------------------------------------- /twitter/stream.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from .util import PY_3_OR_HIGHER 5 | 6 | try: 7 | import ssl 8 | except ImportError: 9 | _HAVE_SSL = False 10 | else: 11 | _HAVE_SSL = True 12 | 13 | if PY_3_OR_HIGHER: 14 | import urllib.request as urllib_request 15 | import urllib.error as urllib_error 16 | else: 17 | import urllib2 as urllib_request 18 | import urllib2 as urllib_error 19 | 20 | import certifi 21 | 22 | import json 23 | from ssl import SSLError 24 | import socket 25 | import codecs 26 | import select, time 27 | 28 | from .api import TwitterCall, wrap_response, TwitterHTTPError 29 | 30 | CRLF = b'\r\n' 31 | MIN_SOCK_TIMEOUT = 0.0 # Apparenty select with zero wait is okay! 32 | MAX_SOCK_TIMEOUT = 10.0 33 | HEARTBEAT_TIMEOUT = 90.0 34 | 35 | Timeout = {'timeout': True} 36 | Hangup = {'hangup': True} 37 | DecodeError = {'hangup': True, 'decode_error': True} 38 | HeartbeatTimeout = {'hangup': True, 'heartbeat_timeout': True} 39 | 40 | 41 | class HttpChunkDecoder(object): 42 | 43 | def __init__(self): 44 | self.buf = bytearray() 45 | self.munch_crlf = False 46 | 47 | def decode(self, data): # -> (bytearray, end_of_stream, decode_error) 48 | chunks = [] 49 | buf = self.buf 50 | munch_crlf = self.munch_crlf 51 | end_of_stream = False 52 | decode_error = False 53 | buf.extend(data) 54 | while True: 55 | if munch_crlf: 56 | # Dang, Twitter, you crazy. Twitter only sends a terminating 57 | # CRLF at the beginning of the *next* message. 58 | if len(buf) >= 2: 59 | buf = buf[2:] 60 | munch_crlf = False 61 | else: 62 | break 63 | 64 | header_end_pos = buf.find(CRLF) 65 | if header_end_pos == -1: 66 | break 67 | 68 | header = buf[:header_end_pos] 69 | data_start_pos = header_end_pos + 2 70 | try: 71 | chunk_len = int(header.decode('ascii'), 16) 72 | except ValueError: 73 | decode_error = True 74 | break 75 | 76 | if chunk_len == 0: 77 | end_of_stream = True 78 | break 79 | 80 | data_end_pos = data_start_pos + chunk_len 81 | 82 | if len(buf) >= data_end_pos: 83 | chunks.append(buf[data_start_pos:data_end_pos]) 84 | buf = buf[data_end_pos:] 85 | munch_crlf = True 86 | else: 87 | break 88 | self.buf = buf 89 | self.munch_crlf = munch_crlf 90 | return bytearray().join(chunks), end_of_stream, decode_error 91 | 92 | 93 | class JsonDecoder(object): 94 | 95 | def __init__(self): 96 | self.buf = "" 97 | self.raw_decode = json.JSONDecoder().raw_decode 98 | 99 | def decode(self, data): 100 | chunks = [] 101 | buf = self.buf + data 102 | while True: 103 | try: 104 | buf = buf.lstrip() 105 | res, ptr = self.raw_decode(buf) 106 | buf = buf[ptr:] 107 | chunks.append(res) 108 | except ValueError: 109 | break 110 | self.buf = buf 111 | return chunks 112 | 113 | 114 | class Timer(object): 115 | 116 | def __init__(self, timeout): 117 | # If timeout is None, we never expire. 118 | self.timeout = timeout 119 | self.reset() 120 | 121 | def reset(self): 122 | self.time = time.time() 123 | 124 | def expired(self): 125 | """ 126 | If expired, reset the timer and return True. 127 | """ 128 | if self.timeout is None: 129 | return False 130 | elif time.time() - self.time > self.timeout: 131 | self.reset() 132 | return True 133 | return False 134 | 135 | 136 | class SockReader(object): 137 | def __init__(self, sock, sock_timeout): 138 | self.sock = sock 139 | self.sock_timeout = sock_timeout 140 | 141 | def read(self): 142 | try: 143 | ready_to_read = select.select([self.sock], [], [], self.sock_timeout)[0] 144 | if ready_to_read: 145 | return self.sock.read() 146 | except SSLError as e: 147 | # Code 2 is error from a non-blocking read of an empty buffer. 148 | if e.errno != 2: 149 | raise 150 | return bytearray() 151 | 152 | 153 | class TwitterJSONIter(object): 154 | 155 | def __init__(self, handle, uri, arg_data, block, timeout, heartbeat_timeout): 156 | self.handle = handle 157 | self.uri = uri 158 | self.arg_data = arg_data 159 | self.timeout_token = Timeout 160 | self.timeout = None 161 | self.heartbeat_timeout = HEARTBEAT_TIMEOUT 162 | if timeout and timeout > 0: 163 | self.timeout = float(timeout) 164 | elif not (block or timeout): 165 | self.timeout_token = None 166 | self.timeout = MIN_SOCK_TIMEOUT 167 | if heartbeat_timeout and heartbeat_timeout > 0: 168 | self.heartbeat_timeout = float(heartbeat_timeout) 169 | 170 | def __iter__(self): 171 | timeouts = [t for t in (self.timeout, self.heartbeat_timeout, MAX_SOCK_TIMEOUT) 172 | if t is not None] 173 | sock_timeout = min(*timeouts) 174 | sock = self.handle.fp.raw._sock if PY_3_OR_HIGHER else self.handle.fp._sock.fp._sock 175 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) 176 | headers = self.handle.headers 177 | sock_reader = SockReader(sock, sock_timeout) 178 | chunk_decoder = HttpChunkDecoder() 179 | utf8_decoder = codecs.getincrementaldecoder("utf-8")() 180 | json_decoder = JsonDecoder() 181 | timer = Timer(self.timeout) 182 | heartbeat_timer = Timer(self.heartbeat_timeout) 183 | 184 | while True: 185 | # Decode all the things: 186 | try: 187 | data = sock_reader.read() 188 | except SSLError: 189 | yield Hangup 190 | break 191 | dechunked_data, end_of_stream, decode_error = chunk_decoder.decode(data) 192 | unicode_data = utf8_decoder.decode(dechunked_data) 193 | json_data = json_decoder.decode(unicode_data) 194 | 195 | # Yield data-like things: 196 | for json_obj in json_data: 197 | yield wrap_response(json_obj, headers) 198 | 199 | # Reset timers: 200 | if dechunked_data: 201 | heartbeat_timer.reset() 202 | if json_data: 203 | timer.reset() 204 | 205 | # Yield timeouts and special things: 206 | if end_of_stream: 207 | yield Hangup 208 | break 209 | if decode_error: 210 | yield DecodeError 211 | break 212 | if heartbeat_timer.expired(): 213 | yield HeartbeatTimeout 214 | break 215 | if timer.expired(): 216 | yield self.timeout_token 217 | 218 | 219 | def handle_stream_response(req, uri, arg_data, block, timeout, heartbeat_timeout, verify_context=True): 220 | try: 221 | context = None 222 | if _HAVE_SSL: 223 | if not verify_context: 224 | context = ssl._create_unverified_context() 225 | else: 226 | context = ssl.create_default_context() 227 | context.load_verify_locations(cafile=certifi.where()) 228 | handle = urllib_request.urlopen(req, context=context) 229 | except urllib_error.HTTPError as e: 230 | raise TwitterHTTPError(e, uri, 'json', arg_data) 231 | return iter(TwitterJSONIter(handle, uri, arg_data, block, timeout, heartbeat_timeout)) 232 | 233 | class TwitterStream(TwitterCall): 234 | """ 235 | The TwitterStream object is an interface to the Twitter Stream 236 | API. This can be used pretty much the same as the Twitter class 237 | except the result of calling a method will be an iterator that 238 | yields objects decoded from the stream. For example:: 239 | 240 | twitter_stream = TwitterStream(auth=OAuth(...)) 241 | iterator = twitter_stream.statuses.sample() 242 | 243 | for tweet in iterator: 244 | # ...do something with this tweet... 245 | 246 | Per default the ``TwitterStream`` object uses 247 | [public streams](https://dev.twitter.com/docs/streaming-apis/streams/public). 248 | If you want to use one of the other 249 | [streaming APIs](https://dev.twitter.com/docs/streaming-apis), specify the URL 250 | manually. 251 | 252 | The iterator will yield until the TCP connection breaks. When the 253 | connection breaks, the iterator yields `{'hangup': True}`, and 254 | raises `StopIteration` if iterated again. 255 | 256 | Similarly, if the stream does not produce heartbeats for more than 257 | 90 seconds, the iterator yields `{'hangup': True, 258 | 'heartbeat_timeout': True}`, and raises `StopIteration` if 259 | iterated again. 260 | 261 | The `timeout` parameter controls the maximum time between 262 | yields. If it is nonzero, then the iterator will yield either 263 | stream data or `{'timeout': True}` within the timeout period. This 264 | is useful if you want your program to do other stuff in between 265 | waiting for tweets. 266 | 267 | The `block` parameter sets the stream to be fully non-blocking. In 268 | this mode, the iterator always yields immediately. It returns 269 | stream data, or `None`. Note that `timeout` supercedes this 270 | argument, so it should also be set `None` to use this mode. 271 | """ 272 | def __init__(self, domain="stream.twitter.com", secure=True, auth=None, 273 | api_version='1.1', block=True, timeout=None, 274 | heartbeat_timeout=90.0, verify_context=True): 275 | uriparts = (str(api_version),) 276 | 277 | class TwitterStreamCall(TwitterCall): 278 | def _handle_response(self, req, uri, arg_data, _timeout=None): 279 | return handle_stream_response( 280 | req, uri, arg_data, block, 281 | _timeout or timeout, heartbeat_timeout, verify_context) 282 | 283 | TwitterCall.__init__( 284 | self, auth=auth, format="json", domain=domain, 285 | callable_cls=TwitterStreamCall, 286 | secure=secure, uriparts=uriparts, timeout=timeout, gzip=False, 287 | retry=False, verify_context=verify_context) 288 | -------------------------------------------------------------------------------- /twitter/stream_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example program for the Stream API. This prints public status messages 3 | from the "sample" stream as fast as possible. Use -h for help. 4 | """ 5 | 6 | from __future__ import print_function 7 | 8 | import argparse 9 | 10 | from twitter.stream import TwitterStream, Timeout, HeartbeatTimeout, Hangup 11 | from twitter.oauth import OAuth 12 | from twitter.util import printNicely 13 | 14 | 15 | def parse_arguments(): 16 | 17 | parser = argparse.ArgumentParser(description=__doc__ or "") 18 | 19 | parser.add_argument('-t', '--token', required=True, 20 | help='The Twitter Access Token.') 21 | parser.add_argument('-ts', '--token-secret', required=True, 22 | help='The Twitter Access Token Secret.') 23 | parser.add_argument('-ck', '--consumer-key', required=True, 24 | help='The Twitter Consumer Key.') 25 | parser.add_argument('-cs', '--consumer-secret', required=True, 26 | help='The Twitter Consumer Secret.') 27 | parser.add_argument('-to', '--timeout', 28 | help='Timeout for the stream (seconds).') 29 | parser.add_argument('-ht', '--heartbeat-timeout', 30 | help='Set heartbeat timeout.', default=90) 31 | parser.add_argument('-nb', '--no-block', action='store_true', 32 | help='Set stream to non-blocking.') 33 | parser.add_argument('-tt', '--track-keywords', 34 | help='Search the stream for specific text.') 35 | return parser.parse_args() 36 | 37 | 38 | def main(): 39 | args = parse_arguments() 40 | 41 | # When using twitter stream you must authorize. 42 | auth = OAuth(args.token, args.token_secret, 43 | args.consumer_key, args.consumer_secret) 44 | 45 | # These arguments are optional: 46 | stream_args = dict( 47 | timeout=args.timeout, 48 | block=not args.no_block, 49 | heartbeat_timeout=args.heartbeat_timeout) 50 | 51 | query_args = dict() 52 | if args.track_keywords: 53 | query_args['track'] = args.track_keywords 54 | 55 | stream = TwitterStream(auth=auth, **stream_args) 56 | if args.track_keywords: 57 | tweet_iter = stream.statuses.filter(**query_args) 58 | else: 59 | tweet_iter = stream.statuses.sample() 60 | 61 | # Iterate over the sample stream. 62 | for tweet in tweet_iter: 63 | # You must test that your tweet has text. It might be a delete 64 | # or data message. 65 | if tweet is None: 66 | printNicely("-- None --") 67 | elif tweet is Timeout: 68 | printNicely("-- Timeout --") 69 | elif tweet is HeartbeatTimeout: 70 | printNicely("-- Heartbeat Timeout --") 71 | elif tweet is Hangup: 72 | printNicely("-- Hangup --") 73 | elif tweet.get('text'): 74 | printNicely(tweet['text']) 75 | else: 76 | printNicely("-- Some data: " + str(tweet)) 77 | 78 | if __name__ == '__main__': 79 | main() 80 | -------------------------------------------------------------------------------- /twitter/timezones.py: -------------------------------------------------------------------------------- 1 | # Retrieved from http://docs.python.org/2/library/datetime.html on 2013-05-24 2 | from datetime import tzinfo, timedelta, datetime 3 | 4 | ZERO = timedelta(0) 5 | HOUR = timedelta(hours=1) 6 | 7 | # A UTC class. 8 | 9 | class UTC(tzinfo): 10 | """UTC""" 11 | 12 | def utcoffset(self, dt): 13 | return ZERO 14 | 15 | def tzname(self, dt): 16 | return "UTC" 17 | 18 | def dst(self, dt): 19 | return ZERO 20 | 21 | utc = UTC() 22 | 23 | # A class building tzinfo objects for fixed-offset time zones. 24 | # Note that FixedOffset(0, "UTC") is a different way to build a 25 | # UTC tzinfo object. 26 | 27 | class FixedOffset(tzinfo): 28 | """Fixed offset in minutes east from UTC.""" 29 | 30 | def __init__(self, offset, name): 31 | self.__offset = timedelta(minutes = offset) 32 | self.__name = name 33 | 34 | def utcoffset(self, dt): 35 | return self.__offset 36 | 37 | def tzname(self, dt): 38 | return self.__name 39 | 40 | def dst(self, dt): 41 | return ZERO 42 | 43 | # A class capturing the platform's idea of local time. 44 | 45 | import time as _time 46 | 47 | STDOFFSET = timedelta(seconds = -_time.timezone) 48 | if _time.daylight: 49 | DSTOFFSET = timedelta(seconds = -_time.altzone) 50 | else: 51 | DSTOFFSET = STDOFFSET 52 | 53 | DSTDIFF = DSTOFFSET - STDOFFSET 54 | 55 | class LocalTimezone(tzinfo): 56 | 57 | def utcoffset(self, dt): 58 | if self._isdst(dt): 59 | return DSTOFFSET 60 | else: 61 | return STDOFFSET 62 | 63 | def dst(self, dt): 64 | if self._isdst(dt): 65 | return DSTDIFF 66 | else: 67 | return ZERO 68 | 69 | def tzname(self, dt): 70 | return _time.tzname[self._isdst(dt)] 71 | 72 | def _isdst(self, dt): 73 | tt = (dt.year, dt.month, dt.day, 74 | dt.hour, dt.minute, dt.second, 75 | dt.weekday(), 0, 0) 76 | stamp = _time.mktime(tt) 77 | tt = _time.localtime(stamp) 78 | return tt.tm_isdst > 0 79 | 80 | Local = LocalTimezone() 81 | 82 | -------------------------------------------------------------------------------- /twitter/twitter_globals.py: -------------------------------------------------------------------------------- 1 | ''' 2 | .. data:: POST_ACTIONS 3 | List of twitter method names that require the use of POST 4 | ''' 5 | 6 | POST_ACTIONS = [ 7 | 8 | # Status Methods 9 | 'update', 'retweet', 'update_with_media', 'statuses/lookup', 10 | 11 | # Direct Message Methods 12 | 'new', 13 | 14 | # Account Methods 15 | 'update_profile_image', 'update_delivery_device', 'update_profile', 16 | 'update_profile_background_image', 'update_profile_colors', 17 | 'update_location', 'end_session', 'settings', 18 | 'update_profile_banner', 'remove_profile_banner', 19 | 20 | # Notification Methods 21 | 'leave', 'follow', 22 | 23 | # Status Methods, Block Methods, Direct Message Methods, 24 | # Friendship Methods, Favorite Methods 25 | 'destroy', 'destroy_all', 26 | 27 | # Block Methods, Friendship Methods, Favorite Methods 28 | 'create', 'create_all', 29 | 30 | # Users Methods 31 | 'users/lookup', 'report_spam', 32 | 33 | # Streaming Methods 34 | 'filter', 'user', 'site', 35 | 36 | # OAuth Methods 37 | 'token', 'access_token', 38 | 'request_token', 'invalidate_token', 39 | 40 | # Upload Methods 41 | 'media/upload', 'media/metadata/create', 42 | 43 | # Collections Methods 44 | 'collections/create', 'collections/destroy', 'collections/update', 45 | 'collections/entries/add', 'collections/entries/curate', 46 | 'collections/entries/move', 'collections/entries/remove' 47 | ] 48 | -------------------------------------------------------------------------------- /twitter/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Internal utility functions. 3 | 4 | `htmlentitydecode` came from here: 5 | http://wiki.python.org/moin/EscapingHtml 6 | """ 7 | 8 | from __future__ import print_function 9 | 10 | import contextlib 11 | import re 12 | import sys 13 | import textwrap 14 | import time 15 | import socket 16 | 17 | PY_3_OR_HIGHER = sys.version_info >= (3, 0) 18 | 19 | try: 20 | from html.entities import name2codepoint 21 | unichr = chr 22 | import urllib.request as urllib2 23 | import urllib.parse as urlparse 24 | except ImportError: 25 | from htmlentitydefs import name2codepoint 26 | import urllib2 27 | import urlparse 28 | 29 | def htmlentitydecode(s): 30 | return re.sub( 31 | '&(%s);' % '|'.join(name2codepoint), 32 | lambda m: unichr(name2codepoint[m.group(1)]), s) 33 | 34 | def smrt_input(globals_, locals_, ps1=">>> ", ps2="... "): 35 | inputs = [] 36 | while True: 37 | if inputs: 38 | prompt = ps2 39 | else: 40 | prompt = ps1 41 | inputs.append(input(prompt)) 42 | try: 43 | ret = eval('\n'.join(inputs), globals_, locals_) 44 | if ret: 45 | print(str(ret)) 46 | return 47 | except SyntaxError: 48 | pass 49 | 50 | def printNicely(string): 51 | if hasattr(sys.stdout, 'buffer'): 52 | sys.stdout.buffer.write(string.encode('utf8')) 53 | print() 54 | sys.stdout.buffer.flush() 55 | sys.stdout.flush() 56 | else: 57 | print(string.encode('utf8')) 58 | 59 | def actually_bytes(stringy): 60 | if PY_3_OR_HIGHER: 61 | if type(stringy) == bytes: 62 | pass 63 | elif type(stringy) != str: 64 | stringy = str(stringy) 65 | if type(stringy) == str: 66 | stringy = stringy.encode("utf-8") 67 | else: 68 | if type(stringy) == str: 69 | pass 70 | elif type(stringy) != unicode: 71 | stringy = str(stringy) 72 | if type(stringy) == unicode: 73 | stringy = stringy.encode("utf-8") 74 | return stringy 75 | 76 | def err(msg=""): 77 | print(msg, file=sys.stderr) 78 | 79 | 80 | class Fail(object): 81 | """A class to count fails during a repetitive task. 82 | 83 | Args: 84 | maximum: An integer for the maximum of fails to allow. 85 | exit: An integer for the exit code when maximum of fail is reached. 86 | 87 | Methods: 88 | count: Count a fail, exit when maximum of fails is reached. 89 | wait: Same as count but also sleep for a given time in seconds. 90 | """ 91 | def __init__(self, maximum=10, exit=1): 92 | self.i = maximum 93 | self.exit = exit 94 | 95 | def count(self): 96 | self.i -= 1 97 | if self.i == 0: 98 | err("Too many consecutive fails, exiting.") 99 | raise SystemExit(self.exit) 100 | 101 | def wait(self, delay=0): 102 | self.count() 103 | if delay > 0: 104 | time.sleep(delay) 105 | 106 | 107 | def find_links(line): 108 | """Find all links in the given line. The function returns a sprintf style 109 | format string (with %s placeholders for the links) and a list of urls.""" 110 | l = line.replace("%", "%%") 111 | regex = "(https?://[^ )]+)" 112 | return ( 113 | re.sub(regex, "%s", l), 114 | [m.group(1) for m in re.finditer(regex, l)]) 115 | 116 | def follow_redirects(link, sites= None): 117 | """Follow directs for the link as long as the redirects are on the given 118 | sites and return the resolved link.""" 119 | def follow(url): 120 | return sites == None or urlparse.urlparse(url).hostname in sites 121 | 122 | class RedirectHandler(urllib2.HTTPRedirectHandler): 123 | def __init__(self): 124 | self.last_url = None 125 | def redirect_request(self, req, fp, code, msg, hdrs, newurl): 126 | self.last_url = newurl 127 | if not follow(newurl): 128 | return None 129 | r = urllib2.HTTPRedirectHandler.redirect_request( 130 | self, req, fp, code, msg, hdrs, newurl) 131 | r.get_method = lambda : 'HEAD' 132 | return r 133 | 134 | if not follow(link): 135 | return link 136 | redirect_handler = RedirectHandler() 137 | opener = urllib2.build_opener(redirect_handler) 138 | req = urllib2.Request(link) 139 | req.get_method = lambda : 'HEAD' 140 | try: 141 | with contextlib.closing(opener.open(req,timeout=1)) as site: 142 | return site.url 143 | except: 144 | return redirect_handler.last_url if redirect_handler.last_url else link 145 | 146 | def expand_line(line, sites): 147 | """Expand the links in the line for the given sites.""" 148 | try: 149 | l = line.strip() 150 | msg_format, links = find_links(l) 151 | args = tuple(follow_redirects(l, sites) for l in links) 152 | line = msg_format % args 153 | except Exception as e: 154 | try: 155 | err("expanding line %s failed due to %s" % (line, unicode(e))) 156 | except: 157 | pass 158 | return line 159 | 160 | def parse_host_list(list_of_hosts): 161 | """Parse the comma separated list of hosts.""" 162 | p = { 163 | m.group(1) for m in re.finditer(r"\s*([^,\s]+)\s*,?\s*", list_of_hosts)} 164 | return p 165 | 166 | 167 | def align_text(text, left_margin=17, max_width=160): 168 | lines = [] 169 | for line in text.split('\n'): 170 | temp_lines = textwrap.wrap(line, max_width - left_margin) 171 | temp_lines = [(' ' * left_margin + line) for line in temp_lines] 172 | lines.append('\n'.join(temp_lines)) 173 | ret = '\n'.join(lines) 174 | return ret.lstrip() 175 | 176 | 177 | __all__ = ["htmlentitydecode", "smrt_input"] 178 | -------------------------------------------------------------------------------- /utils/update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | DEPRECATED 5 | This is a development script, intended for development use only. 6 | 7 | This script generates the POST_ACTIONS variable 8 | for placement in the twitter_globals.py 9 | 10 | Example Usage: 11 | 12 | %prog >twitter/twitter_globals.py 13 | 14 | Dependencies: 15 | 16 | (easy_install) BeautifulSoup 17 | ''' 18 | 19 | import sys 20 | from urllib import urlopen as _open 21 | from BeautifulSoup import BeautifulSoup 22 | from htmlentitydefs import codepoint2name 23 | 24 | def uni2html(u): 25 | ''' 26 | Convert unicode to html. 27 | 28 | Basically leaves ascii chars as is, and attempts to encode unicode chars 29 | as HTML entities. If the conversion fails the character is skipped. 30 | ''' 31 | htmlentities = list() 32 | for c in u: 33 | ord_c = ord(c) 34 | if ord_c < 128: 35 | # ignoring all ^chars like ^M ^R ^E 36 | if ord_c >31: 37 | htmlentities.append(c) 38 | else: 39 | try: 40 | htmlentities.append('&%s;' % codepoint2name[ord_c]) 41 | except KeyError: 42 | pass # Charachter unknown 43 | return ''.join(htmlentities) 44 | 45 | def print_fw(iterable, joins=', ', prefix='', indent=0, width=79, trail=False): 46 | ''' 47 | PPrint an iterable (of stringable elements). 48 | 49 | Entries are joined using `joins` 50 | A fixed_width (fw) is maintained of `width` chars per line 51 | Each line is indented with `indent`*4 spaces 52 | Lines are then prefixed with `prefix` string 53 | if `trail` a trailing comma is sent to stdout 54 | A newline is written after all is printed. 55 | ''' 56 | shift_width = 4 57 | preline = '%s%s' %(' '*shift_width, prefix) 58 | linew = len(preline) 59 | sys.stdout.write(preline) 60 | for i, entry in enumerate(iterable): 61 | if not trail and i == len(iterable) - 1: 62 | sentry = str(entry) 63 | else: 64 | sentry = '%s%s' %(str(entry), joins) 65 | if linew + len(sentry) > width: 66 | sys.stdout.write('\n%s' %(preline)) 67 | linew = len(preline) 68 | sys.stdout.write(sentry) 69 | linew += len(sentry) 70 | sys.stdout.write('\n') 71 | 72 | def main_with_options(options, files): 73 | ''' 74 | Main function the prints twitter's _POST_ACTIONS to stdout 75 | 76 | TODO: look at possibly dividing up this function 77 | ''' 78 | 79 | apifile = _open('http://apiwiki.twitter.com/REST+API+Documentation') 80 | try: 81 | apihtml = uni2html(apifile.read()) 82 | finally: 83 | apifile.close() 84 | 85 | ## Parsing the ApiWiki Page 86 | 87 | apidoc = BeautifulSoup(apihtml) 88 | toc = apidoc.find('div', {'class':'toc'}) 89 | toc_entries = toc.findAll('li', text=lambda text: 'Methods' in text) 90 | method_links = {} 91 | for entry in toc_entries: 92 | links = entry.parent.parent.findAll('a') 93 | method_links[links[0].string] = [x['href'] for x in links[1:]] 94 | 95 | # Create unique hash of mehods with POST_ACTIONS 96 | POST_ACTION_HASH = {} 97 | for method_type, methods in method_links.items(): 98 | for method in methods: 99 | # Strip the hash (#) mark from the method id/name 100 | method = method[1:] 101 | method_body = apidoc.find('a', {'name': method}) 102 | value = list(method_body.findNext( 103 | 'b', text=lambda text: 'Method' in text 104 | ).parent.parent.childGenerator())[-1] 105 | if 'POST' in value: 106 | method_name = method_body.findNext('h3').string 107 | try: 108 | POST_ACTION_HASH[method_name] += (method_type,) 109 | except KeyError: 110 | POST_ACTION_HASH[method_name] = (method_type,) 111 | 112 | # Reverse the POST_ACTION_HASH 113 | # this is done to allow generation of nice comment strings 114 | POST_ACTION_HASH_R = {} 115 | for method, method_types in POST_ACTION_HASH.items(): 116 | try: 117 | POST_ACTION_HASH_R[method_types].append(method) 118 | except KeyError: 119 | POST_ACTION_HASH_R[method_types] = [method] 120 | 121 | ## Print the POST_ACTIONS to stdout as a Python List 122 | print """''' 123 | This module is automatically generated using `update.py` 124 | 125 | .. data:: POST_ACTIONS 126 | List of twitter method names that require the use of POST 127 | ''' 128 | """ 129 | print 'POST_ACTIONS = [\n' 130 | for method_types, methods in POST_ACTION_HASH_R.items(): 131 | print_fw(method_types, prefix='# ', indent=1) 132 | print_fw([repr(str(x)) for x in methods], indent=1, trail=True) 133 | print "" 134 | print ']' 135 | 136 | def main(): 137 | import optparse 138 | 139 | class CustomFormatter(optparse.IndentedHelpFormatter): 140 | """formatter that overrides description reformatting.""" 141 | def format_description(self, description): 142 | ''' indents each line in the description ''' 143 | return "\n".join([ 144 | "%s%s" %(" "*((self.level+1)*self.indent_increment), line) 145 | for line in description.splitlines() 146 | ]) + "\n" 147 | 148 | parser = optparse.OptionParser( 149 | formatter=CustomFormatter(), 150 | description=__doc__ 151 | ) 152 | (options, files) = parser.parse_args() 153 | main_with_options(options, files) 154 | 155 | 156 | if __name__ == "__main__": 157 | main() 158 | --------------------------------------------------------------------------------