├── .gitignore ├── LICENSE ├── README.md └── twitter2 ├── appserver ├── static │ ├── appIcon.png │ ├── appLogo_white_splunkpowered.png │ ├── application.css │ ├── screenshot-full.png │ └── screenshot.png └── templates │ └── redirect.tmpl ├── bin ├── oauthlib │ ├── PKG-INFO │ ├── __init__.py │ ├── common.py │ ├── oauth1 │ │ ├── __init__.py │ │ └── rfc5849 │ │ │ ├── __init__.py │ │ │ ├── endpoints │ │ │ ├── __init__.py │ │ │ ├── access_token.py │ │ │ ├── authorization.py │ │ │ ├── base.py │ │ │ ├── pre_configured.py │ │ │ ├── request_token.py │ │ │ └── resource.py │ │ │ ├── errors.py │ │ │ ├── parameters.py │ │ │ ├── request_validator.py │ │ │ ├── signature.py │ │ │ └── utils.py │ ├── oauth2 │ │ ├── __init__.py │ │ └── rfc6749 │ │ │ ├── __init__.py │ │ │ ├── clients │ │ │ ├── __init__.py │ │ │ ├── backend_application.py │ │ │ ├── base.py │ │ │ ├── legacy_application.py │ │ │ ├── mobile_application.py │ │ │ └── web_application.py │ │ │ ├── endpoints │ │ │ ├── __init__.py │ │ │ ├── authorization.py │ │ │ ├── base.py │ │ │ ├── pre_configured.py │ │ │ ├── resource.py │ │ │ └── token.py │ │ │ ├── errors.py │ │ │ ├── grant_types │ │ │ ├── __init__.py │ │ │ ├── authorization_code.py │ │ │ ├── base.py │ │ │ ├── client_credentials.py │ │ │ ├── implicit.py │ │ │ ├── refresh_token.py │ │ │ └── resource_owner_password_credentials.py │ │ │ ├── parameters.py │ │ │ ├── request_validator.py │ │ │ ├── tokens.py │ │ │ └── utils.py │ └── uri_validate.py ├── requests │ ├── PKG-INFO │ ├── __init__.py │ ├── adapters.py │ ├── api.py │ ├── auth.py │ ├── cacert.pem │ ├── certs.py │ ├── compat.py │ ├── cookies.py │ ├── exceptions.py │ ├── hooks.py │ ├── models.py │ ├── packages │ │ ├── __init__.py │ │ ├── charade │ │ │ ├── __init__.py │ │ │ ├── big5freq.py │ │ │ ├── big5prober.py │ │ │ ├── chardistribution.py │ │ │ ├── charsetgroupprober.py │ │ │ ├── charsetprober.py │ │ │ ├── codingstatemachine.py │ │ │ ├── compat.py │ │ │ ├── constants.py │ │ │ ├── cp949prober.py │ │ │ ├── escprober.py │ │ │ ├── escsm.py │ │ │ ├── eucjpprober.py │ │ │ ├── euckrfreq.py │ │ │ ├── euckrprober.py │ │ │ ├── euctwfreq.py │ │ │ ├── euctwprober.py │ │ │ ├── gb2312freq.py │ │ │ ├── gb2312prober.py │ │ │ ├── hebrewprober.py │ │ │ ├── jisfreq.py │ │ │ ├── jpcntx.py │ │ │ ├── langbulgarianmodel.py │ │ │ ├── langcyrillicmodel.py │ │ │ ├── langgreekmodel.py │ │ │ ├── langhebrewmodel.py │ │ │ ├── langhungarianmodel.py │ │ │ ├── langthaimodel.py │ │ │ ├── latin1prober.py │ │ │ ├── mbcharsetprober.py │ │ │ ├── mbcsgroupprober.py │ │ │ ├── mbcssm.py │ │ │ ├── sbcharsetprober.py │ │ │ ├── sbcsgroupprober.py │ │ │ ├── sjisprober.py │ │ │ ├── universaldetector.py │ │ │ └── utf8prober.py │ │ └── urllib3 │ │ │ ├── __init__.py │ │ │ ├── _collections.py │ │ │ ├── connectionpool.py │ │ │ ├── contrib │ │ │ ├── __init__.py │ │ │ ├── ntlmpool.py │ │ │ └── pyopenssl.py │ │ │ ├── exceptions.py │ │ │ ├── filepost.py │ │ │ ├── packages │ │ │ ├── __init__.py │ │ │ ├── ordered_dict.py │ │ │ ├── six.py │ │ │ └── ssl_match_hostname │ │ │ │ └── __init__.py │ │ │ ├── poolmanager.py │ │ │ ├── request.py │ │ │ ├── response.py │ │ │ └── util.py │ ├── sessions.py │ ├── status_codes.py │ ├── structures.py │ └── utils.py ├── requests_oauthlib │ ├── PKG-INFO │ ├── __init__.py │ ├── core.py │ ├── oauth1_session.py │ ├── oauth2_auth.py │ └── oauth2_session.py ├── splunklib │ ├── __init__.py │ ├── binding.py │ ├── client.py │ ├── data.py │ ├── modularinput │ │ ├── __init__.py │ │ ├── argument.py │ │ ├── event.py │ │ ├── event_writer.py │ │ ├── input_definition.py │ │ ├── scheme.py │ │ ├── script.py │ │ ├── utils.py │ │ └── validation_definition.py │ ├── ordereddict.py │ ├── results.py │ └── searchcommands │ │ ├── __init__.py │ │ ├── csv │ │ ├── __init__.py │ │ ├── dialect.py │ │ ├── dict_reader.py │ │ └── dict_writer.py │ │ ├── decorators.py │ │ ├── generating_command.py │ │ ├── logging.py │ │ ├── reporting_command.py │ │ ├── search_command.py │ │ ├── search_command_internals.py │ │ ├── streaming_command.py │ │ └── validators.py ├── stream_tweets.py ├── twython │ ├── PKG-INFO │ ├── __init__.py │ ├── advisory.py │ ├── api.py │ ├── compat.py │ ├── endpoints.py │ ├── exceptions.py │ ├── helpers.py │ └── streaming │ │ ├── __init__.py │ │ ├── api.py │ │ └── types.py ├── uuid.py └── verify_twitter_oauth_settings.py ├── default ├── app.conf ├── data │ └── ui │ │ ├── nav │ │ └── default.xml │ │ └── views │ │ ├── twitter_drilldown.xml │ │ └── twitter_general.xml ├── indexes.conf ├── inputs.conf ├── props.conf ├── restart_on_install.conf ├── savedsearches.conf ├── setup.xml └── viewstates.conf └── django └── twitter2 ├── __init__.py ├── forms.py ├── templates └── setup.html ├── templatetags └── __init__.py ├── tests.py ├── urls.py └── views.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | *.py[co] 3 | 4 | # OS X 5 | .DS_Store 6 | 7 | # App local data 8 | local 9 | metadata 10 | 11 | # Packaged versions of the app 12 | twitter2.tgz 13 | twitter2.spl 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Twitter App for Splunk 2 | 3 | This app provides a scripted input for [Splunk](http://www.splunk.com/) that 4 | automatically extracts tweets from [Twitter](https://twitter.com/)'s [1% sample 5 | stream](https://dev.twitter.com/docs/api/1.1/get/statuses/sample). 6 | It is tested on Splunk 6.0 and 6.1. Splunk 5.0 and earlier is unsupported. 7 | 8 | A number of dashboards and searches are also included to demonstrate how Splunk 9 | can be used to visualize Twitter activity. 10 | 11 | ## Installation 12 | 13 | * Install the app by copying the `twitter2` directory to 14 | `$SPLUNK_HOME/etc/apps/twitter2`. 15 | 16 | * (Re)start Splunk so that the app is recognized. 17 | 18 | * In the Splunk web interface, from the App menu, select the Twitter app 19 | and press "Continue to app setup page". 20 | 21 | * Enter the OAuth settings for a Twitter application that will be used to access 22 | tweets from the sample stream and click "Save". 23 | 24 | If you don't already have a Twitter account, you can sign up for one at 25 | [https://twitter.com/](https://twitter.com/). If you need to create a Twitter 26 | application for accessing tweets, you can create one at 27 | [https://dev.twitter.com/apps](https://dev.twitter.com/apps). It need only be 28 | enabled for read access to Twitter data. See 29 | [https://dev.twitter.com/docs/application-permission-model](https://dev.twitter.com/docs/application-permission-model) 30 | for details on the Twitter application permission model 31 | 32 | * Wait 15 seconds or so for some tweets to be extracted. 33 | 34 | * Run the search `index=twitter` in Splunk to see the events. If you don't see 35 | any events, open `$SPLUNK_HOME/var/log/splunk/splunkd.log` and look for errors 36 | issued by ExecProcessor related to the 37 | `$SPLUNK_HOME/etc/apps/twitter2/bin/twitter_stream.py` script. 38 | 39 | ## Dashboards and Searches 40 | 41 | ### Views > Twitter General Activity 42 | 43 | Provides information about trending activity during the last 15 minutes. 44 | 45 | ### Views > Twitter Per-User Activity 46 | 47 | Drills down into activity related to a particular user or hashtag. 48 | 49 | This view could be used as a social dashboard for tracking activity related to a 50 | user of interest. 51 | 52 | ### Searches & Reports > Tweet Locations 53 | 54 | _NOTE: This view requires Splunk 6.1 or later._ 55 | 56 | Displays the locations of tweets on a map. 57 | 58 | ## License 59 | 60 | This software is licensed under the Apache License 2.0. Details can be found in 61 | the file LICENSE. 62 | -------------------------------------------------------------------------------- /twitter2/appserver/static/appIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-app-twitter/14ba32cf9b4d4e05f01d7547d88ab6b43e0ad2bb/twitter2/appserver/static/appIcon.png -------------------------------------------------------------------------------- /twitter2/appserver/static/appLogo_white_splunkpowered.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-app-twitter/14ba32cf9b4d4e05f01d7547d88ab6b43e0ad2bb/twitter2/appserver/static/appLogo_white_splunkpowered.png -------------------------------------------------------------------------------- /twitter2/appserver/static/application.css: -------------------------------------------------------------------------------- 1 | .appLogo { 2 | height: 43px; 3 | width: 133px; 4 | background-image: url(appLogo_white_splunkpowered.png); 5 | background-repeat: no-repeat; 6 | } 7 | -------------------------------------------------------------------------------- /twitter2/appserver/static/screenshot-full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-app-twitter/14ba32cf9b4d4e05f01d7547d88ab6b43e0ad2bb/twitter2/appserver/static/screenshot-full.png -------------------------------------------------------------------------------- /twitter2/appserver/static/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-app-twitter/14ba32cf9b4d4e05f01d7547d88ab6b43e0ad2bb/twitter2/appserver/static/screenshot.png -------------------------------------------------------------------------------- /twitter2/appserver/templates/redirect.tmpl: -------------------------------------------------------------------------------- 1 | <% 2 | import cherrypy, re 3 | from lib.i18n import current_lang 4 | root_endpoint = cherrypy.request.config.get('root_endpoint') 5 | root = (root_endpoint or '') + '/' 6 | # Remove duplicate slashes at beginning/end of root_endpoint 7 | root = re.sub(r'(/)\1+$', r'\1', re.sub(r'^(/)\1+', r'\1', root)) 8 | locale = "-".join([ x.lower() for x in current_lang()[0:2] if x is not None ]) 9 | %> 10 | 11 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-app-twitter/14ba32cf9b4d4e05f01d7547d88ab6b43e0ad2bb/twitter2/bin/oauthlib/__init__.py -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth1/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth1 6 | ~~~~~~~~~~~~~~ 7 | 8 | This module is a wrapper for the most recent implementation of OAuth 1.0 Client 9 | and Server classes. 10 | """ 11 | 12 | from .rfc5849 import Client 13 | from .rfc5849 import SIGNATURE_HMAC, SIGNATURE_RSA, SIGNATURE_PLAINTEXT 14 | from .rfc5849 import SIGNATURE_TYPE_AUTH_HEADER, SIGNATURE_TYPE_QUERY 15 | from .rfc5849 import SIGNATURE_TYPE_BODY 16 | from .rfc5849.request_validator import RequestValidator 17 | from .rfc5849.endpoints import RequestTokenEndpoint, AuthorizationEndpoint 18 | from .rfc5849.endpoints import AccessTokenEndpoint, ResourceEndpoint 19 | from .rfc5849.endpoints import WebApplicationServer 20 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth1/rfc5849/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .base import BaseEndpoint 4 | from .request_token import RequestTokenEndpoint 5 | from .authorization import AuthorizationEndpoint 6 | from .access_token import AccessTokenEndpoint 7 | from .resource import ResourceEndpoint 8 | from .pre_configured import WebApplicationServer 9 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth1/rfc5849/endpoints/pre_configured.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from . import RequestTokenEndpoint, AuthorizationEndpoint 4 | from . import AccessTokenEndpoint, ResourceEndpoint 5 | 6 | 7 | class WebApplicationServer(RequestTokenEndpoint, AuthorizationEndpoint, 8 | AccessTokenEndpoint, ResourceEndpoint): 9 | 10 | def __init__(self, request_validator): 11 | RequestTokenEndpoint.__init__(self, request_validator) 12 | AuthorizationEndpoint.__init__(self, request_validator) 13 | AccessTokenEndpoint.__init__(self, request_validator) 14 | ResourceEndpoint.__init__(self, request_validator) 15 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth1/rfc5849/errors.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | oauthlib.oauth2.rfc6749.errors 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Error used both by OAuth 2 clients and provicers to represent the spec 7 | defined error responses for all four core grant types. 8 | """ 9 | from __future__ import unicode_literals 10 | 11 | from oauthlib.common import urlencode, add_params_to_uri 12 | 13 | 14 | class OAuth1Error(Exception): 15 | error = None 16 | 17 | def __init__(self, description=None, uri=None, status_code=400, 18 | request=None): 19 | """ 20 | description: A human-readable ASCII [USASCII] text providing 21 | additional information, used to assist the client 22 | developer in understanding the error that occurred. 23 | Values for the "error_description" parameter MUST NOT 24 | include characters outside the set 25 | x20-21 / x23-5B / x5D-7E. 26 | 27 | uri: A URI identifying a human-readable web page with information 28 | about the error, used to provide the client developer with 29 | additional information about the error. Values for the 30 | "error_uri" parameter MUST conform to the URI- Reference 31 | syntax, and thus MUST NOT include characters outside the set 32 | x21 / x23-5B / x5D-7E. 33 | 34 | state: A CSRF protection value received from the client. 35 | 36 | request: Oauthlib Request object 37 | """ 38 | self.description = description 39 | self.uri = uri 40 | self.status_code = status_code 41 | 42 | def in_uri(self, uri): 43 | return add_params_to_uri(uri, self.twotuples) 44 | 45 | @property 46 | def twotuples(self): 47 | error = [('error', self.error)] 48 | if self.description: 49 | error.append(('error_description', self.description)) 50 | if self.uri: 51 | error.append(('error_uri', self.uri)) 52 | return error 53 | 54 | @property 55 | def urlencoded(self): 56 | return urlencode(self.twotuples) 57 | 58 | 59 | class InsecureTransportError(OAuth1Error): 60 | error = 'insecure_transport_protocol' 61 | description = 'Only HTTPS connections are permitted.' 62 | 63 | 64 | class InvalidSignatureMethodError(OAuth1Error): 65 | error = 'invalid_signature_method' 66 | 67 | 68 | class InvalidRequestError(OAuth1Error): 69 | error = 'invalid_request' 70 | 71 | 72 | class InvalidClientError(OAuth1Error): 73 | error = 'invalid_client' 74 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth1/rfc5849/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.utils 6 | ~~~~~~~~~~~~~~ 7 | 8 | This module contains utility methods used by various parts of the OAuth 9 | spec. 10 | """ 11 | 12 | try: 13 | import urllib2 14 | except ImportError: 15 | import urllib.request as urllib2 16 | 17 | from oauthlib.common import quote, unquote, bytes_type, unicode_type 18 | 19 | UNICODE_ASCII_CHARACTER_SET = ('abcdefghijklmnopqrstuvwxyz' 20 | 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 21 | '0123456789') 22 | 23 | 24 | def filter_params(target): 25 | """Decorator which filters params to remove non-oauth_* parameters 26 | 27 | Assumes the decorated method takes a params dict or list of tuples as its 28 | first argument. 29 | """ 30 | def wrapper(params, *args, **kwargs): 31 | params = filter_oauth_params(params) 32 | return target(params, *args, **kwargs) 33 | 34 | wrapper.__doc__ = target.__doc__ 35 | return wrapper 36 | 37 | 38 | def filter_oauth_params(params): 39 | """Removes all non oauth parameters from a dict or a list of params.""" 40 | is_oauth = lambda kv: kv[0].startswith("oauth_") 41 | if isinstance(params, dict): 42 | return list(filter(is_oauth, list(params.items()))) 43 | else: 44 | return list(filter(is_oauth, params)) 45 | 46 | 47 | def escape(u): 48 | """Escape a unicode string in an OAuth-compatible fashion. 49 | 50 | Per `section 3.6`_ of the spec. 51 | 52 | .. _`section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6 53 | 54 | """ 55 | if not isinstance(u, unicode_type): 56 | raise ValueError('Only unicode objects are escapable. ' + 57 | 'Got %s of type %s.' % (u, type(u))) 58 | # Letters, digits, and the characters '_.-' are already treated as safe 59 | # by urllib.quote(). We need to add '~' to fully support rfc5849. 60 | return quote(u, safe=b'~') 61 | 62 | 63 | def unescape(u): 64 | if not isinstance(u, unicode_type): 65 | raise ValueError('Only unicode objects are unescapable.') 66 | return unquote(u) 67 | 68 | 69 | def urlencode(query): 70 | """Encode a sequence of two-element tuples or dictionary into a URL query string. 71 | 72 | Operates using an OAuth-safe escape() method, in contrast to urllib.urlencode. 73 | """ 74 | # Convert dictionaries to list of tuples 75 | if isinstance(query, dict): 76 | query = query.items() 77 | return "&".join(['='.join([escape(k), escape(v)]) for k, v in query]) 78 | 79 | 80 | def parse_keqv_list(l): 81 | """A unicode-safe version of urllib2.parse_keqv_list""" 82 | # With Python 2.6, parse_http_list handles unicode fine 83 | return urllib2.parse_keqv_list(l) 84 | 85 | 86 | def parse_http_list(u): 87 | """A unicode-safe version of urllib2.parse_http_list""" 88 | # With Python 2.6, parse_http_list handles unicode fine 89 | return urllib2.parse_http_list(u) 90 | 91 | 92 | def parse_authorization_header(authorization_header): 93 | """Parse an OAuth authorization header into a list of 2-tuples""" 94 | auth_scheme = 'OAuth '.lower() 95 | if authorization_header[:len(auth_scheme)].lower().startswith(auth_scheme): 96 | items = parse_http_list(authorization_header[len(auth_scheme):]) 97 | try: 98 | return list(parse_keqv_list(items).items()) 99 | except (IndexError, ValueError): 100 | pass 101 | raise ValueError('Malformed authorization header') 102 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2 6 | ~~~~~~~~~~~~~~ 7 | 8 | This module is a wrapper for the most recent implementation of OAuth 2.0 Client 9 | and Server classes. 10 | """ 11 | 12 | from .rfc6749.clients import Client 13 | from .rfc6749.clients import WebApplicationClient 14 | from .rfc6749.clients import MobileApplicationClient 15 | from .rfc6749.clients import LegacyApplicationClient 16 | from .rfc6749.clients import BackendApplicationClient 17 | from .rfc6749.endpoints import AuthorizationEndpoint 18 | from .rfc6749.endpoints import TokenEndpoint 19 | from .rfc6749.endpoints import ResourceEndpoint 20 | from .rfc6749.endpoints import Server 21 | from .rfc6749.endpoints import WebApplicationServer 22 | from .rfc6749.endpoints import MobileApplicationServer 23 | from .rfc6749.endpoints import LegacyApplicationServer 24 | from .rfc6749.endpoints import BackendApplicationServer 25 | from .rfc6749.errors import * 26 | from .rfc6749.grant_types import AuthorizationCodeGrant 27 | from .rfc6749.grant_types import ImplicitGrant 28 | from .rfc6749.grant_types import ResourceOwnerPasswordCredentialsGrant 29 | from .rfc6749.grant_types import ClientCredentialsGrant 30 | from .rfc6749.grant_types import RefreshTokenGrant 31 | from .rfc6749.request_validator import RequestValidator 32 | from .rfc6749.tokens import BearerToken 33 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming and providing OAuth 2.0 RFC6749. 10 | """ 11 | import functools 12 | 13 | from oauthlib.common import log 14 | from .errors import TemporarilyUnavailableError, ServerError 15 | from .errors import FatalClientError, OAuth2Error 16 | 17 | 18 | class BaseEndpoint(object): 19 | def __init__(self): 20 | self._available = True 21 | self._catch_errors = False 22 | 23 | @property 24 | def available(self): 25 | return self._available 26 | 27 | @available.setter 28 | def available(self, available): 29 | self._available = available 30 | 31 | @property 32 | def catch_errors(self): 33 | return self._catch_errors 34 | 35 | @catch_errors.setter 36 | def catch_errors(self, catch_errors): 37 | self._catch_errors = catch_errors 38 | 39 | 40 | def catch_errors_and_unavailability(f): 41 | @functools.wraps(f) 42 | def wrapper(endpoint, uri, *args, **kwargs): 43 | if not endpoint.available: 44 | e = TemporarilyUnavailableError() 45 | log.info('Endpoint unavailable, ignoring request %s.' % uri) 46 | return None, {}, e.json, 503 47 | 48 | if endpoint.catch_errors: 49 | try: 50 | return f(endpoint, uri, *args, **kwargs) 51 | except OAuth2Error: 52 | raise 53 | except FatalClientError: 54 | raise 55 | except Exception as e: 56 | error = ServerError() 57 | log.warning('Exception caught while processing request, %s.' % e) 58 | return None, {}, error.json, 500 59 | else: 60 | return f(endpoint, uri, *args, **kwargs) 61 | return wrapper 62 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/clients/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming OAuth 2.0 RFC6749. 10 | """ 11 | 12 | from .base import * 13 | from .web_application import WebApplicationClient 14 | from .mobile_application import MobileApplicationClient 15 | from .legacy_application import LegacyApplicationClient 16 | from .backend_application import BackendApplicationClient 17 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming and providing OAuth 2.0 RFC6749. 10 | """ 11 | 12 | from .authorization import AuthorizationEndpoint 13 | from .token import TokenEndpoint 14 | from .resource import ResourceEndpoint 15 | from .pre_configured import Server 16 | from .pre_configured import WebApplicationServer 17 | from .pre_configured import MobileApplicationServer 18 | from .pre_configured import LegacyApplicationServer 19 | from .pre_configured import BackendApplicationServer 20 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/endpoints/authorization.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming and providing OAuth 2.0 RFC6749. 10 | """ 11 | 12 | from oauthlib.common import Request, log 13 | 14 | from .base import BaseEndpoint, catch_errors_and_unavailability 15 | 16 | 17 | class AuthorizationEndpoint(BaseEndpoint): 18 | """Authorization endpoint - used by the client to obtain authorization 19 | from the resource owner via user-agent redirection. 20 | 21 | The authorization endpoint is used to interact with the resource 22 | owner and obtain an authorization grant. The authorization server 23 | MUST first verify the identity of the resource owner. The way in 24 | which the authorization server authenticates the resource owner (e.g. 25 | username and password login, session cookies) is beyond the scope of 26 | this specification. 27 | 28 | The endpoint URI MAY include an "application/x-www-form-urlencoded" 29 | formatted (per `Appendix B`_) query component, 30 | which MUST be retained when adding additional query parameters. The 31 | endpoint URI MUST NOT include a fragment component:: 32 | 33 | https://example.com/path?query=component # OK 34 | https://example.com/path?query=component#fragment # Not OK 35 | 36 | Since requests to the authorization endpoint result in user 37 | authentication and the transmission of clear-text credentials (in the 38 | HTTP response), the authorization server MUST require the use of TLS 39 | as described in Section 1.6 when sending requests to the 40 | authorization endpoint:: 41 | 42 | # We will deny any request which URI schema is not with https 43 | 44 | The authorization server MUST support the use of the HTTP "GET" 45 | method [RFC2616] for the authorization endpoint, and MAY support the 46 | use of the "POST" method as well:: 47 | 48 | # HTTP method is currently not enforced 49 | 50 | Parameters sent without a value MUST be treated as if they were 51 | omitted from the request. The authorization server MUST ignore 52 | unrecognized request parameters. Request and response parameters 53 | MUST NOT be included more than once:: 54 | 55 | # Enforced through the design of oauthlib.common.Request 56 | 57 | .. _`Appendix B`: http://tools.ietf.org/html/rfc6749#appendix-B 58 | """ 59 | 60 | def __init__(self, default_response_type, default_token_type, 61 | response_types): 62 | BaseEndpoint.__init__(self) 63 | self._response_types = response_types 64 | self._default_response_type = default_response_type 65 | self._default_token_type = default_token_type 66 | 67 | @property 68 | def response_types(self): 69 | return self._response_types 70 | 71 | @property 72 | def default_response_type(self): 73 | return self._default_response_type 74 | 75 | @property 76 | def default_response_type_handler(self): 77 | return self.response_types.get(self.default_response_type) 78 | 79 | @property 80 | def default_token_type(self): 81 | return self._default_token_type 82 | 83 | @catch_errors_and_unavailability 84 | def create_authorization_response(self, uri, http_method='GET', body=None, 85 | headers=None, scopes=None, credentials=None): 86 | """Extract response_type and route to the designated handler.""" 87 | request = Request(uri, http_method=http_method, body=body, headers=headers) 88 | request.scopes = scopes 89 | # TODO: decide whether this should be a required argument 90 | request.user = None # TODO: explain this in docs 91 | for k, v in (credentials or {}).items(): 92 | setattr(request, k, v) 93 | response_type_handler = self.response_types.get( 94 | request.response_type, self.default_response_type_handler) 95 | log.debug('Dispatching response_type %s request to %r.', 96 | request.response_type, response_type_handler) 97 | return response_type_handler.create_authorization_response( 98 | request, self.default_token_type) 99 | 100 | @catch_errors_and_unavailability 101 | def validate_authorization_request(self, uri, http_method='GET', body=None, 102 | headers=None): 103 | """Extract response_type and route to the designated handler.""" 104 | request = Request(uri, http_method=http_method, body=body, headers=headers) 105 | request.scopes = None 106 | response_type_handler = self.response_types.get( 107 | request.response_type, self.default_response_type_handler) 108 | return response_type_handler.validate_authorization_request(request) 109 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/endpoints/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming and providing OAuth 2.0 RFC6749. 10 | """ 11 | import functools 12 | 13 | from oauthlib.common import log 14 | 15 | from ..errors import TemporarilyUnavailableError, ServerError 16 | from ..errors import FatalClientError, OAuth2Error 17 | 18 | 19 | class BaseEndpoint(object): 20 | def __init__(self): 21 | self._available = True 22 | self._catch_errors = False 23 | 24 | @property 25 | def available(self): 26 | return self._available 27 | 28 | @available.setter 29 | def available(self, available): 30 | self._available = available 31 | 32 | @property 33 | def catch_errors(self): 34 | return self._catch_errors 35 | 36 | @catch_errors.setter 37 | def catch_errors(self, catch_errors): 38 | self._catch_errors = catch_errors 39 | 40 | 41 | def catch_errors_and_unavailability(f): 42 | @functools.wraps(f) 43 | def wrapper(endpoint, uri, *args, **kwargs): 44 | if not endpoint.available: 45 | e = TemporarilyUnavailableError() 46 | log.info('Endpoint unavailable, ignoring request %s.' % uri) 47 | return None, {}, e.json, 503 48 | 49 | if endpoint.catch_errors: 50 | try: 51 | return f(endpoint, uri, *args, **kwargs) 52 | except OAuth2Error: 53 | raise 54 | except FatalClientError: 55 | raise 56 | except Exception as e: 57 | error = ServerError() 58 | log.warning('Exception caught while processing request, %s.' % e) 59 | return None, {}, error.json, 500 60 | else: 61 | return f(endpoint, uri, *args, **kwargs) 62 | return wrapper 63 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/endpoints/resource.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming and providing OAuth 2.0 RFC6749. 10 | """ 11 | from oauthlib.common import Request, log 12 | 13 | from .base import BaseEndpoint, catch_errors_and_unavailability 14 | 15 | 16 | class ResourceEndpoint(BaseEndpoint): 17 | """Authorizes access to protected resources. 18 | 19 | The client accesses protected resources by presenting the access 20 | token to the resource server. The resource server MUST validate the 21 | access token and ensure that it has not expired and that its scope 22 | covers the requested resource. The methods used by the resource 23 | server to validate the access token (as well as any error responses) 24 | are beyond the scope of this specification but generally involve an 25 | interaction or coordination between the resource server and the 26 | authorization server:: 27 | 28 | # For most cases, returning a 403 should suffice. 29 | 30 | The method in which the client utilizes the access token to 31 | authenticate with the resource server depends on the type of access 32 | token issued by the authorization server. Typically, it involves 33 | using the HTTP "Authorization" request header field [RFC2617] with an 34 | authentication scheme defined by the specification of the access 35 | token type used, such as [RFC6750]:: 36 | 37 | # Access tokens may also be provided in query and body 38 | https://example.com/protected?access_token=kjfch2345sdf # Query 39 | access_token=sdf23409df # Body 40 | """ 41 | def __init__(self, default_token, token_types): 42 | BaseEndpoint.__init__(self) 43 | self._tokens = token_types 44 | self._default_token = default_token 45 | 46 | @property 47 | def default_token(self): 48 | return self._default_token 49 | 50 | @property 51 | def default_token_type_handler(self): 52 | return self.tokens.get(self.default_token) 53 | 54 | @property 55 | def tokens(self): 56 | return self._tokens 57 | 58 | @catch_errors_and_unavailability 59 | def verify_request(self, uri, http_method='GET', body=None, headers=None, 60 | scopes=None): 61 | """Validate client, code etc, return body + headers""" 62 | request = Request(uri, http_method, body, headers) 63 | request.token_type = self.find_token_type(request) 64 | request.scopes = scopes 65 | token_type_handler = self.tokens.get(request.token_type, 66 | self.default_token_type_handler) 67 | log.debug('Dispatching token_type %s request to %r.', 68 | request.token_type, token_type_handler) 69 | return token_type_handler.validate_request(request), request 70 | 71 | def find_token_type(self, request): 72 | """Token type identification. 73 | 74 | RFC 6749 does not provide a method for easily differentiating between 75 | different token types during protected resource access. We estimate 76 | the most likely token type (if any) by asking each known token type 77 | to give an estimation based on the request. 78 | """ 79 | estimates = sorted(((t.estimate_type(request), n) for n, t in self.tokens.items())) 80 | return estimates[0][1] if len(estimates) else None 81 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/endpoints/token.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.oauth2.rfc6749 6 | ~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | This module is an implementation of various logic needed 9 | for consuming and providing OAuth 2.0 RFC6749. 10 | """ 11 | from oauthlib.common import Request, log 12 | 13 | from .base import BaseEndpoint, catch_errors_and_unavailability 14 | 15 | 16 | class TokenEndpoint(BaseEndpoint): 17 | """Token issuing endpoint. 18 | 19 | The token endpoint is used by the client to obtain an access token by 20 | presenting its authorization grant or refresh token. The token 21 | endpoint is used with every authorization grant except for the 22 | implicit grant type (since an access token is issued directly). 23 | 24 | The means through which the client obtains the location of the token 25 | endpoint are beyond the scope of this specification, but the location 26 | is typically provided in the service documentation. 27 | 28 | The endpoint URI MAY include an "application/x-www-form-urlencoded" 29 | formatted (per `Appendix B`_) query component, 30 | which MUST be retained when adding additional query parameters. The 31 | endpoint URI MUST NOT include a fragment component:: 32 | 33 | https://example.com/path?query=component # OK 34 | https://example.com/path?query=component#fragment # Not OK 35 | 36 | Since requests to the authorization endpoint result in user 37 | Since requests to the token endpoint result in the transmission of 38 | clear-text credentials (in the HTTP request and response), the 39 | authorization server MUST require the use of TLS as described in 40 | Section 1.6 when sending requests to the token endpoint:: 41 | 42 | # We will deny any request which URI schema is not with https 43 | 44 | The client MUST use the HTTP "POST" method when making access token 45 | requests:: 46 | 47 | # HTTP method is currently not enforced 48 | 49 | Parameters sent without a value MUST be treated as if they were 50 | omitted from the request. The authorization server MUST ignore 51 | unrecognized request parameters. Request and response parameters 52 | MUST NOT be included more than once:: 53 | 54 | # Delegated to each grant type. 55 | 56 | .. _`Appendix B`: http://tools.ietf.org/html/rfc6749#appendix-B 57 | """ 58 | 59 | def __init__(self, default_grant_type, default_token_type, grant_types): 60 | BaseEndpoint.__init__(self) 61 | self._grant_types = grant_types 62 | self._default_token_type = default_token_type 63 | self._default_grant_type = default_grant_type 64 | 65 | @property 66 | def grant_types(self): 67 | return self._grant_types 68 | 69 | @property 70 | def default_grant_type(self): 71 | return self._default_grant_type 72 | 73 | @property 74 | def default_grant_type_handler(self): 75 | return self.grant_types.get(self.default_grant_type) 76 | 77 | @property 78 | def default_token_type(self): 79 | return self._default_token_type 80 | 81 | @catch_errors_and_unavailability 82 | def create_token_response(self, uri, http_method='GET', body=None, 83 | headers=None, credentials=None): 84 | """Extract grant_type and route to the designated handler.""" 85 | request = Request(uri, http_method=http_method, body=body, headers=headers) 86 | request.scopes = None 87 | request.extra_credentials = credentials 88 | grant_type_handler = self.grant_types.get(request.grant_type, 89 | self.default_grant_type_handler) 90 | log.debug('Dispatching grant_type %s request to %r.', 91 | request.grant_type, grant_type_handler) 92 | return grant_type_handler.create_token_response( 93 | request, self.default_token_type) 94 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/grant_types/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | oauthlib.oauth2.rfc6749.grant_types 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | """ 6 | from __future__ import unicode_literals, absolute_import 7 | 8 | from .authorization_code import AuthorizationCodeGrant 9 | from .implicit import ImplicitGrant 10 | from .resource_owner_password_credentials import ResourceOwnerPasswordCredentialsGrant 11 | from .client_credentials import ClientCredentialsGrant 12 | from .refresh_token import RefreshTokenGrant 13 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/grant_types/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | oauthlib.oauth2.rfc6749.grant_types 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | """ 6 | from __future__ import unicode_literals, absolute_import 7 | 8 | from oauthlib.common import log 9 | from oauthlib.oauth2.rfc6749 import errors, utils 10 | 11 | 12 | class GrantTypeBase(object): 13 | error_uri = None 14 | request_validator = None 15 | 16 | def create_authorization_response(self, request, token_handler): 17 | raise NotImplementedError('Subclasses must implement this method.') 18 | 19 | def create_token_response(self, request, token_handler): 20 | raise NotImplementedError('Subclasses must implement this method.') 21 | 22 | def validate_grant_type(self, request): 23 | if not self.request_validator.validate_grant_type(request.client_id, 24 | request.grant_type, request.client, request): 25 | log.debug('Unauthorized from %r (%r) access to grant type %s.', 26 | request.client_id, request.client, request.grant_type) 27 | raise errors.UnauthorizedClientError(request=request) 28 | 29 | def validate_scopes(self, request): 30 | if not request.scopes: 31 | request.scopes = utils.scope_to_list(request.scope) or utils.scope_to_list( 32 | self.request_validator.get_default_scopes(request.client_id, request)) 33 | log.debug('Validating access to scopes %r for client %r (%r).', 34 | request.scopes, request.client_id, request.client) 35 | if not self.request_validator.validate_scopes(request.client_id, 36 | request.scopes, request.client, request): 37 | raise errors.InvalidScopeError(state=request.state, request=request) 38 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/grant_types/client_credentials.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | oauthlib.oauth2.rfc6749.grant_types 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | """ 6 | from __future__ import unicode_literals, absolute_import 7 | import json 8 | from oauthlib.common import log 9 | 10 | from .base import GrantTypeBase 11 | from .. import errors 12 | from ..request_validator import RequestValidator 13 | 14 | 15 | class ClientCredentialsGrant(GrantTypeBase): 16 | """`Client Credentials Grant`_ 17 | 18 | The client can request an access token using only its client 19 | credentials (or other supported means of authentication) when the 20 | client is requesting access to the protected resources under its 21 | control, or those of another resource owner that have been previously 22 | arranged with the authorization server (the method of which is beyond 23 | the scope of this specification). 24 | 25 | The client credentials grant type MUST only be used by confidential 26 | clients:: 27 | 28 | +---------+ +---------------+ 29 | : : : : 30 | : :>-- A - Client Authentication --->: Authorization : 31 | : Client : : Server : 32 | : :<-- B ---- Access Token ---------<: : 33 | : : : : 34 | +---------+ +---------------+ 35 | 36 | Figure 6: Client Credentials Flow 37 | 38 | The flow illustrated in Figure 6 includes the following steps: 39 | 40 | (A) The client authenticates with the authorization server and 41 | requests an access token from the token endpoint. 42 | 43 | (B) The authorization server authenticates the client, and if valid, 44 | issues an access token. 45 | 46 | .. _`Client Credentials Grant`: http://tools.ietf.org/html/rfc6749#section-4.4 47 | """ 48 | 49 | def __init__(self, request_validator=None): 50 | self.request_validator = request_validator or RequestValidator() 51 | 52 | def create_token_response(self, request, token_handler): 53 | """Return token or error in JSON format. 54 | 55 | If the access token request is valid and authorized, the 56 | authorization server issues an access token as described in 57 | `Section 5.1`_. A refresh token SHOULD NOT be included. If the request 58 | failed client authentication or is invalid, the authorization server 59 | returns an error response as described in `Section 5.2`_. 60 | 61 | .. _`Section 5.1`: http://tools.ietf.org/html/rfc6749#section-5.1 62 | .. _`Section 5.2`: http://tools.ietf.org/html/rfc6749#section-5.2 63 | """ 64 | try: 65 | log.debug('Validating access token request, %r.', request) 66 | self.validate_token_request(request) 67 | except errors.OAuth2Error as e: 68 | log.debug('Client error in token request. %s.', e) 69 | return None, {}, e.json, e.status_code 70 | 71 | token = token_handler.create_token(request, refresh_token=False) 72 | log.debug('Issuing token to client id %r (%r), %r.', 73 | request.client_id, request.client, token) 74 | return None, {}, json.dumps(token), 200 75 | 76 | def validate_token_request(self, request): 77 | if not getattr(request, 'grant_type'): 78 | raise errors.InvalidRequestError('Request is missing grant type.', 79 | request=request) 80 | 81 | if not request.grant_type == 'client_credentials': 82 | raise errors.UnsupportedGrantTypeError(request=request) 83 | 84 | for param in ('grant_type', 'scope'): 85 | if param in request.duplicate_params: 86 | raise errors.InvalidRequestError(state=request.state, 87 | description='Duplicate %s parameter.' % param, 88 | request=request) 89 | 90 | log.debug('Authenticating client, %r.', request) 91 | if not self.request_validator.authenticate_client(request): 92 | log.debug('Client authentication failed, %r.', request) 93 | raise errors.InvalidClientError(request=request) 94 | else: 95 | if not hasattr(request.client, 'client_id'): 96 | raise NotImplementedError('Authenticate client must set the ' 97 | 'request.client.client_id attribute ' 98 | 'in authenticate_client.') 99 | # Ensure client is authorized use of this grant type 100 | self.validate_grant_type(request) 101 | 102 | log.debug('Authorizing access to user %r.', request.user) 103 | request.client_id = request.client_id or request.client.client_id 104 | self.validate_scopes(request) 105 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/grant_types/refresh_token.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | oauthlib.oauth2.rfc6749.grant_types 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | """ 6 | from __future__ import unicode_literals, absolute_import 7 | import json 8 | 9 | from oauthlib.common import log 10 | 11 | from .base import GrantTypeBase 12 | from .. import errors, utils 13 | from ..request_validator import RequestValidator 14 | 15 | 16 | class RefreshTokenGrant(GrantTypeBase): 17 | """`Refresh token grant`_ 18 | 19 | .. _`Refresh token grant`: http://tools.ietf.org/html/rfc6749#section-6 20 | """ 21 | 22 | @property 23 | def issue_new_refresh_tokens(self): 24 | return True 25 | 26 | def __init__(self, request_validator=None, issue_new_refresh_tokens=True): 27 | self.request_validator = request_validator or RequestValidator() 28 | 29 | def create_token_response(self, request, token_handler): 30 | """Create a new access token from a refresh_token. 31 | 32 | If valid and authorized, the authorization server issues an access 33 | token as described in `Section 5.1`_. If the request failed 34 | verification or is invalid, the authorization server returns an error 35 | response as described in `Section 5.2`_. 36 | 37 | The authorization server MAY issue a new refresh token, in which case 38 | the client MUST discard the old refresh token and replace it with the 39 | new refresh token. The authorization server MAY revoke the old 40 | refresh token after issuing a new refresh token to the client. If a 41 | new refresh token is issued, the refresh token scope MUST be 42 | identical to that of the refresh token included by the client in the 43 | request. 44 | 45 | .. _`Section 5.1`: http://tools.ietf.org/html/rfc6749#section-5.1 46 | .. _`Section 5.2`: http://tools.ietf.org/html/rfc6749#section-5.2 47 | """ 48 | headers = { 49 | 'Content-Type': 'application/json;charset=UTF-8', 50 | 'Cache-Control': 'no-store', 51 | 'Pragma': 'no-cache', 52 | } 53 | try: 54 | log.debug('Validating refresh token request, %r.', request) 55 | self.validate_token_request(request) 56 | except errors.OAuth2Error as e: 57 | return None, headers, e.json, e.status_code 58 | 59 | token = token_handler.create_token(request, 60 | refresh_token=self.issue_new_refresh_tokens) 61 | log.debug('Issuing new token to client id %r (%r), %r.', 62 | request.client_id, request.client, token) 63 | return None, headers, json.dumps(token), 200 64 | 65 | def validate_token_request(self, request): 66 | # REQUIRED. Value MUST be set to "refresh_token". 67 | if request.grant_type != 'refresh_token': 68 | raise errors.UnsupportedGrantTypeError(request=request) 69 | 70 | if request.refresh_token is None: 71 | raise errors.InvalidRequestError( 72 | description='Missing refresh token parameter.', 73 | request=request) 74 | 75 | # Because refresh tokens are typically long-lasting credentials used to 76 | # request additional access tokens, the refresh token is bound to the 77 | # client to which it was issued. If the client type is confidential or 78 | # the client was issued client credentials (or assigned other 79 | # authentication requirements), the client MUST authenticate with the 80 | # authorization server as described in Section 3.2.1. 81 | # http://tools.ietf.org/html/rfc6749#section-3.2.1 82 | log.debug('Authenticating client, %r.', request) 83 | if not self.request_validator.authenticate_client(request): 84 | log.debug('Invalid client (%r), denying access.', request) 85 | raise errors.InvalidClientError(request=request, status_code=401) 86 | 87 | # Ensure client is authorized use of this grant type 88 | self.validate_grant_type(request) 89 | 90 | # REQUIRED. The refresh token issued to the client. 91 | log.debug('Validating refresh token %s for client %r.', 92 | request.refresh_token, request.client) 93 | if not self.request_validator.validate_refresh_token( 94 | request.refresh_token, request.client, request): 95 | log.debug('Invalid refresh token, %s, for client %r.', 96 | request.refresh_token, request.client) 97 | raise errors.InvalidGrantError(request=request) 98 | 99 | original_scopes = utils.scope_to_list( 100 | self.request_validator.get_original_scopes( 101 | request.refresh_token, request)) 102 | 103 | if request.scope: 104 | request.scopes = utils.scope_to_list(request.scope) 105 | if not all((s in original_scopes for s in request.scopes)): 106 | log.debug('Refresh token %s lack requested scopes, %r.', 107 | request.refresh_token, request.scopes) 108 | raise errors.InvalidScopeError( 109 | state=request.state, request=request, status_code=401) 110 | else: 111 | request.scopes = original_scopes 112 | -------------------------------------------------------------------------------- /twitter2/bin/oauthlib/oauth2/rfc6749/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | """ 5 | oauthlib.utils 6 | ~~~~~~~~~~~~~~ 7 | 8 | This module contains utility methods used by various parts of the OAuth 2 spec. 9 | """ 10 | 11 | import os 12 | import datetime 13 | try: 14 | from urllib import quote 15 | except ImportError: 16 | from urllib.parse import quote 17 | try: 18 | from urlparse import urlparse 19 | except ImportError: 20 | from urllib.parse import urlparse 21 | from oauthlib.common import unicode_type, urldecode 22 | 23 | 24 | def list_to_scope(scope): 25 | """Convert a list of scopes to a space separated string.""" 26 | if isinstance(scope, unicode_type) or scope is None: 27 | return scope 28 | elif isinstance(scope, list): 29 | return " ".join(scope) 30 | else: 31 | raise ValueError("Invalid scope, must be string or list.") 32 | 33 | 34 | def scope_to_list(scope): 35 | """Convert a space separated string to a list of scopes.""" 36 | if isinstance(scope, list) or scope is None: 37 | return scope 38 | else: 39 | return scope.split(" ") 40 | 41 | 42 | def params_from_uri(uri): 43 | params = dict(urldecode(urlparse(uri).query)) 44 | if 'scope' in params: 45 | params['scope'] = scope_to_list(params['scope']) 46 | return params 47 | 48 | 49 | def host_from_uri(uri): 50 | """Extract hostname and port from URI. 51 | 52 | Will use default port for HTTP and HTTPS if none is present in the URI. 53 | """ 54 | default_ports = { 55 | 'HTTP': '80', 56 | 'HTTPS': '443', 57 | } 58 | 59 | sch, netloc, path, par, query, fra = urlparse(uri) 60 | if ':' in netloc: 61 | netloc, port = netloc.split(':', 1) 62 | else: 63 | port = default_ports.get(sch.upper()) 64 | 65 | return netloc, port 66 | 67 | 68 | def escape(u): 69 | """Escape a string in an OAuth-compatible fashion. 70 | 71 | TODO: verify whether this can in fact be used for OAuth 2 72 | 73 | """ 74 | if not isinstance(u, unicode_type): 75 | raise ValueError('Only unicode objects are escapable.') 76 | return quote(u.encode('utf-8'), safe=b'~') 77 | 78 | 79 | def generate_age(issue_time): 80 | """Generate a age parameter for MAC authentication draft 00.""" 81 | td = datetime.datetime.now() - issue_time 82 | age = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 83 | return unicode_type(age) 84 | 85 | 86 | def is_secure_transport(uri): 87 | """Check if the uri is over ssl.""" 88 | if os.environ.get('DEBUG'): 89 | return True 90 | return uri.lower().startswith('https://') 91 | -------------------------------------------------------------------------------- /twitter2/bin/requests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # __ 4 | # /__) _ _ _ _ _/ _ 5 | # / ( (- (/ (/ (- _) / _) 6 | # / 7 | 8 | """ 9 | requests HTTP library 10 | ~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | Requests is an HTTP library, written in Python, for human beings. Basic GET 13 | usage: 14 | 15 | >>> import requests 16 | >>> r = requests.get('http://python.org') 17 | >>> r.status_code 18 | 200 19 | >>> 'Python is a programming language' in r.content 20 | True 21 | 22 | ... or POST: 23 | 24 | >>> payload = dict(key1='value1', key2='value2') 25 | >>> r = requests.post("http://httpbin.org/post", data=payload) 26 | >>> print r.text 27 | { 28 | ... 29 | "form": { 30 | "key2": "value2", 31 | "key1": "value1" 32 | }, 33 | ... 34 | } 35 | 36 | The other HTTP methods are supported - see `requests.api`. Full documentation 37 | is at . 38 | 39 | :copyright: (c) 2013 by Kenneth Reitz. 40 | :license: Apache 2.0, see LICENSE for more details. 41 | 42 | """ 43 | 44 | __title__ = 'requests' 45 | __version__ = '1.2.3' 46 | __build__ = 0x010203 47 | __author__ = 'Kenneth Reitz' 48 | __license__ = 'Apache 2.0' 49 | __copyright__ = 'Copyright 2013 Kenneth Reitz' 50 | 51 | # Attempt to enable urllib3's SNI support, if possible 52 | try: 53 | from requests.packages.urllib3.contrib import pyopenssl 54 | pyopenssl.inject_into_urllib3() 55 | except ImportError: 56 | pass 57 | 58 | from . import utils 59 | from .models import Request, Response, PreparedRequest 60 | from .api import request, get, head, post, patch, put, delete, options 61 | from .sessions import session, Session 62 | from .status_codes import codes 63 | from .exceptions import ( 64 | RequestException, Timeout, URLRequired, 65 | TooManyRedirects, HTTPError, ConnectionError 66 | ) 67 | 68 | # Set default logging handler to avoid "No handler found" warnings. 69 | import logging 70 | try: # Python 2.7+ 71 | from logging import NullHandler 72 | except ImportError: 73 | class NullHandler(logging.Handler): 74 | def emit(self, record): 75 | pass 76 | 77 | logging.getLogger(__name__).addHandler(NullHandler()) 78 | -------------------------------------------------------------------------------- /twitter2/bin/requests/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.api 5 | ~~~~~~~~~~~~ 6 | 7 | This module implements the Requests API. 8 | 9 | :copyright: (c) 2012 by Kenneth Reitz. 10 | :license: Apache2, see LICENSE for more details. 11 | 12 | """ 13 | 14 | from . import sessions 15 | 16 | 17 | def request(method, url, **kwargs): 18 | """Constructs and sends a :class:`Request `. 19 | Returns :class:`Response ` object. 20 | 21 | :param method: method for the new :class:`Request` object. 22 | :param url: URL for the new :class:`Request` object. 23 | :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. 24 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 25 | :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. 26 | :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. 27 | :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload. 28 | :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. 29 | :param timeout: (optional) Float describing the timeout of the request. 30 | :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. 31 | :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. 32 | :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. 33 | :param stream: (optional) if ``False``, the response content will be immediately downloaded. 34 | :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. 35 | 36 | Usage:: 37 | 38 | >>> import requests 39 | >>> req = requests.request('GET', 'http://httpbin.org/get') 40 | 41 | """ 42 | 43 | session = sessions.Session() 44 | return session.request(method=method, url=url, **kwargs) 45 | 46 | 47 | def get(url, **kwargs): 48 | """Sends a GET request. Returns :class:`Response` object. 49 | 50 | :param url: URL for the new :class:`Request` object. 51 | :param \*\*kwargs: Optional arguments that ``request`` takes. 52 | """ 53 | 54 | kwargs.setdefault('allow_redirects', True) 55 | return request('get', url, **kwargs) 56 | 57 | 58 | def options(url, **kwargs): 59 | """Sends a OPTIONS request. Returns :class:`Response` object. 60 | 61 | :param url: URL for the new :class:`Request` object. 62 | :param \*\*kwargs: Optional arguments that ``request`` takes. 63 | """ 64 | 65 | kwargs.setdefault('allow_redirects', True) 66 | return request('options', url, **kwargs) 67 | 68 | 69 | def head(url, **kwargs): 70 | """Sends a HEAD request. Returns :class:`Response` object. 71 | 72 | :param url: URL for the new :class:`Request` object. 73 | :param \*\*kwargs: Optional arguments that ``request`` takes. 74 | """ 75 | 76 | kwargs.setdefault('allow_redirects', False) 77 | return request('head', url, **kwargs) 78 | 79 | 80 | def post(url, data=None, **kwargs): 81 | """Sends a POST request. Returns :class:`Response` object. 82 | 83 | :param url: URL for the new :class:`Request` object. 84 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 85 | :param \*\*kwargs: Optional arguments that ``request`` takes. 86 | """ 87 | 88 | return request('post', url, data=data, **kwargs) 89 | 90 | 91 | def put(url, data=None, **kwargs): 92 | """Sends a PUT request. Returns :class:`Response` object. 93 | 94 | :param url: URL for the new :class:`Request` object. 95 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 96 | :param \*\*kwargs: Optional arguments that ``request`` takes. 97 | """ 98 | 99 | return request('put', url, data=data, **kwargs) 100 | 101 | 102 | def patch(url, data=None, **kwargs): 103 | """Sends a PATCH request. Returns :class:`Response` object. 104 | 105 | :param url: URL for the new :class:`Request` object. 106 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 107 | :param \*\*kwargs: Optional arguments that ``request`` takes. 108 | """ 109 | 110 | return request('patch', url, data=data, **kwargs) 111 | 112 | 113 | def delete(url, **kwargs): 114 | """Sends a DELETE request. Returns :class:`Response` object. 115 | 116 | :param url: URL for the new :class:`Request` object. 117 | :param \*\*kwargs: Optional arguments that ``request`` takes. 118 | """ 119 | 120 | return request('delete', url, **kwargs) 121 | -------------------------------------------------------------------------------- /twitter2/bin/requests/certs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | certs.py 6 | ~~~~~~~~ 7 | 8 | This module returns the preferred default CA certificate bundle. 9 | 10 | If you are packaging Requests, e.g., for a Linux distribution or a managed 11 | environment, you can change the definition of where() to return a separately 12 | packaged CA bundle. 13 | """ 14 | 15 | import os.path 16 | 17 | 18 | def where(): 19 | """Return the preferred certificate bundle.""" 20 | # vendored bundle inside Requests 21 | return os.path.join(os.path.dirname(__file__), 'cacert.pem') 22 | 23 | if __name__ == '__main__': 24 | print(where()) 25 | -------------------------------------------------------------------------------- /twitter2/bin/requests/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | pythoncompat 5 | """ 6 | 7 | from .packages import charade as chardet 8 | 9 | import sys 10 | 11 | # ------- 12 | # Pythons 13 | # ------- 14 | 15 | # Syntax sugar. 16 | _ver = sys.version_info 17 | 18 | #: Python 2.x? 19 | is_py2 = (_ver[0] == 2) 20 | 21 | #: Python 3.x? 22 | is_py3 = (_ver[0] == 3) 23 | 24 | #: Python 3.0.x 25 | is_py30 = (is_py3 and _ver[1] == 0) 26 | 27 | #: Python 3.1.x 28 | is_py31 = (is_py3 and _ver[1] == 1) 29 | 30 | #: Python 3.2.x 31 | is_py32 = (is_py3 and _ver[1] == 2) 32 | 33 | #: Python 3.3.x 34 | is_py33 = (is_py3 and _ver[1] == 3) 35 | 36 | #: Python 3.4.x 37 | is_py34 = (is_py3 and _ver[1] == 4) 38 | 39 | #: Python 2.7.x 40 | is_py27 = (is_py2 and _ver[1] == 7) 41 | 42 | #: Python 2.6.x 43 | is_py26 = (is_py2 and _ver[1] == 6) 44 | 45 | #: Python 2.5.x 46 | is_py25 = (is_py2 and _ver[1] == 5) 47 | 48 | #: Python 2.4.x 49 | is_py24 = (is_py2 and _ver[1] == 4) # I'm assuming this is not by choice. 50 | 51 | 52 | # --------- 53 | # Platforms 54 | # --------- 55 | 56 | 57 | # Syntax sugar. 58 | _ver = sys.version.lower() 59 | 60 | is_pypy = ('pypy' in _ver) 61 | is_jython = ('jython' in _ver) 62 | is_ironpython = ('iron' in _ver) 63 | 64 | # Assume CPython, if nothing else. 65 | is_cpython = not any((is_pypy, is_jython, is_ironpython)) 66 | 67 | # Windows-based system. 68 | is_windows = 'win32' in str(sys.platform).lower() 69 | 70 | # Standard Linux 2+ system. 71 | is_linux = ('linux' in str(sys.platform).lower()) 72 | is_osx = ('darwin' in str(sys.platform).lower()) 73 | is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess. 74 | is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess. 75 | 76 | try: 77 | import simplejson as json 78 | except ImportError: 79 | import json 80 | 81 | # --------- 82 | # Specifics 83 | # --------- 84 | 85 | if is_py2: 86 | from urllib import quote, unquote, quote_plus, unquote_plus, urlencode 87 | from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag 88 | from urllib2 import parse_http_list 89 | import cookielib 90 | from Cookie import Morsel 91 | from StringIO import StringIO 92 | from .packages.urllib3.packages.ordered_dict import OrderedDict 93 | 94 | builtin_str = str 95 | bytes = str 96 | str = unicode 97 | basestring = basestring 98 | numeric_types = (int, long, float) 99 | 100 | 101 | elif is_py3: 102 | from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag 103 | from urllib.request import parse_http_list 104 | from http import cookiejar as cookielib 105 | from http.cookies import Morsel 106 | from io import StringIO 107 | from collections import OrderedDict 108 | 109 | builtin_str = str 110 | str = str 111 | bytes = bytes 112 | basestring = (str, bytes) 113 | numeric_types = (int, float) 114 | -------------------------------------------------------------------------------- /twitter2/bin/requests/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.exceptions 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | This module contains the set of Requests' exceptions. 8 | 9 | """ 10 | 11 | 12 | class RequestException(RuntimeError): 13 | """There was an ambiguous exception that occurred while handling your 14 | request.""" 15 | 16 | 17 | class HTTPError(RequestException): 18 | """An HTTP error occurred.""" 19 | 20 | def __init__(self, *args, **kwargs): 21 | """ Initializes HTTPError with optional `response` object. """ 22 | self.response = kwargs.pop('response', None) 23 | super(HTTPError, self).__init__(*args, **kwargs) 24 | 25 | 26 | class ConnectionError(RequestException): 27 | """A Connection error occurred.""" 28 | 29 | 30 | class SSLError(ConnectionError): 31 | """An SSL error occurred.""" 32 | 33 | 34 | class Timeout(RequestException): 35 | """The request timed out.""" 36 | 37 | 38 | class URLRequired(RequestException): 39 | """A valid URL is required to make a request.""" 40 | 41 | 42 | class TooManyRedirects(RequestException): 43 | """Too many redirects.""" 44 | 45 | 46 | class MissingSchema(RequestException, ValueError): 47 | """The URL schema (e.g. http or https) is missing.""" 48 | 49 | 50 | class InvalidSchema(RequestException, ValueError): 51 | """See defaults.py for valid schemas.""" 52 | 53 | 54 | class InvalidURL(RequestException, ValueError): 55 | """ The URL provided was somehow invalid. """ 56 | -------------------------------------------------------------------------------- /twitter2/bin/requests/hooks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.hooks 5 | ~~~~~~~~~~~~~~ 6 | 7 | This module provides the capabilities for the Requests hooks system. 8 | 9 | Available hooks: 10 | 11 | ``response``: 12 | The response generated from a Request. 13 | 14 | """ 15 | 16 | 17 | HOOKS = ['response'] 18 | 19 | 20 | def default_hooks(): 21 | hooks = {} 22 | for event in HOOKS: 23 | hooks[event] = [] 24 | return hooks 25 | 26 | # TODO: response is the only one 27 | 28 | 29 | def dispatch_hook(key, hooks, hook_data, **kwargs): 30 | """Dispatches a hook dictionary on a given piece of data.""" 31 | 32 | hooks = hooks or dict() 33 | 34 | if key in hooks: 35 | hooks = hooks.get(key) 36 | 37 | if hasattr(hooks, '__call__'): 38 | hooks = [hooks] 39 | 40 | for hook in hooks: 41 | _hook_data = hook(hook_data, **kwargs) 42 | if _hook_data is not None: 43 | hook_data = _hook_data 44 | 45 | return hook_data 46 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import urllib3 4 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/__init__.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # This library is free software; you can redistribute it and/or 3 | # modify it under the terms of the GNU Lesser General Public 4 | # License as published by the Free Software Foundation; either 5 | # version 2.1 of the License, or (at your option) any later version. 6 | # 7 | # This library is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 | # Lesser General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU Lesser General Public 13 | # License along with this library; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 15 | # 02110-1301 USA 16 | ######################### END LICENSE BLOCK ######################### 17 | 18 | __version__ = "1.0.3" 19 | from sys import version_info 20 | 21 | 22 | def detect(aBuf): 23 | if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or 24 | (version_info >= (3, 0) and not isinstance(aBuf, bytes))): 25 | raise ValueError('Expected a bytes object, not a unicode object') 26 | 27 | from . import universaldetector 28 | u = universaldetector.UniversalDetector() 29 | u.reset() 30 | u.feed(aBuf) 31 | u.close() 32 | return u.result 33 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/big5prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import Big5DistributionAnalysis 31 | from .mbcssm import Big5SMModel 32 | 33 | 34 | class Big5Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(Big5SMModel) 38 | self._mDistributionAnalyzer = Big5DistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "Big5" 43 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/charsetgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | import sys 30 | from .charsetprober import CharSetProber 31 | 32 | 33 | class CharSetGroupProber(CharSetProber): 34 | def __init__(self): 35 | CharSetProber.__init__(self) 36 | self._mActiveNum = 0 37 | self._mProbers = [] 38 | self._mBestGuessProber = None 39 | 40 | def reset(self): 41 | CharSetProber.reset(self) 42 | self._mActiveNum = 0 43 | for prober in self._mProbers: 44 | if prober: 45 | prober.reset() 46 | prober.active = True 47 | self._mActiveNum += 1 48 | self._mBestGuessProber = None 49 | 50 | def get_charset_name(self): 51 | if not self._mBestGuessProber: 52 | self.get_confidence() 53 | if not self._mBestGuessProber: 54 | return None 55 | # self._mBestGuessProber = self._mProbers[0] 56 | return self._mBestGuessProber.get_charset_name() 57 | 58 | def feed(self, aBuf): 59 | for prober in self._mProbers: 60 | if not prober: 61 | continue 62 | if not prober.active: 63 | continue 64 | st = prober.feed(aBuf) 65 | if not st: 66 | continue 67 | if st == constants.eFoundIt: 68 | self._mBestGuessProber = prober 69 | return self.get_state() 70 | elif st == constants.eNotMe: 71 | prober.active = False 72 | self._mActiveNum -= 1 73 | if self._mActiveNum <= 0: 74 | self._mState = constants.eNotMe 75 | return self.get_state() 76 | return self.get_state() 77 | 78 | def get_confidence(self): 79 | st = self.get_state() 80 | if st == constants.eFoundIt: 81 | return 0.99 82 | elif st == constants.eNotMe: 83 | return 0.01 84 | bestConf = 0.0 85 | self._mBestGuessProber = None 86 | for prober in self._mProbers: 87 | if not prober: 88 | continue 89 | if not prober.active: 90 | if constants._debug: 91 | sys.stderr.write(prober.get_charset_name() 92 | + ' not active\n') 93 | continue 94 | cf = prober.get_confidence() 95 | if constants._debug: 96 | sys.stderr.write('%s confidence = %s\n' % 97 | (prober.get_charset_name(), cf)) 98 | if bestConf < cf: 99 | bestConf = cf 100 | self._mBestGuessProber = prober 101 | if not self._mBestGuessProber: 102 | return 0.0 103 | return bestConf 104 | # else: 105 | # self._mBestGuessProber = self._mProbers[0] 106 | # return self._mBestGuessProber.get_confidence() 107 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/charsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from . import constants 30 | import re 31 | 32 | 33 | class CharSetProber: 34 | def __init__(self): 35 | pass 36 | 37 | def reset(self): 38 | self._mState = constants.eDetecting 39 | 40 | def get_charset_name(self): 41 | return None 42 | 43 | def feed(self, aBuf): 44 | pass 45 | 46 | def get_state(self): 47 | return self._mState 48 | 49 | def get_confidence(self): 50 | return 0.0 51 | 52 | def filter_high_bit_only(self, aBuf): 53 | aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf) 54 | return aBuf 55 | 56 | def filter_without_english_letters(self, aBuf): 57 | aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf) 58 | return aBuf 59 | 60 | def filter_with_english_letters(self, aBuf): 61 | # TODO 62 | return aBuf 63 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/codingstatemachine.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .constants import eStart 29 | from .compat import wrap_ord 30 | 31 | 32 | class CodingStateMachine: 33 | def __init__(self, sm): 34 | self._mModel = sm 35 | self._mCurrentBytePos = 0 36 | self._mCurrentCharLen = 0 37 | self.reset() 38 | 39 | def reset(self): 40 | self._mCurrentState = eStart 41 | 42 | def next_state(self, c): 43 | # for each byte we get its class 44 | # if it is first byte, we also get byte length 45 | # PY3K: aBuf is a byte stream, so c is an int, not a byte 46 | byteCls = self._mModel['classTable'][wrap_ord(c)] 47 | if self._mCurrentState == eStart: 48 | self._mCurrentBytePos = 0 49 | self._mCurrentCharLen = self._mModel['charLenTable'][byteCls] 50 | # from byte's class and stateTable, we get its next state 51 | curr_state = (self._mCurrentState * self._mModel['classFactor'] 52 | + byteCls) 53 | self._mCurrentState = self._mModel['stateTable'][curr_state] 54 | self._mCurrentBytePos += 1 55 | return self._mCurrentState 56 | 57 | def get_current_charlen(self): 58 | return self._mCurrentCharLen 59 | 60 | def get_coding_state_machine(self): 61 | return self._mModel['name'] 62 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/compat.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # Contributor(s): 3 | # Ian Cordasco - port to Python 4 | # 5 | # This library is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU Lesser General Public 7 | # License as published by the Free Software Foundation; either 8 | # version 2.1 of the License, or (at your option) any later version. 9 | # 10 | # This library is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public 16 | # License along with this library; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 18 | # 02110-1301 USA 19 | ######################### END LICENSE BLOCK ######################### 20 | 21 | import sys 22 | 23 | 24 | if sys.version_info < (3, 0): 25 | base_str = (str, unicode) 26 | else: 27 | base_str = (bytes, str) 28 | 29 | 30 | def wrap_ord(a): 31 | if sys.version_info < (3, 0) and isinstance(a, base_str): 32 | return ord(a) 33 | else: 34 | return a 35 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/constants.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | _debug = 0 30 | 31 | eDetecting = 0 32 | eFoundIt = 1 33 | eNotMe = 2 34 | 35 | eStart = 0 36 | eError = 1 37 | eItsMe = 2 38 | 39 | SHORTCUT_THRESHOLD = 0.95 40 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/cp949prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import CP949SMModel 32 | 33 | 34 | class CP949Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(CP949SMModel) 38 | # NOTE: CP949 is a superset of EUC-KR, so the distribution should be 39 | # not different. 40 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 41 | self.reset() 42 | 43 | def get_charset_name(self): 44 | return "CP949" 45 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/escprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, 30 | ISO2022KRSMModel) 31 | from .charsetprober import CharSetProber 32 | from .codingstatemachine import CodingStateMachine 33 | from .compat import wrap_ord 34 | 35 | 36 | class EscCharSetProber(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = [ 40 | CodingStateMachine(HZSMModel), 41 | CodingStateMachine(ISO2022CNSMModel), 42 | CodingStateMachine(ISO2022JPSMModel), 43 | CodingStateMachine(ISO2022KRSMModel) 44 | ] 45 | self.reset() 46 | 47 | def reset(self): 48 | CharSetProber.reset(self) 49 | for codingSM in self._mCodingSM: 50 | if not codingSM: 51 | continue 52 | codingSM.active = True 53 | codingSM.reset() 54 | self._mActiveSM = len(self._mCodingSM) 55 | self._mDetectedCharset = None 56 | 57 | def get_charset_name(self): 58 | return self._mDetectedCharset 59 | 60 | def get_confidence(self): 61 | if self._mDetectedCharset: 62 | return 0.99 63 | else: 64 | return 0.00 65 | 66 | def feed(self, aBuf): 67 | for c in aBuf: 68 | # PY3K: aBuf is a byte array, so c is an int, not a byte 69 | for codingSM in self._mCodingSM: 70 | if not codingSM: 71 | continue 72 | if not codingSM.active: 73 | continue 74 | codingState = codingSM.next_state(wrap_ord(c)) 75 | if codingState == constants.eError: 76 | codingSM.active = False 77 | self._mActiveSM -= 1 78 | if self._mActiveSM <= 0: 79 | self._mState = constants.eNotMe 80 | return self.get_state() 81 | elif codingState == constants.eItsMe: 82 | self._mState = constants.eFoundIt 83 | self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8 84 | return self.get_state() 85 | 86 | return self.get_state() 87 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/eucjpprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from . import constants 30 | from .mbcharsetprober import MultiByteCharSetProber 31 | from .codingstatemachine import CodingStateMachine 32 | from .chardistribution import EUCJPDistributionAnalysis 33 | from .jpcntx import EUCJPContextAnalysis 34 | from .mbcssm import EUCJPSMModel 35 | 36 | 37 | class EUCJPProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(EUCJPSMModel) 41 | self._mDistributionAnalyzer = EUCJPDistributionAnalysis() 42 | self._mContextAnalyzer = EUCJPContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "EUC-JP" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mContextAnalyzer.feed(self._mLastChar, charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen) 75 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 76 | charLen) 77 | 78 | self._mLastChar[0] = aBuf[aLen - 1] 79 | 80 | if self.get_state() == constants.eDetecting: 81 | if (self._mContextAnalyzer.got_enough_data() and 82 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 83 | self._mState = constants.eFoundIt 84 | 85 | return self.get_state() 86 | 87 | def get_confidence(self): 88 | contxtCf = self._mContextAnalyzer.get_confidence() 89 | distribCf = self._mDistributionAnalyzer.get_confidence() 90 | return max(contxtCf, distribCf) 91 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/euckrprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import EUCKRSMModel 32 | 33 | 34 | class EUCKRProber(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(EUCKRSMModel) 38 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "EUC-KR" 43 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/euctwprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCTWDistributionAnalysis 31 | from .mbcssm import EUCTWSMModel 32 | 33 | class EUCTWProber(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(EUCTWSMModel) 37 | self._mDistributionAnalyzer = EUCTWDistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "EUC-TW" 42 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/gb2312prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import GB2312DistributionAnalysis 31 | from .mbcssm import GB2312SMModel 32 | 33 | class GB2312Prober(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(GB2312SMModel) 37 | self._mDistributionAnalyzer = GB2312DistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "GB2312" 42 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/mbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | import sys 31 | from . import constants 32 | from .charsetprober import CharSetProber 33 | 34 | 35 | class MultiByteCharSetProber(CharSetProber): 36 | def __init__(self): 37 | CharSetProber.__init__(self) 38 | self._mDistributionAnalyzer = None 39 | self._mCodingSM = None 40 | self._mLastChar = [0, 0] 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | if self._mCodingSM: 45 | self._mCodingSM.reset() 46 | if self._mDistributionAnalyzer: 47 | self._mDistributionAnalyzer.reset() 48 | self._mLastChar = [0, 0] 49 | 50 | def get_charset_name(self): 51 | pass 52 | 53 | def feed(self, aBuf): 54 | aLen = len(aBuf) 55 | for i in range(0, aLen): 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 72 | else: 73 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 74 | charLen) 75 | 76 | self._mLastChar[0] = aBuf[aLen - 1] 77 | 78 | if self.get_state() == constants.eDetecting: 79 | if (self._mDistributionAnalyzer.got_enough_data() and 80 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 81 | self._mState = constants.eFoundIt 82 | 83 | return self.get_state() 84 | 85 | def get_confidence(self): 86 | return self._mDistributionAnalyzer.get_confidence() 87 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/mbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | from .charsetgroupprober import CharSetGroupProber 31 | from .utf8prober import UTF8Prober 32 | from .sjisprober import SJISProber 33 | from .eucjpprober import EUCJPProber 34 | from .gb2312prober import GB2312Prober 35 | from .euckrprober import EUCKRProber 36 | from .cp949prober import CP949Prober 37 | from .big5prober import Big5Prober 38 | from .euctwprober import EUCTWProber 39 | 40 | 41 | class MBCSGroupProber(CharSetGroupProber): 42 | def __init__(self): 43 | CharSetGroupProber.__init__(self) 44 | self._mProbers = [ 45 | UTF8Prober(), 46 | SJISProber(), 47 | EUCJPProber(), 48 | GB2312Prober(), 49 | EUCKRProber(), 50 | CP949Prober(), 51 | Big5Prober(), 52 | EUCTWProber() 53 | ] 54 | self.reset() 55 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/sbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | import sys 30 | from . import constants 31 | from .charsetprober import CharSetProber 32 | from .compat import wrap_ord 33 | 34 | SAMPLE_SIZE = 64 35 | SB_ENOUGH_REL_THRESHOLD = 1024 36 | POSITIVE_SHORTCUT_THRESHOLD = 0.95 37 | NEGATIVE_SHORTCUT_THRESHOLD = 0.05 38 | SYMBOL_CAT_ORDER = 250 39 | NUMBER_OF_SEQ_CAT = 4 40 | POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1 41 | #NEGATIVE_CAT = 0 42 | 43 | 44 | class SingleByteCharSetProber(CharSetProber): 45 | def __init__(self, model, reversed=False, nameProber=None): 46 | CharSetProber.__init__(self) 47 | self._mModel = model 48 | # TRUE if we need to reverse every pair in the model lookup 49 | self._mReversed = reversed 50 | # Optional auxiliary prober for name decision 51 | self._mNameProber = nameProber 52 | self.reset() 53 | 54 | def reset(self): 55 | CharSetProber.reset(self) 56 | # char order of last character 57 | self._mLastOrder = 255 58 | self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT 59 | self._mTotalSeqs = 0 60 | self._mTotalChar = 0 61 | # characters that fall in our sampling range 62 | self._mFreqChar = 0 63 | 64 | def get_charset_name(self): 65 | if self._mNameProber: 66 | return self._mNameProber.get_charset_name() 67 | else: 68 | return self._mModel['charsetName'] 69 | 70 | def feed(self, aBuf): 71 | if not self._mModel['keepEnglishLetter']: 72 | aBuf = self.filter_without_english_letters(aBuf) 73 | aLen = len(aBuf) 74 | if not aLen: 75 | return self.get_state() 76 | for c in aBuf: 77 | order = self._mModel['charToOrderMap'][wrap_ord(c)] 78 | if order < SYMBOL_CAT_ORDER: 79 | self._mTotalChar += 1 80 | if order < SAMPLE_SIZE: 81 | self._mFreqChar += 1 82 | if self._mLastOrder < SAMPLE_SIZE: 83 | self._mTotalSeqs += 1 84 | if not self._mReversed: 85 | i = (self._mLastOrder * SAMPLE_SIZE) + order 86 | model = self._mModel['precedenceMatrix'][i] 87 | else: # reverse the order of the letters in the lookup 88 | i = (order * SAMPLE_SIZE) + self._mLastOrder 89 | model = self._mModel['precedenceMatrix'][i] 90 | self._mSeqCounters[model] += 1 91 | self._mLastOrder = order 92 | 93 | if self.get_state() == constants.eDetecting: 94 | if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD: 95 | cf = self.get_confidence() 96 | if cf > POSITIVE_SHORTCUT_THRESHOLD: 97 | if constants._debug: 98 | sys.stderr.write('%s confidence = %s, we have a' 99 | 'winner\n' % 100 | (self._mModel['charsetName'], cf)) 101 | self._mState = constants.eFoundIt 102 | elif cf < NEGATIVE_SHORTCUT_THRESHOLD: 103 | if constants._debug: 104 | sys.stderr.write('%s confidence = %s, below negative' 105 | 'shortcut threshhold %s\n' % 106 | (self._mModel['charsetName'], cf, 107 | NEGATIVE_SHORTCUT_THRESHOLD)) 108 | self._mState = constants.eNotMe 109 | 110 | return self.get_state() 111 | 112 | def get_confidence(self): 113 | r = 0.01 114 | if self._mTotalSeqs > 0: 115 | r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs 116 | / self._mModel['mTypicalPositiveRatio']) 117 | r = r * self._mFreqChar / self._mTotalChar 118 | if r >= 1.0: 119 | r = 0.99 120 | return r 121 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/sbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetgroupprober import CharSetGroupProber 30 | from .sbcharsetprober import SingleByteCharSetProber 31 | from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel, 32 | Latin5CyrillicModel, MacCyrillicModel, 33 | Ibm866Model, Ibm855Model) 34 | from .langgreekmodel import Latin7GreekModel, Win1253GreekModel 35 | from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel 36 | from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel 37 | from .langthaimodel import TIS620ThaiModel 38 | from .langhebrewmodel import Win1255HebrewModel 39 | from .hebrewprober import HebrewProber 40 | 41 | 42 | class SBCSGroupProber(CharSetGroupProber): 43 | def __init__(self): 44 | CharSetGroupProber.__init__(self) 45 | self._mProbers = [ 46 | SingleByteCharSetProber(Win1251CyrillicModel), 47 | SingleByteCharSetProber(Koi8rModel), 48 | SingleByteCharSetProber(Latin5CyrillicModel), 49 | SingleByteCharSetProber(MacCyrillicModel), 50 | SingleByteCharSetProber(Ibm866Model), 51 | SingleByteCharSetProber(Ibm855Model), 52 | SingleByteCharSetProber(Latin7GreekModel), 53 | SingleByteCharSetProber(Win1253GreekModel), 54 | SingleByteCharSetProber(Latin5BulgarianModel), 55 | SingleByteCharSetProber(Win1251BulgarianModel), 56 | SingleByteCharSetProber(Latin2HungarianModel), 57 | SingleByteCharSetProber(Win1250HungarianModel), 58 | SingleByteCharSetProber(TIS620ThaiModel), 59 | ] 60 | hebrewProber = HebrewProber() 61 | logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, 62 | False, hebrewProber) 63 | visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True, 64 | hebrewProber) 65 | hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber) 66 | self._mProbers.extend([hebrewProber, logicalHebrewProber, 67 | visualHebrewProber]) 68 | 69 | self.reset() 70 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/sjisprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from .mbcharsetprober import MultiByteCharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .chardistribution import SJISDistributionAnalysis 32 | from .jpcntx import SJISContextAnalysis 33 | from .mbcssm import SJISSMModel 34 | from . import constants 35 | 36 | 37 | class SJISProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(SJISSMModel) 41 | self._mDistributionAnalyzer = SJISDistributionAnalysis() 42 | self._mContextAnalyzer = SJISContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "SHIFT_JIS" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | codingState = self._mCodingSM.next_state(aBuf[i]) 56 | if codingState == constants.eError: 57 | if constants._debug: 58 | sys.stderr.write(self.get_charset_name() 59 | + ' prober hit error at byte ' + str(i) 60 | + '\n') 61 | self._mState = constants.eNotMe 62 | break 63 | elif codingState == constants.eItsMe: 64 | self._mState = constants.eFoundIt 65 | break 66 | elif codingState == constants.eStart: 67 | charLen = self._mCodingSM.get_current_charlen() 68 | if i == 0: 69 | self._mLastChar[1] = aBuf[0] 70 | self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:], 71 | charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3 75 | - charLen], charLen) 76 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 77 | charLen) 78 | 79 | self._mLastChar[0] = aBuf[aLen - 1] 80 | 81 | if self.get_state() == constants.eDetecting: 82 | if (self._mContextAnalyzer.got_enough_data() and 83 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 84 | self._mState = constants.eFoundIt 85 | 86 | return self.get_state() 87 | 88 | def get_confidence(self): 89 | contxtCf = self._mContextAnalyzer.get_confidence() 90 | distribCf = self._mDistributionAnalyzer.get_confidence() 91 | return max(contxtCf, distribCf) 92 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/charade/utf8prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .charsetprober import CharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .mbcssm import UTF8SMModel 32 | 33 | ONE_CHAR_PROB = 0.5 34 | 35 | 36 | class UTF8Prober(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = CodingStateMachine(UTF8SMModel) 40 | self.reset() 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | self._mCodingSM.reset() 45 | self._mNumOfMBChar = 0 46 | 47 | def get_charset_name(self): 48 | return "utf-8" 49 | 50 | def feed(self, aBuf): 51 | for c in aBuf: 52 | codingState = self._mCodingSM.next_state(c) 53 | if codingState == constants.eError: 54 | self._mState = constants.eNotMe 55 | break 56 | elif codingState == constants.eItsMe: 57 | self._mState = constants.eFoundIt 58 | break 59 | elif codingState == constants.eStart: 60 | if self._mCodingSM.get_current_charlen() >= 2: 61 | self._mNumOfMBChar += 1 62 | 63 | if self.get_state() == constants.eDetecting: 64 | if self.get_confidence() > constants.SHORTCUT_THRESHOLD: 65 | self._mState = constants.eFoundIt 66 | 67 | return self.get_state() 68 | 69 | def get_confidence(self): 70 | unlike = 0.99 71 | if self._mNumOfMBChar < 6: 72 | for i in range(0, self._mNumOfMBChar): 73 | unlike = unlike * ONE_CHAR_PROB 74 | return 1.0 - unlike 75 | else: 76 | return unlike 77 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/__init__.py: -------------------------------------------------------------------------------- 1 | # urllib3/__init__.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """ 8 | urllib3 - Thread-safe connection pooling and re-using. 9 | """ 10 | 11 | __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' 12 | __license__ = 'MIT' 13 | __version__ = 'dev' 14 | 15 | 16 | from .connectionpool import ( 17 | HTTPConnectionPool, 18 | HTTPSConnectionPool, 19 | connection_from_url 20 | ) 21 | 22 | from . import exceptions 23 | from .filepost import encode_multipart_formdata 24 | from .poolmanager import PoolManager, ProxyManager, proxy_from_url 25 | from .response import HTTPResponse 26 | from .util import make_headers, get_host 27 | 28 | 29 | # Set default logging handler to avoid "No handler found" warnings. 30 | import logging 31 | try: # Python 2.7+ 32 | from logging import NullHandler 33 | except ImportError: 34 | class NullHandler(logging.Handler): 35 | def emit(self, record): 36 | pass 37 | 38 | logging.getLogger(__name__).addHandler(NullHandler()) 39 | 40 | def add_stderr_logger(level=logging.DEBUG): 41 | """ 42 | Helper for quickly adding a StreamHandler to the logger. Useful for 43 | debugging. 44 | 45 | Returns the handler after adding it. 46 | """ 47 | # This method needs to be in this __init__.py to get the __name__ correct 48 | # even if urllib3 is vendored within another package. 49 | logger = logging.getLogger(__name__) 50 | handler = logging.StreamHandler() 51 | handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) 52 | logger.addHandler(handler) 53 | logger.setLevel(level) 54 | logger.debug('Added an stderr logging handler to logger: %s' % __name__) 55 | return handler 56 | 57 | # ... Clean up. 58 | del NullHandler 59 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/_collections.py: -------------------------------------------------------------------------------- 1 | # urllib3/_collections.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | from collections import MutableMapping 8 | from threading import Lock 9 | 10 | try: # Python 2.7+ 11 | from collections import OrderedDict 12 | except ImportError: 13 | from .packages.ordered_dict import OrderedDict 14 | 15 | 16 | __all__ = ['RecentlyUsedContainer'] 17 | 18 | 19 | _Null = object() 20 | 21 | 22 | class RecentlyUsedContainer(MutableMapping): 23 | """ 24 | Provides a thread-safe dict-like container which maintains up to 25 | ``maxsize`` keys while throwing away the least-recently-used keys beyond 26 | ``maxsize``. 27 | 28 | :param maxsize: 29 | Maximum number of recent elements to retain. 30 | 31 | :param dispose_func: 32 | Every time an item is evicted from the container, 33 | ``dispose_func(value)`` is called. Callback which will get called 34 | """ 35 | 36 | ContainerCls = OrderedDict 37 | 38 | def __init__(self, maxsize=10, dispose_func=None): 39 | self._maxsize = maxsize 40 | self.dispose_func = dispose_func 41 | 42 | self._container = self.ContainerCls() 43 | self._lock = Lock() 44 | 45 | def __getitem__(self, key): 46 | # Re-insert the item, moving it to the end of the eviction line. 47 | with self._lock: 48 | item = self._container.pop(key) 49 | self._container[key] = item 50 | return item 51 | 52 | def __setitem__(self, key, value): 53 | evicted_value = _Null 54 | with self._lock: 55 | # Possibly evict the existing value of 'key' 56 | evicted_value = self._container.get(key, _Null) 57 | self._container[key] = value 58 | 59 | # If we didn't evict an existing value, we might have to evict the 60 | # least recently used item from the beginning of the container. 61 | if len(self._container) > self._maxsize: 62 | _key, evicted_value = self._container.popitem(last=False) 63 | 64 | if self.dispose_func and evicted_value is not _Null: 65 | self.dispose_func(evicted_value) 66 | 67 | def __delitem__(self, key): 68 | with self._lock: 69 | value = self._container.pop(key) 70 | 71 | if self.dispose_func: 72 | self.dispose_func(value) 73 | 74 | def __len__(self): 75 | with self._lock: 76 | return len(self._container) 77 | 78 | def __iter__(self): 79 | raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.') 80 | 81 | def clear(self): 82 | with self._lock: 83 | # Copy pointers to all values, then wipe the mapping 84 | # under Python 2, this copies the list of values twice :-| 85 | values = list(self._container.values()) 86 | self._container.clear() 87 | 88 | if self.dispose_func: 89 | for value in values: 90 | self.dispose_func(value) 91 | 92 | def keys(self): 93 | with self._lock: 94 | return self._container.keys() 95 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-app-twitter/14ba32cf9b4d4e05f01d7547d88ab6b43e0ad2bb/twitter2/bin/requests/packages/urllib3/contrib/__init__.py -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/contrib/ntlmpool.py: -------------------------------------------------------------------------------- 1 | # urllib3/contrib/ntlmpool.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """ 8 | NTLM authenticating pool, contributed by erikcederstran 9 | 10 | Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 11 | """ 12 | 13 | try: 14 | from http.client import HTTPSConnection 15 | except ImportError: 16 | from httplib import HTTPSConnection 17 | from logging import getLogger 18 | from ntlm import ntlm 19 | 20 | from urllib3 import HTTPSConnectionPool 21 | 22 | 23 | log = getLogger(__name__) 24 | 25 | 26 | class NTLMConnectionPool(HTTPSConnectionPool): 27 | """ 28 | Implements an NTLM authentication version of an urllib3 connection pool 29 | """ 30 | 31 | scheme = 'https' 32 | 33 | def __init__(self, user, pw, authurl, *args, **kwargs): 34 | """ 35 | authurl is a random URL on the server that is protected by NTLM. 36 | user is the Windows user, probably in the DOMAIN\username format. 37 | pw is the password for the user. 38 | """ 39 | super(NTLMConnectionPool, self).__init__(*args, **kwargs) 40 | self.authurl = authurl 41 | self.rawuser = user 42 | user_parts = user.split('\\', 1) 43 | self.domain = user_parts[0].upper() 44 | self.user = user_parts[1] 45 | self.pw = pw 46 | 47 | def _new_conn(self): 48 | # Performs the NTLM handshake that secures the connection. The socket 49 | # must be kept open while requests are performed. 50 | self.num_connections += 1 51 | log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % 52 | (self.num_connections, self.host, self.authurl)) 53 | 54 | headers = {} 55 | headers['Connection'] = 'Keep-Alive' 56 | req_header = 'Authorization' 57 | resp_header = 'www-authenticate' 58 | 59 | conn = HTTPSConnection(host=self.host, port=self.port) 60 | 61 | # Send negotiation message 62 | headers[req_header] = ( 63 | 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) 64 | log.debug('Request headers: %s' % headers) 65 | conn.request('GET', self.authurl, None, headers) 66 | res = conn.getresponse() 67 | reshdr = dict(res.getheaders()) 68 | log.debug('Response status: %s %s' % (res.status, res.reason)) 69 | log.debug('Response headers: %s' % reshdr) 70 | log.debug('Response data: %s [...]' % res.read(100)) 71 | 72 | # Remove the reference to the socket, so that it can not be closed by 73 | # the response object (we want to keep the socket open) 74 | res.fp = None 75 | 76 | # Server should respond with a challenge message 77 | auth_header_values = reshdr[resp_header].split(', ') 78 | auth_header_value = None 79 | for s in auth_header_values: 80 | if s[:5] == 'NTLM ': 81 | auth_header_value = s[5:] 82 | if auth_header_value is None: 83 | raise Exception('Unexpected %s response header: %s' % 84 | (resp_header, reshdr[resp_header])) 85 | 86 | # Send authentication message 87 | ServerChallenge, NegotiateFlags = \ 88 | ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) 89 | auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, 90 | self.user, 91 | self.domain, 92 | self.pw, 93 | NegotiateFlags) 94 | headers[req_header] = 'NTLM %s' % auth_msg 95 | log.debug('Request headers: %s' % headers) 96 | conn.request('GET', self.authurl, None, headers) 97 | res = conn.getresponse() 98 | log.debug('Response status: %s %s' % (res.status, res.reason)) 99 | log.debug('Response headers: %s' % dict(res.getheaders())) 100 | log.debug('Response data: %s [...]' % res.read()[:100]) 101 | if res.status != 200: 102 | if res.status == 401: 103 | raise Exception('Server rejected request: wrong ' 104 | 'username or password') 105 | raise Exception('Wrong server response: %s %s' % 106 | (res.status, res.reason)) 107 | 108 | res.fp = None 109 | log.debug('Connection established') 110 | return conn 111 | 112 | def urlopen(self, method, url, body=None, headers=None, retries=3, 113 | redirect=True, assert_same_host=True): 114 | if headers is None: 115 | headers = {} 116 | headers['Connection'] = 'Keep-Alive' 117 | return super(NTLMConnectionPool, self).urlopen(method, url, body, 118 | headers, retries, 119 | redirect, 120 | assert_same_host) 121 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/exceptions.py: -------------------------------------------------------------------------------- 1 | # urllib3/exceptions.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | 8 | ## Base Exceptions 9 | 10 | class HTTPError(Exception): 11 | "Base exception used by this module." 12 | pass 13 | 14 | 15 | class PoolError(HTTPError): 16 | "Base exception for errors caused within a pool." 17 | def __init__(self, pool, message): 18 | self.pool = pool 19 | HTTPError.__init__(self, "%s: %s" % (pool, message)) 20 | 21 | def __reduce__(self): 22 | # For pickling purposes. 23 | return self.__class__, (None, None) 24 | 25 | 26 | class RequestError(PoolError): 27 | "Base exception for PoolErrors that have associated URLs." 28 | def __init__(self, pool, url, message): 29 | self.url = url 30 | PoolError.__init__(self, pool, message) 31 | 32 | def __reduce__(self): 33 | # For pickling purposes. 34 | return self.__class__, (None, self.url, None) 35 | 36 | 37 | class SSLError(HTTPError): 38 | "Raised when SSL certificate fails in an HTTPS connection." 39 | pass 40 | 41 | 42 | class DecodeError(HTTPError): 43 | "Raised when automatic decoding based on Content-Type fails." 44 | pass 45 | 46 | 47 | ## Leaf Exceptions 48 | 49 | class MaxRetryError(RequestError): 50 | "Raised when the maximum number of retries is exceeded." 51 | 52 | def __init__(self, pool, url, reason=None): 53 | self.reason = reason 54 | 55 | message = "Max retries exceeded with url: %s" % url 56 | if reason: 57 | message += " (Caused by %s: %s)" % (type(reason), reason) 58 | else: 59 | message += " (Caused by redirect)" 60 | 61 | RequestError.__init__(self, pool, url, message) 62 | 63 | 64 | class HostChangedError(RequestError): 65 | "Raised when an existing pool gets a request for a foreign host." 66 | 67 | def __init__(self, pool, url, retries=3): 68 | message = "Tried to open a foreign host with url: %s" % url 69 | RequestError.__init__(self, pool, url, message) 70 | self.retries = retries 71 | 72 | 73 | class TimeoutError(RequestError): 74 | "Raised when a socket timeout occurs." 75 | pass 76 | 77 | 78 | class EmptyPoolError(PoolError): 79 | "Raised when a pool runs out of connections and no more are allowed." 80 | pass 81 | 82 | 83 | class ClosedPoolError(PoolError): 84 | "Raised when a request enters a pool after the pool has been closed." 85 | pass 86 | 87 | 88 | class LocationParseError(ValueError, HTTPError): 89 | "Raised when get_host or similar fails to parse the URL input." 90 | 91 | def __init__(self, location): 92 | message = "Failed to parse: %s" % location 93 | HTTPError.__init__(self, message) 94 | 95 | self.location = location 96 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/filepost.py: -------------------------------------------------------------------------------- 1 | # urllib3/filepost.py 2 | # Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | import codecs 8 | import mimetypes 9 | 10 | from uuid import uuid4 11 | from io import BytesIO 12 | 13 | from .packages import six 14 | from .packages.six import b 15 | 16 | writer = codecs.lookup('utf-8')[3] 17 | 18 | 19 | def choose_boundary(): 20 | """ 21 | Our embarassingly-simple replacement for mimetools.choose_boundary. 22 | """ 23 | return uuid4().hex 24 | 25 | 26 | def get_content_type(filename): 27 | return mimetypes.guess_type(filename)[0] or 'application/octet-stream' 28 | 29 | 30 | def iter_fields(fields): 31 | """ 32 | Iterate over fields. 33 | 34 | Supports list of (k, v) tuples and dicts. 35 | """ 36 | if isinstance(fields, dict): 37 | return ((k, v) for k, v in six.iteritems(fields)) 38 | 39 | return ((k, v) for k, v in fields) 40 | 41 | 42 | def encode_multipart_formdata(fields, boundary=None): 43 | """ 44 | Encode a dictionary of ``fields`` using the multipart/form-data MIME format. 45 | 46 | :param fields: 47 | Dictionary of fields or list of (key, value) or (key, value, MIME type) 48 | field tuples. The key is treated as the field name, and the value as 49 | the body of the form-data bytes. If the value is a tuple of two 50 | elements, then the first element is treated as the filename of the 51 | form-data section and a suitable MIME type is guessed based on the 52 | filename. If the value is a tuple of three elements, then the third 53 | element is treated as an explicit MIME type of the form-data section. 54 | 55 | Field names and filenames must be unicode. 56 | 57 | :param boundary: 58 | If not specified, then a random boundary will be generated using 59 | :func:`mimetools.choose_boundary`. 60 | """ 61 | body = BytesIO() 62 | if boundary is None: 63 | boundary = choose_boundary() 64 | 65 | for fieldname, value in iter_fields(fields): 66 | body.write(b('--%s\r\n' % (boundary))) 67 | 68 | if isinstance(value, tuple): 69 | if len(value) == 3: 70 | filename, data, content_type = value 71 | else: 72 | filename, data = value 73 | content_type = get_content_type(filename) 74 | writer(body).write('Content-Disposition: form-data; name="%s"; ' 75 | 'filename="%s"\r\n' % (fieldname, filename)) 76 | body.write(b('Content-Type: %s\r\n\r\n' % 77 | (content_type,))) 78 | else: 79 | data = value 80 | writer(body).write('Content-Disposition: form-data; name="%s"\r\n' 81 | % (fieldname)) 82 | body.write(b'\r\n') 83 | 84 | if isinstance(data, int): 85 | data = str(data) # Backwards compatibility 86 | 87 | if isinstance(data, six.text_type): 88 | writer(body).write(data) 89 | else: 90 | body.write(data) 91 | 92 | body.write(b'\r\n') 93 | 94 | body.write(b('--%s--\r\n' % (boundary))) 95 | 96 | content_type = str('multipart/form-data; boundary=%s' % boundary) 97 | 98 | return body.getvalue(), content_type 99 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import ssl_match_hostname 4 | 5 | -------------------------------------------------------------------------------- /twitter2/bin/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py: -------------------------------------------------------------------------------- 1 | """The match_hostname() function from Python 3.2, essential when using SSL.""" 2 | 3 | import re 4 | 5 | __version__ = '3.2.2' 6 | 7 | class CertificateError(ValueError): 8 | pass 9 | 10 | def _dnsname_to_pat(dn): 11 | pats = [] 12 | for frag in dn.split(r'.'): 13 | if frag == '*': 14 | # When '*' is a fragment by itself, it matches a non-empty dotless 15 | # fragment. 16 | pats.append('[^.]+') 17 | else: 18 | # Otherwise, '*' matches any dotless fragment. 19 | frag = re.escape(frag) 20 | pats.append(frag.replace(r'\*', '[^.]*')) 21 | return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) 22 | 23 | def match_hostname(cert, hostname): 24 | """Verify that *cert* (in decoded format as returned by 25 | SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules 26 | are mostly followed, but IP addresses are not accepted for *hostname*. 27 | 28 | CertificateError is raised on failure. On success, the function 29 | returns nothing. 30 | """ 31 | if not cert: 32 | raise ValueError("empty or no certificate") 33 | dnsnames = [] 34 | san = cert.get('subjectAltName', ()) 35 | for key, value in san: 36 | if key == 'DNS': 37 | if _dnsname_to_pat(value).match(hostname): 38 | return 39 | dnsnames.append(value) 40 | if not dnsnames: 41 | # The subject is only checked when there is no dNSName entry 42 | # in subjectAltName 43 | for sub in cert.get('subject', ()): 44 | for key, value in sub: 45 | # XXX according to RFC 2818, the most specific Common Name 46 | # must be used. 47 | if key == 'commonName': 48 | if _dnsname_to_pat(value).match(hostname): 49 | return 50 | dnsnames.append(value) 51 | if len(dnsnames) > 1: 52 | raise CertificateError("hostname %r " 53 | "doesn't match either of %s" 54 | % (hostname, ', '.join(map(repr, dnsnames)))) 55 | elif len(dnsnames) == 1: 56 | raise CertificateError("hostname %r " 57 | "doesn't match %r" 58 | % (hostname, dnsnames[0])) 59 | else: 60 | raise CertificateError("no appropriate commonName or " 61 | "subjectAltName fields were found") 62 | -------------------------------------------------------------------------------- /twitter2/bin/requests/status_codes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .structures import LookupDict 4 | 5 | _codes = { 6 | 7 | # Informational. 8 | 100: ('continue',), 9 | 101: ('switching_protocols',), 10 | 102: ('processing',), 11 | 103: ('checkpoint',), 12 | 122: ('uri_too_long', 'request_uri_too_long'), 13 | 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'), 14 | 201: ('created',), 15 | 202: ('accepted',), 16 | 203: ('non_authoritative_info', 'non_authoritative_information'), 17 | 204: ('no_content',), 18 | 205: ('reset_content', 'reset'), 19 | 206: ('partial_content', 'partial'), 20 | 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), 21 | 208: ('im_used',), 22 | 23 | # Redirection. 24 | 300: ('multiple_choices',), 25 | 301: ('moved_permanently', 'moved', '\\o-'), 26 | 302: ('found',), 27 | 303: ('see_other', 'other'), 28 | 304: ('not_modified',), 29 | 305: ('use_proxy',), 30 | 306: ('switch_proxy',), 31 | 307: ('temporary_redirect', 'temporary_moved', 'temporary'), 32 | 308: ('resume_incomplete', 'resume'), 33 | 34 | # Client Error. 35 | 400: ('bad_request', 'bad'), 36 | 401: ('unauthorized',), 37 | 402: ('payment_required', 'payment'), 38 | 403: ('forbidden',), 39 | 404: ('not_found', '-o-'), 40 | 405: ('method_not_allowed', 'not_allowed'), 41 | 406: ('not_acceptable',), 42 | 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), 43 | 408: ('request_timeout', 'timeout'), 44 | 409: ('conflict',), 45 | 410: ('gone',), 46 | 411: ('length_required',), 47 | 412: ('precondition_failed', 'precondition'), 48 | 413: ('request_entity_too_large',), 49 | 414: ('request_uri_too_large',), 50 | 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), 51 | 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 52 | 417: ('expectation_failed',), 53 | 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), 54 | 422: ('unprocessable_entity', 'unprocessable'), 55 | 423: ('locked',), 56 | 424: ('failed_dependency', 'dependency'), 57 | 425: ('unordered_collection', 'unordered'), 58 | 426: ('upgrade_required', 'upgrade'), 59 | 428: ('precondition_required', 'precondition'), 60 | 429: ('too_many_requests', 'too_many'), 61 | 431: ('header_fields_too_large', 'fields_too_large'), 62 | 444: ('no_response', 'none'), 63 | 449: ('retry_with', 'retry'), 64 | 450: ('blocked_by_windows_parental_controls', 'parental_controls'), 65 | 451: ('unavailable_for_legal_reasons', 'legal_reasons'), 66 | 499: ('client_closed_request',), 67 | 68 | # Server Error. 69 | 500: ('internal_server_error', 'server_error', '/o\\', '✗'), 70 | 501: ('not_implemented',), 71 | 502: ('bad_gateway',), 72 | 503: ('service_unavailable', 'unavailable'), 73 | 504: ('gateway_timeout',), 74 | 505: ('http_version_not_supported', 'http_version'), 75 | 506: ('variant_also_negotiates',), 76 | 507: ('insufficient_storage',), 77 | 509: ('bandwidth_limit_exceeded', 'bandwidth'), 78 | 510: ('not_extended',), 79 | } 80 | 81 | codes = LookupDict(name='status_codes') 82 | 83 | for (code, titles) in list(_codes.items()): 84 | for title in titles: 85 | setattr(codes, title, code) 86 | if not title.startswith('\\'): 87 | setattr(codes, title.upper(), code) 88 | -------------------------------------------------------------------------------- /twitter2/bin/requests/structures.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.structures 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | Data structures that power Requests. 8 | 9 | """ 10 | 11 | import os 12 | import collections 13 | from itertools import islice 14 | 15 | 16 | class IteratorProxy(object): 17 | """docstring for IteratorProxy""" 18 | def __init__(self, i): 19 | self.i = i 20 | # self.i = chain.from_iterable(i) 21 | 22 | def __iter__(self): 23 | return self.i 24 | 25 | def __len__(self): 26 | if hasattr(self.i, '__len__'): 27 | return len(self.i) 28 | if hasattr(self.i, 'len'): 29 | return self.i.len 30 | if hasattr(self.i, 'fileno'): 31 | return os.fstat(self.i.fileno()).st_size 32 | 33 | def read(self, n): 34 | return "".join(islice(self.i, None, n)) 35 | 36 | 37 | class CaseInsensitiveDict(collections.MutableMapping): 38 | """ 39 | A case-insensitive ``dict``-like object. 40 | 41 | Implements all methods and operations of 42 | ``collections.MutableMapping`` as well as dict's ``copy``. Also 43 | provides ``lower_items``. 44 | 45 | All keys are expected to be strings. The structure remembers the 46 | case of the last key to be set, and ``iter(instance)``, 47 | ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` 48 | will contain case-sensitive keys. However, querying and contains 49 | testing is case insensitive: 50 | 51 | cid = CaseInsensitiveDict() 52 | cid['Accept'] = 'application/json' 53 | cid['aCCEPT'] == 'application/json' # True 54 | list(cid) == ['Accept'] # True 55 | 56 | For example, ``headers['content-encoding']`` will return the 57 | value of a ``'Content-Encoding'`` response header, regardless 58 | of how the header name was originally stored. 59 | 60 | If the constructor, ``.update``, or equality comparison 61 | operations are given keys that have equal ``.lower()``s, the 62 | behavior is undefined. 63 | 64 | """ 65 | def __init__(self, data=None, **kwargs): 66 | self._store = dict() 67 | if data is None: 68 | data = {} 69 | self.update(data, **kwargs) 70 | 71 | def __setitem__(self, key, value): 72 | # Use the lowercased key for lookups, but store the actual 73 | # key alongside the value. 74 | self._store[key.lower()] = (key, value) 75 | 76 | def __getitem__(self, key): 77 | return self._store[key.lower()][1] 78 | 79 | def __delitem__(self, key): 80 | del self._store[key.lower()] 81 | 82 | def __iter__(self): 83 | return (casedkey for casedkey, mappedvalue in self._store.values()) 84 | 85 | def __len__(self): 86 | return len(self._store) 87 | 88 | def lower_items(self): 89 | """Like iteritems(), but with all lowercase keys.""" 90 | return ( 91 | (lowerkey, keyval[1]) 92 | for (lowerkey, keyval) 93 | in self._store.items() 94 | ) 95 | 96 | def __eq__(self, other): 97 | if isinstance(other, collections.Mapping): 98 | other = CaseInsensitiveDict(other) 99 | else: 100 | return NotImplemented 101 | # Compare insensitively 102 | return dict(self.lower_items()) == dict(other.lower_items()) 103 | 104 | # Copy is required 105 | def copy(self): 106 | return CaseInsensitiveDict(self._store.values()) 107 | 108 | def __repr__(self): 109 | return '%s(%r)' % (self.__class__.__name__, dict(self.items())) 110 | 111 | 112 | class LookupDict(dict): 113 | """Dictionary lookup object.""" 114 | 115 | def __init__(self, name=None): 116 | self.name = name 117 | super(LookupDict, self).__init__() 118 | 119 | def __repr__(self): 120 | return '' % (self.name) 121 | 122 | def __getitem__(self, key): 123 | # We allow fall-through here, so values default to None 124 | 125 | return self.__dict__.get(key, None) 126 | 127 | def get(self, key, default=None): 128 | return self.__dict__.get(key, default) 129 | -------------------------------------------------------------------------------- /twitter2/bin/requests_oauthlib/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import OAuth1 2 | from .oauth1_session import OAuth1Session 3 | from .oauth2_auth import OAuth2 4 | from .oauth2_session import OAuth2Session, TokenUpdated 5 | -------------------------------------------------------------------------------- /twitter2/bin/requests_oauthlib/core.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | from oauthlib.common import extract_params 4 | from oauthlib.oauth1 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER) 5 | 6 | CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' 7 | CONTENT_TYPE_MULTI_PART = 'multipart/form-data' 8 | 9 | import sys 10 | if sys.version > "3": 11 | unicode = str 12 | 13 | def to_native_str(string): 14 | return string.decode('utf-8') 15 | else: 16 | def to_native_str(string): 17 | return string 18 | 19 | # OBS!: Correct signing of requests are conditional on invoking OAuth1 20 | # as the last step of preparing a request, or at least having the 21 | # content-type set properly. 22 | class OAuth1(object): 23 | """Signs the request using OAuth 1 (RFC5849)""" 24 | def __init__(self, client_key, 25 | client_secret=None, 26 | resource_owner_key=None, 27 | resource_owner_secret=None, 28 | callback_uri=None, 29 | signature_method=SIGNATURE_HMAC, 30 | signature_type=SIGNATURE_TYPE_AUTH_HEADER, 31 | rsa_key=None, verifier=None, 32 | decoding='utf-8'): 33 | 34 | try: 35 | signature_type = signature_type.upper() 36 | except AttributeError: 37 | pass 38 | 39 | self.client = Client(client_key, client_secret, resource_owner_key, 40 | resource_owner_secret, callback_uri, signature_method, 41 | signature_type, rsa_key, verifier, decoding=decoding) 42 | 43 | def __call__(self, r): 44 | """Add OAuth parameters to the request. 45 | 46 | Parameters may be included from the body if the content-type is 47 | urlencoded, if no content type is set a guess is made. 48 | """ 49 | # Overwriting url is safe here as request will not modify it past 50 | # this point. 51 | 52 | content_type = r.headers.get('Content-Type'.encode('utf-8'), '') 53 | if not isinstance(content_type, unicode): 54 | content_type = content_type.decode('utf-8') 55 | 56 | is_form_encoded = (CONTENT_TYPE_FORM_URLENCODED in content_type) 57 | 58 | if is_form_encoded or extract_params(r.body): 59 | r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED 60 | r.url, r.headers, r.body = self.client.sign( 61 | unicode(r.url), unicode(r.method), r.body or '', r.headers) 62 | else: 63 | # Omit body data in the signing of non form-encoded requests 64 | r.url, r.headers, _ = self.client.sign( 65 | unicode(r.url), unicode(r.method), None, r.headers) 66 | 67 | r.url = to_native_str(r.url) 68 | 69 | return r 70 | -------------------------------------------------------------------------------- /twitter2/bin/requests_oauthlib/oauth2_auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from oauthlib.oauth2 import WebApplicationClient, InsecureTransportError 3 | 4 | 5 | class OAuth2(object): 6 | """Adds proof of authorization (OAuth2 token) to the request.""" 7 | 8 | def __init__(self, client_id=None, client=None, token=None): 9 | """Construct a new OAuth 2 authorization object. 10 | 11 | :param client_id: Client id obtained during registration 12 | :param client: oauthlib.oauth2.Client to be used. Default is 13 | WebApplicationClient which is useful for any 14 | hosted application but not mobile or desktop. 15 | :param token: Token dictionary, must include access_token 16 | and token_type. 17 | """ 18 | self._client = client or WebApplicationClient(client_id, token=token) 19 | if token: 20 | for k, v in token.items(): 21 | setattr(self._client, k, v) 22 | 23 | def __call__(self, r): 24 | """Append an OAuth 2 token to the request. 25 | 26 | Note that currently HTTPS is required for all requests. There may be 27 | a token type that allows for plain HTTP in the future and then this 28 | should be updated to allow plain HTTP on a white list basis. 29 | """ 30 | if not r.url.startswith('https://'): 31 | raise InsecureTransportError() 32 | r.url, r.headers, r.body = self._client.add_token(r.url, 33 | http_method=r.method, body=r.body, headers=r.headers) 34 | return r 35 | -------------------------------------------------------------------------------- /twitter2/bin/splunklib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2014 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Python library for Splunk.""" 16 | 17 | __version_info__ = (1, 2, 3) 18 | __version__ = ".".join(map(str, __version_info__)) 19 | 20 | -------------------------------------------------------------------------------- /twitter2/bin/splunklib/modularinput/__init__.py: -------------------------------------------------------------------------------- 1 | """The following imports allow these classes to be imported via 2 | the splunklib.modularinput package like so: 3 | 4 | from splunklib.modularinput import * 5 | """ 6 | from argument import Argument 7 | from event import Event 8 | from event_writer import EventWriter 9 | from input_definition import InputDefinition 10 | from scheme import Scheme 11 | from script import Script 12 | from validation_definition import ValidationDefinition -------------------------------------------------------------------------------- /twitter2/bin/splunklib/modularinput/argument.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2014 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | try: 16 | import xml.etree.ElementTree as ET 17 | except ImportError: 18 | import xml.etree.cElementTree as ET 19 | 20 | class Argument(object): 21 | """Class representing an argument to a modular input kind. 22 | 23 | ``Argument`` is meant to be used with ``Scheme`` to generate an XML 24 | definition of the modular input kind that Splunk understands. 25 | 26 | ``name`` is the only required parameter for the constructor. 27 | 28 | **Example with least parameters**:: 29 | 30 | arg1 = Argument(name="arg1") 31 | 32 | **Example with all parameters**:: 33 | 34 | arg2 = Argument( 35 | name="arg2", 36 | description="This is an argument with lots of parameters", 37 | validation="is_pos_int('some_name')", 38 | data_type=Argument.data_type_number, 39 | required_on_edit=True, 40 | required_on_create=True 41 | ) 42 | """ 43 | 44 | # Constant values, do not change. 45 | # These should be used for setting the value of an Argument object's data_type field. 46 | data_type_boolean = "BOOLEAN" 47 | data_type_number = "NUMBER" 48 | data_type_string = "STRING" 49 | 50 | def __init__(self, name, description=None, validation=None, 51 | data_type=data_type_string, required_on_edit=False, required_on_create=False): 52 | """ 53 | :param name: ``string``, identifier for this argument in Splunk. 54 | :param description: ``string``, human-readable description of the argument. 55 | :param validation: ``string`` specifying how the argument should be validated, if using internal validation. 56 | If using external validation, this will be ignored. 57 | :param data_type: ``string``, data type of this field; use the class constants. 58 | "data_type_boolean", "data_type_number", or "data_type_string". 59 | :param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind. 60 | :param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind. 61 | """ 62 | self.name = name 63 | self.description = description 64 | self.validation = validation 65 | self.data_type = data_type 66 | self.required_on_edit = required_on_edit 67 | self.required_on_create = required_on_create 68 | 69 | def add_to_document(self, parent): 70 | """Adds an ``Argument`` object to this ElementTree document. 71 | 72 | Adds an subelement to the parent element, typically 73 | and sets up its subelements with their respective text. 74 | 75 | :param parent: An ``ET.Element`` to be the parent of a new subelement 76 | :returns: An ``ET.Element`` object representing this argument. 77 | """ 78 | arg = ET.SubElement(parent, "arg") 79 | arg.set("name", self.name) 80 | 81 | if self.description is not None: 82 | ET.SubElement(arg, "description").text = self.description 83 | 84 | if self.validation is not None: 85 | ET.SubElement(arg, "validation").text = self.validation 86 | 87 | # add all other subelements to this Argument, represented by (tag, text) 88 | subelements = [ 89 | ("data_type", self.data_type), 90 | ("required_on_edit", self.required_on_edit), 91 | ("required_on_create", self.required_on_create) 92 | ] 93 | 94 | for name, value in subelements: 95 | ET.SubElement(arg, name).text = str(value).lower() 96 | 97 | return arg -------------------------------------------------------------------------------- /twitter2/bin/splunklib/modularinput/event.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2014 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | try: 16 | import xml.etree.cElementTree as ET 17 | except ImportError as ie: 18 | import xml.etree.ElementTree as ET 19 | 20 | class Event(object): 21 | """Represents an event or fragment of an event to be written by this modular input to Splunk. 22 | 23 | To write an input to a stream, call the ``write_to`` function, passing in a stream. 24 | """ 25 | def __init__(self, data=None, stanza=None, time=None, host=None, index=None, source=None, 26 | sourcetype=None, done=True, unbroken=True): 27 | """There are no required parameters for constructing an Event 28 | 29 | **Example with minimal configuration**:: 30 | 31 | my_event = Event( 32 | data="This is a test of my new event.", 33 | stanza="myStanzaName", 34 | time="%.3f" % 1372187084.000 35 | ) 36 | 37 | **Example with full configuration**:: 38 | 39 | excellent_event = Event( 40 | data="This is a test of my excellent event.", 41 | stanza="excellenceOnly", 42 | time="%.3f" % 1372274622.493, 43 | host="localhost", 44 | index="main", 45 | source="Splunk", 46 | sourcetype="misc", 47 | done=True, 48 | unbroken=True 49 | ) 50 | 51 | :param data: ``string``, the event's text. 52 | :param stanza: ``string``, name of the input this event should be sent to. 53 | :param time: ``float``, time in seconds, including up to 3 decimal places to represent milliseconds. 54 | :param host: ``string``, the event's host, ex: localhost. 55 | :param index: ``string``, the index this event is specified to write to, or None if default index. 56 | :param source: ``string``, the source of this event, or None to have Splunk guess. 57 | :param sourcetype: ``string``, source type currently set on this event, or None to have Splunk guess. 58 | :param done: ``boolean``, is this a complete ``Event``? False if an ``Event`` fragment. 59 | :param unbroken: ``boolean``, Is this event completely encapsulated in this ``Event`` object? 60 | """ 61 | self.data = data 62 | self.done = done 63 | self.host = host 64 | self.index = index 65 | self.source = source 66 | self.sourceType = sourcetype 67 | self.stanza = stanza 68 | self.time = time 69 | self.unbroken = unbroken 70 | 71 | def write_to(self, stream): 72 | """Write an XML representation of self, an ``Event`` object, to the given stream. 73 | 74 | The ``Event`` object will only be written if its data field is defined, 75 | otherwise a ``ValueError`` is raised. 76 | 77 | :param stream: stream to write XML to. 78 | """ 79 | if self.data is None: 80 | raise ValueError("Events must have at least the data field set to be written to XML.") 81 | 82 | event = ET.Element("event") 83 | if self.stanza is not None: 84 | event.set("stanza", self.stanza) 85 | event.set("unbroken", str(int(self.unbroken))) 86 | 87 | # if a time isn't set, let Splunk guess by not creating a