├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE.txt ├── README.md ├── app.py ├── app.yaml ├── batch.py ├── config.template ├── container.yaml.template ├── libs ├── apiclient │ ├── __init__.py │ ├── channel.py │ ├── discovery.py │ ├── errors.py │ ├── http.py │ ├── mimeparse.py │ ├── model.py │ ├── sample_tools.py │ └── schema.py ├── config.py ├── gnippy │ ├── __init__.py │ ├── config.py │ ├── errors.py │ ├── powertrackclient.py │ ├── rules.py │ └── searchclient.py ├── httplib2 │ ├── __init__.py │ ├── cacerts.txt │ ├── iri2uri.py │ └── socks.py ├── oauth2client │ ├── __init__.py │ ├── anyjson.py │ ├── appengine.py │ ├── client.py │ ├── clientsecrets.py │ ├── crypt.py │ ├── django_orm.py │ ├── file.py │ ├── gce.py │ ├── keyring_storage.py │ ├── locked_file.py │ ├── multistore_file.py │ ├── old_run.py │ ├── tools.py │ ├── util.py │ └── xsrfutil.py ├── requests │ ├── __init__.py │ ├── adapters.py │ ├── api.py │ ├── auth.py │ ├── certs.py │ ├── compat.py │ ├── cookies.py │ ├── exceptions.py │ ├── hooks.py │ ├── models.py │ ├── packages │ │ ├── __init__.py │ │ ├── charade │ │ │ ├── __init__.py │ │ │ ├── big5freq.py │ │ │ ├── big5prober.py │ │ │ ├── chardistribution.py │ │ │ ├── charsetgroupprober.py │ │ │ ├── charsetprober.py │ │ │ ├── codingstatemachine.py │ │ │ ├── compat.py │ │ │ ├── constants.py │ │ │ ├── cp949prober.py │ │ │ ├── escprober.py │ │ │ ├── escsm.py │ │ │ ├── eucjpprober.py │ │ │ ├── euckrfreq.py │ │ │ ├── euckrprober.py │ │ │ ├── euctwfreq.py │ │ │ ├── euctwprober.py │ │ │ ├── gb2312freq.py │ │ │ ├── gb2312prober.py │ │ │ ├── hebrewprober.py │ │ │ ├── jisfreq.py │ │ │ ├── jpcntx.py │ │ │ ├── langbulgarianmodel.py │ │ │ ├── langcyrillicmodel.py │ │ │ ├── langgreekmodel.py │ │ │ ├── langhebrewmodel.py │ │ │ ├── langhungarianmodel.py │ │ │ ├── langthaimodel.py │ │ │ ├── latin1prober.py │ │ │ ├── mbcharsetprober.py │ │ │ ├── mbcsgroupprober.py │ │ │ ├── mbcssm.py │ │ │ ├── sbcharsetprober.py │ │ │ ├── sbcsgroupprober.py │ │ │ ├── sjisprober.py │ │ │ ├── universaldetector.py │ │ │ └── utf8prober.py │ │ └── urllib3 │ │ │ ├── __init__.py │ │ │ ├── _collections.py │ │ │ ├── connectionpool.py │ │ │ ├── contrib │ │ │ ├── __init__.py │ │ │ ├── ntlmpool.py │ │ │ └── pyopenssl.py │ │ │ ├── exceptions.py │ │ │ ├── fields.py │ │ │ ├── filepost.py │ │ │ ├── packages │ │ │ ├── __init__.py │ │ │ ├── ordered_dict.py │ │ │ ├── six.py │ │ │ └── ssl_match_hostname │ │ │ │ └── __init__.py │ │ │ ├── poolmanager.py │ │ │ ├── request.py │ │ │ ├── response.py │ │ │ └── util.py │ ├── sessions.py │ ├── status_codes.py │ ├── structures.py │ └── utils.py └── uritemplate │ └── __init__.py ├── load.py ├── logging.conf ├── queue.yaml ├── requirements.txt ├── schema ├── sample_tweet_gnip.json ├── sample_twet_twitter.json ├── schema_gnip.json └── schema_twitter.json ├── screenshot.png ├── setup.py ├── static ├── css │ ├── c3.css │ ├── c3.min.css │ ├── style.css │ ├── theme │ │ ├── bootstrap.min.css │ │ └── styles.css │ └── tipsy.css ├── img │ ├── bird.ico │ ├── bird_large.png │ ├── config_fields.png │ ├── loading.gif │ ├── screenshot.png │ ├── settings_1.png │ └── settings_2.png └── js │ ├── bootstrap.min.js │ ├── c3.js │ ├── c3.min.js │ ├── d3.js │ ├── jquery.tipsy.js │ ├── mustache.js │ └── script.js ├── templates ├── admin.html ├── base.html ├── chart_partial.html ├── home.html ├── rule_add_partial.html ├── rule_list.html ├── table_detail.html └── table_list.html └── utils.py /.dockerignore: -------------------------------------------------------------------------------- 1 | data/* 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | .build 22 | 23 | # Installer logs 24 | pip-log.txt 25 | 26 | # Unit test / coverage reports 27 | .coverage 28 | .tox 29 | nosetests.xml 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | client_secrets.json 36 | .project 37 | .pydevproject 38 | *.p12 39 | *.pem 40 | **/*json 41 | **/*conf 42 | **/*txt 43 | config 44 | container.yaml 45 | schema.json 46 | output.txt 47 | *.sh 48 | data/* 49 | fabfile.py 50 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM google/python 2 | 3 | RUN apt-get install -y libffi-dev 4 | 5 | RUN pip install --upgrade pip 6 | RUN pip install config 7 | RUN pip install tweepy 8 | RUN pip install bigquery-python 9 | RUN pip install --upgrade google-api-python-client 10 | 11 | ADD config /config 12 | ADD schema /schema 13 | ADD key.p12 /key.p12 14 | ADD logging.conf /logging.conf 15 | ADD libs /libs 16 | ADD utils.py /utils.py 17 | ADD load.py /load.py 18 | 19 | CMD python load.py 20 | -------------------------------------------------------------------------------- /app.yaml: -------------------------------------------------------------------------------- 1 | # https://cloud.google.com/appengine/docs/python/modules/ 2 | application: twitter-gnip 3 | version: 1 4 | runtime: python27 5 | api_version: 1 6 | threadsafe: true 7 | 8 | handlers: 9 | - url: /static 10 | static_dir: static 11 | 12 | - url: /.* 13 | script: app.application 14 | 15 | libraries: 16 | - name: jinja2 17 | version: latest 18 | - name: pycrypto 19 | version: latest 20 | - name: ssl 21 | version: latest 22 | 23 | skip_files: 24 | - ^data$ 25 | -------------------------------------------------------------------------------- /batch.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os import walk 3 | import sys 4 | import thread 5 | import gzip 6 | from multiprocessing import Pool, Process, Queue 7 | 8 | PROCESS_COUNT = 8 9 | 10 | # file utilities 11 | class Utils: 12 | 13 | @staticmethod 14 | def rename(file, to): 15 | 16 | call_rename = "mv %s %s" % (file, to) 17 | print call_rename 18 | os.system(call_rename) 19 | 20 | return to 21 | 22 | @staticmethod 23 | def archive(file): 24 | 25 | if ".archive" in file: 26 | return file 27 | 28 | file_archive = "%s.archive" % file 29 | Utils.rename(file, file_archive) 30 | 31 | return file_archive 32 | 33 | @staticmethod 34 | def unarchive(file): 35 | 36 | if ".archive" not in file: 37 | return file 38 | 39 | file2 = file[:-8] 40 | Utils.rename(file, file2) 41 | 42 | return file2 43 | 44 | @staticmethod 45 | def gzip(file): 46 | 47 | if ".gz" in file: 48 | return file 49 | 50 | call_zip = "gzip %s" % (file) 51 | print call_zip 52 | os.system(call_zip) 53 | 54 | return "%s.gz" % file 55 | 56 | @staticmethod 57 | def gunzip(file): 58 | 59 | if ".gz" not in file: 60 | return file 61 | 62 | call_unzip = "gunzip %s" % (file) 63 | print call_unzip 64 | os.system(call_unzip) 65 | 66 | return file[:-3] 67 | 68 | @staticmethod 69 | def cat_all(path, file): 70 | 71 | call_cat = "find %s -type f -exec cat {} + > %s" % (path, file) 72 | print call_cat 73 | os.system(call_cat) 74 | 75 | @staticmethod 76 | def get_files(path): 77 | 78 | files = [] 79 | 80 | # process all files and let async handle archiving 81 | for (dirpath, dirnames, filenames) in walk(path): 82 | for f in filenames: 83 | file = "%s/%s" % (dirpath, f) 84 | files.append(file) 85 | 86 | return files 87 | 88 | # restore file to original gzip 89 | def reset_file(file, table, output=None): 90 | 91 | if ".archive" in file: 92 | file = Utils.unarchive(file) 93 | 94 | # ignore archive files 95 | if file.endswith(".json"): 96 | file = Utils.gzip(file) 97 | 98 | if output: 99 | output.put(file) 100 | 101 | return file 102 | 103 | def process_files(path, table): 104 | 105 | files = Utils.get_files(path) 106 | 107 | # if zipped, unzip for loading 108 | for f in files: 109 | if "json.gz" in f: 110 | file = Utils.gunzip(f) 111 | 112 | # get new list of files 113 | files = Utils.get_files(path) 114 | 115 | file_result = "master.json" 116 | Utils.cat_all(path, file_result) 117 | 118 | file_gz = Utils.gzip(file_result) 119 | 120 | # load to bigquery 121 | call_batch = "bq load --source_format=NEWLINE_DELIMITED_JSON --max_bad_records=500000 %s %s" % (table, file_result) 122 | print call_batch 123 | os.system(call_batch) 124 | 125 | return file_result 126 | 127 | if __name__ == '__main__': 128 | 129 | if len(sys.argv) != 4: 130 | print "Usage: batch.py [reset|process] [file|directory] " 131 | 132 | (script, action, path, table) = sys.argv 133 | 134 | files = [] 135 | processes = [] 136 | 137 | 138 | function = None 139 | if action == 'reset': 140 | 141 | if os.path.isfile(path): 142 | files.append(path) 143 | else: 144 | files = Utils.get_files(path) 145 | 146 | pool = Pool(processes=PROCESS_COUNT) 147 | results = [pool.apply_async(reset_file, args=(f, table)) for f in files] 148 | 149 | output = [p.get() for p in results] 150 | print output 151 | 152 | elif action == 'process': 153 | 154 | process_files(path, table) 155 | 156 | 157 | -------------------------------------------------------------------------------- /config.template: -------------------------------------------------------------------------------- 1 | # Google Cloud Project: Project ID as listed in the Google Developers Console (https://console.developers.google.com/project) 2 | PROJECT_ID: 3 | PROJECT_NUMBER: 4 | 5 | # Google Cloud Project: Service account email address in "APIs & auth->Credentials" (https://console.developers.google.com/project/PROJECT_ID/apiui/credential) 6 | SERVICE_ACCOUNT: 7 | 8 | # Google Cloud Project: PKCS12 P12 key in "APIs & auth->Credentials" (https://console.developers.google.com/project/PROJECT_ID/apiui/credential) 9 | KEY_FILE: "key.p12" 10 | 11 | # Google BigQuery: Existing dataset and table you want to stream data into (https://bigquery.cloud.google.com/project/PROJECT_ID) 12 | DATASET_ID: "gnip" 13 | TABLE_ID: "tweets" 14 | 15 | # Gnip: Endpoints and user credentials (http://console.gnip.com) 16 | GNIP_STREAM_URL: "https://stream.gnip.com:443/accounts/YOUR_ACCOUNT/publishers/twitter/streams/track/prod.json" 17 | GNIP_STREAM_USERNAME: 18 | GNIP_STREAM_PASSWORD: 19 | 20 | GNIP_SEARCH_URL: "https://search.gnip.com/accounts/YOUR_ACCOUNT/search/STREAM.json" 21 | GNIP_SEARCH_USERNAME: 22 | GNIP_SEARCH_PASSWORD: 23 | 24 | # Twitter API: App keys and tokens (http://apps.twitter.com) 25 | CONSUMER_KEY: 26 | CONSUMER_SECRET: 27 | ACCESS_TOKEN: 28 | ACCESS_TOKEN_SECRET: 29 | 30 | # Daemon mode: gnip or twitter 31 | MODE: "gnip" 32 | -------------------------------------------------------------------------------- /container.yaml.template: -------------------------------------------------------------------------------- 1 | version: v1beta2 2 | containers: 3 | - name: image_proj 4 | image: gcr.io/{{PROJECT_ID}}/image 5 | command: ['python' 'load.py'] 6 | ports: 7 | - name: image_proj 8 | hostPort: 8080 9 | containerPort: 8080 10 | -------------------------------------------------------------------------------- /libs/apiclient/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __version__ = "1.2" 16 | -------------------------------------------------------------------------------- /libs/apiclient/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.4 2 | # 3 | # Copyright (C) 2010 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Errors for the library. 18 | 19 | All exceptions defined by the library 20 | should be defined in this file. 21 | """ 22 | 23 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 24 | 25 | 26 | from oauth2client import util 27 | from oauth2client.anyjson import simplejson 28 | 29 | 30 | class Error(Exception): 31 | """Base error for this module.""" 32 | pass 33 | 34 | 35 | class HttpError(Error): 36 | """HTTP data was invalid or unexpected.""" 37 | 38 | @util.positional(3) 39 | def __init__(self, resp, content, uri=None): 40 | self.resp = resp 41 | self.content = content 42 | self.uri = uri 43 | 44 | def _get_reason(self): 45 | """Calculate the reason for the error from the response content.""" 46 | reason = self.resp.reason 47 | try: 48 | data = simplejson.loads(self.content) 49 | reason = data['error']['message'] 50 | except (ValueError, KeyError): 51 | pass 52 | if reason is None: 53 | reason = '' 54 | return reason 55 | 56 | def __repr__(self): 57 | if self.uri: 58 | return '' % ( 59 | self.resp.status, self.uri, self._get_reason().strip()) 60 | else: 61 | return '' % (self.resp.status, self._get_reason()) 62 | 63 | __str__ = __repr__ 64 | 65 | 66 | class InvalidJsonError(Error): 67 | """The JSON returned could not be parsed.""" 68 | pass 69 | 70 | 71 | class UnknownFileType(Error): 72 | """File type unknown or unexpected.""" 73 | pass 74 | 75 | 76 | class UnknownLinkType(Error): 77 | """Link type unknown or unexpected.""" 78 | pass 79 | 80 | 81 | class UnknownApiNameOrVersion(Error): 82 | """No API with that name and version exists.""" 83 | pass 84 | 85 | 86 | class UnacceptableMimeTypeError(Error): 87 | """That is an unacceptable mimetype for this operation.""" 88 | pass 89 | 90 | 91 | class MediaUploadSizeError(Error): 92 | """Media is larger than the method can accept.""" 93 | pass 94 | 95 | 96 | class ResumableUploadError(HttpError): 97 | """Error occured during resumable upload.""" 98 | pass 99 | 100 | 101 | class InvalidChunkSizeError(Error): 102 | """The given chunksize is not valid.""" 103 | pass 104 | 105 | class InvalidNotificationError(Error): 106 | """The channel Notification is invalid.""" 107 | pass 108 | 109 | class BatchError(HttpError): 110 | """Error occured during batch operations.""" 111 | 112 | @util.positional(2) 113 | def __init__(self, reason, resp=None, content=None): 114 | self.resp = resp 115 | self.content = content 116 | self.reason = reason 117 | 118 | def __repr__(self): 119 | return '' % (self.resp.status, self.reason) 120 | 121 | __str__ = __repr__ 122 | 123 | 124 | class UnexpectedMethodError(Error): 125 | """Exception raised by RequestMockBuilder on unexpected calls.""" 126 | 127 | @util.positional(1) 128 | def __init__(self, methodId=None): 129 | """Constructor for an UnexpectedMethodError.""" 130 | super(UnexpectedMethodError, self).__init__( 131 | 'Received unexpected call %s' % methodId) 132 | 133 | 134 | class UnexpectedBodyError(Error): 135 | """Exception raised by RequestMockBuilder on unexpected bodies.""" 136 | 137 | def __init__(self, expected, provided): 138 | """Constructor for an UnexpectedMethodError.""" 139 | super(UnexpectedBodyError, self).__init__( 140 | 'Expected: [%s] - Provided: [%s]' % (expected, provided)) 141 | -------------------------------------------------------------------------------- /libs/apiclient/sample_tools.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities for making samples. 16 | 17 | Consolidates a lot of code commonly repeated in sample applications. 18 | """ 19 | 20 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 21 | __all__ = ['init'] 22 | 23 | 24 | import argparse 25 | import httplib2 26 | import os 27 | 28 | from apiclient import discovery 29 | from oauth2client import client 30 | from oauth2client import file 31 | from oauth2client import tools 32 | 33 | 34 | def init(argv, name, version, doc, filename, scope=None, parents=[]): 35 | """A common initialization routine for samples. 36 | 37 | Many of the sample applications do the same initialization, which has now 38 | been consolidated into this function. This function uses common idioms found 39 | in almost all the samples, i.e. for an API with name 'apiname', the 40 | credentials are stored in a file named apiname.dat, and the 41 | client_secrets.json file is stored in the same directory as the application 42 | main file. 43 | 44 | Args: 45 | argv: list of string, the command-line parameters of the application. 46 | name: string, name of the API. 47 | version: string, version of the API. 48 | doc: string, description of the application. Usually set to __doc__. 49 | file: string, filename of the application. Usually set to __file__. 50 | parents: list of argparse.ArgumentParser, additional command-line flags. 51 | scope: string, The OAuth scope used. 52 | 53 | Returns: 54 | A tuple of (service, flags), where service is the service object and flags 55 | is the parsed command-line flags. 56 | """ 57 | if scope is None: 58 | scope = 'https://www.googleapis.com/auth/' + name 59 | 60 | # Parser command-line arguments. 61 | parent_parsers = [tools.argparser] 62 | parent_parsers.extend(parents) 63 | parser = argparse.ArgumentParser( 64 | description=doc, 65 | formatter_class=argparse.RawDescriptionHelpFormatter, 66 | parents=parent_parsers) 67 | flags = parser.parse_args(argv[1:]) 68 | 69 | # Name of a file containing the OAuth 2.0 information for this 70 | # application, including client_id and client_secret, which are found 71 | # on the API Access tab on the Google APIs 72 | # Console . 73 | client_secrets = os.path.join(os.path.dirname(filename), 74 | 'client_secrets.json') 75 | 76 | # Set up a Flow object to be used if we need to authenticate. 77 | flow = client.flow_from_clientsecrets(client_secrets, 78 | scope=scope, 79 | message=tools.message_if_missing(client_secrets)) 80 | 81 | # Prepare credentials, and authorize HTTP object with them. 82 | # If the credentials don't exist or are invalid run through the native client 83 | # flow. The Storage object will ensure that if successful the good 84 | # credentials will get written back to a file. 85 | storage = file.Storage(name + '.dat') 86 | credentials = storage.get() 87 | if credentials is None or credentials.invalid: 88 | credentials = tools.run_flow(flow, storage, flags) 89 | http = credentials.authorize(http = httplib2.Http()) 90 | 91 | # Construct a service object via the discovery service. 92 | service = discovery.build(name, version, http=http) 93 | return (service, flags) 94 | -------------------------------------------------------------------------------- /libs/gnippy/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # gnippy - GNIP for Python 3 | 4 | __title__ = 'gnippy' 5 | __version__ = '0.3.4' 6 | __author__ = 'Abhinav Ajgaonkar' 7 | __license__ = 'Apache 2.0' 8 | __copyright__ = 'Copyright 2012-2013 Abhinav Ajgaonkar' 9 | 10 | from powertrackclient import PowerTrackClient 11 | from searchclient import SearchClient 12 | -------------------------------------------------------------------------------- /libs/gnippy/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import ConfigParser 4 | import os 5 | 6 | from gnippy.errors import ConfigFileNotFoundException, IncompleteConfigurationException 7 | 8 | 9 | def get_default_config_file_path(): 10 | """ 11 | Returns the absolute path to the default placement of the 12 | config file (~/.gnippy) 13 | """ 14 | # --- This section has been borrowed from boto ------------------------ 15 | # Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ 16 | # Copyright (c) 2011 Chris Moyer http://coredumped.org/ 17 | # https://raw.github.com/boto/boto/develop/boto/pyami/config.py 18 | # 19 | # If running in Google App Engine there is no "user" and 20 | # os.path.expanduser() will fail. Attempt to detect this case and use a 21 | # no-op expanduser function in this case. 22 | try: 23 | os.path.expanduser('~') 24 | expanduser = os.path.expanduser 25 | except (AttributeError, ImportError): 26 | # This is probably running on App Engine. 27 | expanduser = (lambda x: x) 28 | # ---End borrowed section --------------------------------------------- 29 | return os.path.join(expanduser("~"), ".gnippy") 30 | 31 | 32 | def get_config(config_file_path=None): 33 | """ 34 | Parses the .gnippy file at the provided location. 35 | Returns a dictionary with all the possible configuration options, 36 | with None for the options that were not provided. 37 | """ 38 | if not os.path.isfile(config_file_path): 39 | raise ConfigFileNotFoundException("Could not find %s" % config_file_path) 40 | 41 | # Attempt to parse the config file 42 | result = {} 43 | parser = ConfigParser.SafeConfigParser() 44 | parser.read(config_file_path) 45 | 46 | # These are all the configurable settings by setting 47 | options = { 48 | "Credentials": ('username', 'password'), 49 | "PowerTrack": ("url", ) 50 | } 51 | 52 | for section in options: 53 | keys = options[section] 54 | values = {} 55 | for key in keys: 56 | try: 57 | values[key] = parser.get(section, key) 58 | except ConfigParser.NoOptionError: 59 | values[key] = None 60 | except ConfigParser.NoSectionError: 61 | values[key] = None 62 | 63 | result[section] = values 64 | 65 | return result 66 | 67 | 68 | def resolve(kwarg_dict): 69 | """ 70 | Look for auth and url info in the kwargs. 71 | If they don't exist, look for a config file path and resolve auth & url info from it. 72 | If no config file path exists, try to load the config file from the default path. 73 | If this method returns without errors, the dictionary is guaranteed to contain: 74 | { 75 | "auth": ("username", "password"), 76 | "url": "PowerTrackUrl" 77 | } 78 | """ 79 | conf = {} 80 | if "auth" in kwarg_dict: 81 | conf['auth'] = kwarg_dict['auth'] 82 | 83 | if "url" in kwarg_dict: 84 | conf['url'] = kwarg_dict['url'] 85 | 86 | if "auth" not in conf or "url" not in conf: 87 | if "config_file_path" in kwarg_dict: 88 | file_conf = get_config(config_file_path=kwarg_dict['config_file_path']) 89 | 90 | else: 91 | file_conf = get_config(config_file_path=get_default_config_file_path()) 92 | 93 | if "auth" not in conf: 94 | creds = file_conf['Credentials'] 95 | if creds['username'] and creds['password']: 96 | conf['auth'] = (creds['username'], creds['password']) 97 | else: 98 | raise IncompleteConfigurationException("Incomplete authentication information provided. Please provide"\ 99 | + " a username and password.") 100 | 101 | if "url" not in conf: 102 | if file_conf['PowerTrack']['url']: 103 | conf['url'] = file_conf['PowerTrack']['url'] 104 | else: 105 | raise IncompleteConfigurationException("Please provide a PowerTrack url.") 106 | 107 | return conf -------------------------------------------------------------------------------- /libs/gnippy/errors.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | class ConfigFileNotFoundException(Exception): 4 | """ Raised when an invalid config_file_path argument was passed. """ 5 | pass 6 | 7 | 8 | class IncompleteConfigurationException(Exception): 9 | """ Raised when no .gnippy file is found. """ 10 | pass 11 | 12 | 13 | class BadArgumentException(Exception): 14 | """ Raised when an invalid argument is detected. """ 15 | pass 16 | 17 | 18 | class RuleAddFailedException(Exception): 19 | """ Raised when a rule add fails. """ 20 | pass 21 | 22 | 23 | class RulesListFormatException(Exception): 24 | """ Raised when rules_list is not in the correct format. """ 25 | pass 26 | 27 | 28 | class RulesGetFailedException(Exception): 29 | """ Raised when listing the current rule set fails. """ 30 | pass 31 | 32 | 33 | class BadPowerTrackUrlException(Exception): 34 | """ Raised when the PowerTrack URL looks like its incorrect. """ 35 | pass 36 | 37 | 38 | class RuleDeleteFailedException(Exception): 39 | """ Raised when a rule delete fails. """ 40 | pass -------------------------------------------------------------------------------- /libs/gnippy/powertrackclient.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import threading 4 | 5 | import requests 6 | 7 | from gnippy import config 8 | 9 | class PowerTrackClient(): 10 | """ 11 | PowerTrackClient allows you to connect to the GNIP 12 | power track stream and fetch data. 13 | """ 14 | callback = None 15 | url = None 16 | auth = None 17 | 18 | def __init__(self, callback, **kwargs): 19 | self.callback = callback 20 | c = config.resolve(kwargs) 21 | self.url = c['url'] 22 | self.auth = c['auth'] 23 | 24 | def connect(self): 25 | self.worker = Worker(self.url, self.auth, self.callback) 26 | self.worker.setDaemon(True) 27 | self.worker.start() 28 | 29 | def disconnect(self): 30 | self.worker.stop() 31 | self.worker.join() 32 | 33 | def load_config_from_file(self, url, auth, config_file_path): 34 | """ Attempt to load the config from a file. """ 35 | conf = config.get_config(config_file_path=config_file_path) 36 | 37 | if url is None: 38 | conf_url = conf['PowerTrack']['url'] 39 | if conf_url: 40 | self.url = conf['PowerTrack']['url'] 41 | else: 42 | self.url = url 43 | 44 | if auth is None: 45 | conf_uname = conf['Credentials']['username'] 46 | conf_pwd = conf['Credentials']['password'] 47 | self.auth = (conf_uname, conf_pwd) 48 | else: 49 | self.auth = auth 50 | 51 | 52 | class Worker(threading.Thread): 53 | """ Background worker to fetch data without blocking """ 54 | def __init__(self, url, auth, callback): 55 | super(Worker, self).__init__() 56 | self.url = url 57 | self.auth = auth 58 | self.on_data = callback 59 | self._stop = threading.Event() 60 | 61 | def stop(self): 62 | self._stop.set() 63 | 64 | def stopped(self): 65 | return self._stop.isSet() 66 | 67 | def run(self): 68 | r = requests.get(self.url, auth=self.auth, stream=True) 69 | for line in r.iter_lines(): 70 | if self.stopped(): 71 | break 72 | elif line: 73 | self.on_data(line) -------------------------------------------------------------------------------- /libs/httplib2/iri2uri.py: -------------------------------------------------------------------------------- 1 | """ 2 | iri2uri 3 | 4 | Converts an IRI to a URI. 5 | 6 | """ 7 | __author__ = "Joe Gregorio (joe@bitworking.org)" 8 | __copyright__ = "Copyright 2006, Joe Gregorio" 9 | __contributors__ = [] 10 | __version__ = "1.0.0" 11 | __license__ = "MIT" 12 | __history__ = """ 13 | """ 14 | 15 | import urlparse 16 | 17 | 18 | # Convert an IRI to a URI following the rules in RFC 3987 19 | # 20 | # The characters we need to enocde and escape are defined in the spec: 21 | # 22 | # iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD 23 | # ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF 24 | # / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD 25 | # / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD 26 | # / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD 27 | # / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD 28 | # / %xD0000-DFFFD / %xE1000-EFFFD 29 | 30 | escape_range = [ 31 | (0xA0, 0xD7FF), 32 | (0xE000, 0xF8FF), 33 | (0xF900, 0xFDCF), 34 | (0xFDF0, 0xFFEF), 35 | (0x10000, 0x1FFFD), 36 | (0x20000, 0x2FFFD), 37 | (0x30000, 0x3FFFD), 38 | (0x40000, 0x4FFFD), 39 | (0x50000, 0x5FFFD), 40 | (0x60000, 0x6FFFD), 41 | (0x70000, 0x7FFFD), 42 | (0x80000, 0x8FFFD), 43 | (0x90000, 0x9FFFD), 44 | (0xA0000, 0xAFFFD), 45 | (0xB0000, 0xBFFFD), 46 | (0xC0000, 0xCFFFD), 47 | (0xD0000, 0xDFFFD), 48 | (0xE1000, 0xEFFFD), 49 | (0xF0000, 0xFFFFD), 50 | (0x100000, 0x10FFFD), 51 | ] 52 | 53 | def encode(c): 54 | retval = c 55 | i = ord(c) 56 | for low, high in escape_range: 57 | if i < low: 58 | break 59 | if i >= low and i <= high: 60 | retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) 61 | break 62 | return retval 63 | 64 | 65 | def iri2uri(uri): 66 | """Convert an IRI to a URI. Note that IRIs must be 67 | passed in a unicode strings. That is, do not utf-8 encode 68 | the IRI before passing it into the function.""" 69 | if isinstance(uri ,unicode): 70 | (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) 71 | authority = authority.encode('idna') 72 | # For each character in 'ucschar' or 'iprivate' 73 | # 1. encode as utf-8 74 | # 2. then %-encode each octet of that utf-8 75 | uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) 76 | uri = "".join([encode(c) for c in uri]) 77 | return uri 78 | 79 | if __name__ == "__main__": 80 | import unittest 81 | 82 | class Test(unittest.TestCase): 83 | 84 | def test_uris(self): 85 | """Test that URIs are invariant under the transformation.""" 86 | invariant = [ 87 | u"ftp://ftp.is.co.za/rfc/rfc1808.txt", 88 | u"http://www.ietf.org/rfc/rfc2396.txt", 89 | u"ldap://[2001:db8::7]/c=GB?objectClass?one", 90 | u"mailto:John.Doe@example.com", 91 | u"news:comp.infosystems.www.servers.unix", 92 | u"tel:+1-816-555-1212", 93 | u"telnet://192.0.2.16:80/", 94 | u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] 95 | for uri in invariant: 96 | self.assertEqual(uri, iri2uri(uri)) 97 | 98 | def test_iri(self): 99 | """ Test that the right type of escaping is done for each part of the URI.""" 100 | self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) 101 | self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}")) 102 | self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}")) 103 | self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}")) 104 | self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")) 105 | self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))) 106 | self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8'))) 107 | 108 | unittest.main() 109 | 110 | 111 | -------------------------------------------------------------------------------- /libs/oauth2client/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.2" 2 | 3 | GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/auth' 4 | GOOGLE_REVOKE_URI = 'https://accounts.google.com/o/oauth2/revoke' 5 | GOOGLE_TOKEN_URI = 'https://accounts.google.com/o/oauth2/token' 6 | -------------------------------------------------------------------------------- /libs/oauth2client/anyjson.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2010 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utility module to import a JSON module 16 | 17 | Hides all the messy details of exactly where 18 | we get a simplejson module from. 19 | """ 20 | 21 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 22 | 23 | 24 | try: # pragma: no cover 25 | # Should work for Python2.6 and higher. 26 | import json as simplejson 27 | except ImportError: # pragma: no cover 28 | try: 29 | import simplejson 30 | except ImportError: 31 | # Try to import from django, should work on App Engine 32 | from django.utils import simplejson 33 | -------------------------------------------------------------------------------- /libs/oauth2client/clientsecrets.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities for reading OAuth 2.0 client secret files. 16 | 17 | A client_secrets.json file contains all the information needed to interact with 18 | an OAuth 2.0 protected service. 19 | """ 20 | 21 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 22 | 23 | 24 | from anyjson import simplejson 25 | 26 | # Properties that make a client_secrets.json file valid. 27 | TYPE_WEB = 'web' 28 | TYPE_INSTALLED = 'installed' 29 | 30 | VALID_CLIENT = { 31 | TYPE_WEB: { 32 | 'required': [ 33 | 'client_id', 34 | 'client_secret', 35 | 'redirect_uris', 36 | 'auth_uri', 37 | 'token_uri', 38 | ], 39 | 'string': [ 40 | 'client_id', 41 | 'client_secret', 42 | ], 43 | }, 44 | TYPE_INSTALLED: { 45 | 'required': [ 46 | 'client_id', 47 | 'client_secret', 48 | 'redirect_uris', 49 | 'auth_uri', 50 | 'token_uri', 51 | ], 52 | 'string': [ 53 | 'client_id', 54 | 'client_secret', 55 | ], 56 | }, 57 | } 58 | 59 | 60 | class Error(Exception): 61 | """Base error for this module.""" 62 | pass 63 | 64 | 65 | class InvalidClientSecretsError(Error): 66 | """Format of ClientSecrets file is invalid.""" 67 | pass 68 | 69 | 70 | def _validate_clientsecrets(obj): 71 | if obj is None or len(obj) != 1: 72 | raise InvalidClientSecretsError('Invalid file format.') 73 | client_type = obj.keys()[0] 74 | if client_type not in VALID_CLIENT.keys(): 75 | raise InvalidClientSecretsError('Unknown client type: %s.' % client_type) 76 | client_info = obj[client_type] 77 | for prop_name in VALID_CLIENT[client_type]['required']: 78 | if prop_name not in client_info: 79 | raise InvalidClientSecretsError( 80 | 'Missing property "%s" in a client type of "%s".' % (prop_name, 81 | client_type)) 82 | for prop_name in VALID_CLIENT[client_type]['string']: 83 | if client_info[prop_name].startswith('[['): 84 | raise InvalidClientSecretsError( 85 | 'Property "%s" is not configured.' % prop_name) 86 | return client_type, client_info 87 | 88 | 89 | def load(fp): 90 | obj = simplejson.load(fp) 91 | return _validate_clientsecrets(obj) 92 | 93 | 94 | def loads(s): 95 | obj = simplejson.loads(s) 96 | return _validate_clientsecrets(obj) 97 | 98 | 99 | def _loadfile(filename): 100 | try: 101 | fp = file(filename, 'r') 102 | try: 103 | obj = simplejson.load(fp) 104 | finally: 105 | fp.close() 106 | except IOError: 107 | raise InvalidClientSecretsError('File not found: "%s"' % filename) 108 | return _validate_clientsecrets(obj) 109 | 110 | 111 | def loadfile(filename, cache=None): 112 | """Loading of client_secrets JSON file, optionally backed by a cache. 113 | 114 | Typical cache storage would be App Engine memcache service, 115 | but you can pass in any other cache client that implements 116 | these methods: 117 | - get(key, namespace=ns) 118 | - set(key, value, namespace=ns) 119 | 120 | Usage: 121 | # without caching 122 | client_type, client_info = loadfile('secrets.json') 123 | # using App Engine memcache service 124 | from google.appengine.api import memcache 125 | client_type, client_info = loadfile('secrets.json', cache=memcache) 126 | 127 | Args: 128 | filename: string, Path to a client_secrets.json file on a filesystem. 129 | cache: An optional cache service client that implements get() and set() 130 | methods. If not specified, the file is always being loaded from 131 | a filesystem. 132 | 133 | Raises: 134 | InvalidClientSecretsError: In case of a validation error or some 135 | I/O failure. Can happen only on cache miss. 136 | 137 | Returns: 138 | (client_type, client_info) tuple, as _loadfile() normally would. 139 | JSON contents is validated only during first load. Cache hits are not 140 | validated. 141 | """ 142 | _SECRET_NAMESPACE = 'oauth2client:secrets#ns' 143 | 144 | if not cache: 145 | return _loadfile(filename) 146 | 147 | obj = cache.get(filename, namespace=_SECRET_NAMESPACE) 148 | if obj is None: 149 | client_type, client_info = _loadfile(filename) 150 | obj = {client_type: client_info} 151 | cache.set(filename, obj, namespace=_SECRET_NAMESPACE) 152 | 153 | return obj.iteritems().next() 154 | -------------------------------------------------------------------------------- /libs/oauth2client/django_orm.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2010 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """OAuth 2.0 utilities for Django. 16 | 17 | Utilities for using OAuth 2.0 in conjunction with 18 | the Django datastore. 19 | """ 20 | 21 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 22 | 23 | import oauth2client 24 | import base64 25 | import pickle 26 | 27 | from django.db import models 28 | from oauth2client.client import Storage as BaseStorage 29 | 30 | class CredentialsField(models.Field): 31 | 32 | __metaclass__ = models.SubfieldBase 33 | 34 | def __init__(self, *args, **kwargs): 35 | if 'null' not in kwargs: 36 | kwargs['null'] = True 37 | super(CredentialsField, self).__init__(*args, **kwargs) 38 | 39 | def get_internal_type(self): 40 | return "TextField" 41 | 42 | def to_python(self, value): 43 | if value is None: 44 | return None 45 | if isinstance(value, oauth2client.client.Credentials): 46 | return value 47 | return pickle.loads(base64.b64decode(value)) 48 | 49 | def get_db_prep_value(self, value, connection, prepared=False): 50 | if value is None: 51 | return None 52 | return base64.b64encode(pickle.dumps(value)) 53 | 54 | 55 | class FlowField(models.Field): 56 | 57 | __metaclass__ = models.SubfieldBase 58 | 59 | def __init__(self, *args, **kwargs): 60 | if 'null' not in kwargs: 61 | kwargs['null'] = True 62 | super(FlowField, self).__init__(*args, **kwargs) 63 | 64 | def get_internal_type(self): 65 | return "TextField" 66 | 67 | def to_python(self, value): 68 | if value is None: 69 | return None 70 | if isinstance(value, oauth2client.client.Flow): 71 | return value 72 | return pickle.loads(base64.b64decode(value)) 73 | 74 | def get_db_prep_value(self, value, connection, prepared=False): 75 | if value is None: 76 | return None 77 | return base64.b64encode(pickle.dumps(value)) 78 | 79 | 80 | class Storage(BaseStorage): 81 | """Store and retrieve a single credential to and from 82 | the datastore. 83 | 84 | This Storage helper presumes the Credentials 85 | have been stored as a CredenialsField 86 | on a db model class. 87 | """ 88 | 89 | def __init__(self, model_class, key_name, key_value, property_name): 90 | """Constructor for Storage. 91 | 92 | Args: 93 | model: db.Model, model class 94 | key_name: string, key name for the entity that has the credentials 95 | key_value: string, key value for the entity that has the credentials 96 | property_name: string, name of the property that is an CredentialsProperty 97 | """ 98 | self.model_class = model_class 99 | self.key_name = key_name 100 | self.key_value = key_value 101 | self.property_name = property_name 102 | 103 | def locked_get(self): 104 | """Retrieve Credential from datastore. 105 | 106 | Returns: 107 | oauth2client.Credentials 108 | """ 109 | credential = None 110 | 111 | query = {self.key_name: self.key_value} 112 | entities = self.model_class.objects.filter(**query) 113 | if len(entities) > 0: 114 | credential = getattr(entities[0], self.property_name) 115 | if credential and hasattr(credential, 'set_store'): 116 | credential.set_store(self) 117 | return credential 118 | 119 | def locked_put(self, credentials): 120 | """Write a Credentials to the datastore. 121 | 122 | Args: 123 | credentials: Credentials, the credentials to store. 124 | """ 125 | args = {self.key_name: self.key_value} 126 | entity = self.model_class(**args) 127 | setattr(entity, self.property_name, credentials) 128 | entity.save() 129 | 130 | def locked_delete(self): 131 | """Delete Credentials from the datastore.""" 132 | 133 | query = {self.key_name: self.key_value} 134 | entities = self.model_class.objects.filter(**query).delete() 135 | -------------------------------------------------------------------------------- /libs/oauth2client/file.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2010 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities for OAuth. 16 | 17 | Utilities for making it easier to work with OAuth 2.0 18 | credentials. 19 | """ 20 | 21 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 22 | 23 | import os 24 | import stat 25 | import threading 26 | 27 | from anyjson import simplejson 28 | from client import Storage as BaseStorage 29 | from client import Credentials 30 | 31 | 32 | class CredentialsFileSymbolicLinkError(Exception): 33 | """Credentials files must not be symbolic links.""" 34 | 35 | 36 | class Storage(BaseStorage): 37 | """Store and retrieve a single credential to and from a file.""" 38 | 39 | def __init__(self, filename): 40 | self._filename = filename 41 | self._lock = threading.Lock() 42 | 43 | def _validate_file(self): 44 | if os.path.islink(self._filename): 45 | raise CredentialsFileSymbolicLinkError( 46 | 'File: %s is a symbolic link.' % self._filename) 47 | 48 | def acquire_lock(self): 49 | """Acquires any lock necessary to access this Storage. 50 | 51 | This lock is not reentrant.""" 52 | self._lock.acquire() 53 | 54 | def release_lock(self): 55 | """Release the Storage lock. 56 | 57 | Trying to release a lock that isn't held will result in a 58 | RuntimeError. 59 | """ 60 | self._lock.release() 61 | 62 | def locked_get(self): 63 | """Retrieve Credential from file. 64 | 65 | Returns: 66 | oauth2client.client.Credentials 67 | 68 | Raises: 69 | CredentialsFileSymbolicLinkError if the file is a symbolic link. 70 | """ 71 | credentials = None 72 | self._validate_file() 73 | try: 74 | f = open(self._filename, 'rb') 75 | content = f.read() 76 | f.close() 77 | except IOError: 78 | return credentials 79 | 80 | try: 81 | credentials = Credentials.new_from_json(content) 82 | credentials.set_store(self) 83 | except ValueError: 84 | pass 85 | 86 | return credentials 87 | 88 | def _create_file_if_needed(self): 89 | """Create an empty file if necessary. 90 | 91 | This method will not initialize the file. Instead it implements a 92 | simple version of "touch" to ensure the file has been created. 93 | """ 94 | if not os.path.exists(self._filename): 95 | old_umask = os.umask(0177) 96 | try: 97 | open(self._filename, 'a+b').close() 98 | finally: 99 | os.umask(old_umask) 100 | 101 | def locked_put(self, credentials): 102 | """Write Credentials to file. 103 | 104 | Args: 105 | credentials: Credentials, the credentials to store. 106 | 107 | Raises: 108 | CredentialsFileSymbolicLinkError if the file is a symbolic link. 109 | """ 110 | 111 | self._create_file_if_needed() 112 | self._validate_file() 113 | f = open(self._filename, 'wb') 114 | f.write(credentials.to_json()) 115 | f.close() 116 | 117 | def locked_delete(self): 118 | """Delete Credentials file. 119 | 120 | Args: 121 | credentials: Credentials, the credentials to store. 122 | """ 123 | 124 | os.unlink(self._filename) 125 | -------------------------------------------------------------------------------- /libs/oauth2client/gce.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities for Google Compute Engine 16 | 17 | Utilities for making it easier to use OAuth 2.0 on Google Compute Engine. 18 | """ 19 | 20 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 21 | 22 | import httplib2 23 | import logging 24 | import uritemplate 25 | 26 | from oauth2client import util 27 | from oauth2client.anyjson import simplejson 28 | from oauth2client.client import AccessTokenRefreshError 29 | from oauth2client.client import AssertionCredentials 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | # URI Template for the endpoint that returns access_tokens. 34 | META = ('http://metadata.google.internal/0.1/meta-data/service-accounts/' 35 | 'default/acquire{?scope}') 36 | 37 | 38 | class AppAssertionCredentials(AssertionCredentials): 39 | """Credentials object for Compute Engine Assertion Grants 40 | 41 | This object will allow a Compute Engine instance to identify itself to 42 | Google and other OAuth 2.0 servers that can verify assertions. It can be used 43 | for the purpose of accessing data stored under an account assigned to the 44 | Compute Engine instance itself. 45 | 46 | This credential does not require a flow to instantiate because it represents 47 | a two legged flow, and therefore has all of the required information to 48 | generate and refresh its own access tokens. 49 | """ 50 | 51 | @util.positional(2) 52 | def __init__(self, scope, **kwargs): 53 | """Constructor for AppAssertionCredentials 54 | 55 | Args: 56 | scope: string or iterable of strings, scope(s) of the credentials being 57 | requested. 58 | """ 59 | self.scope = util.scopes_to_string(scope) 60 | 61 | # Assertion type is no longer used, but still in the parent class signature. 62 | super(AppAssertionCredentials, self).__init__(None) 63 | 64 | @classmethod 65 | def from_json(cls, json): 66 | data = simplejson.loads(json) 67 | return AppAssertionCredentials(data['scope']) 68 | 69 | def _refresh(self, http_request): 70 | """Refreshes the access_token. 71 | 72 | Skip all the storage hoops and just refresh using the API. 73 | 74 | Args: 75 | http_request: callable, a callable that matches the method signature of 76 | httplib2.Http.request, used to make the refresh request. 77 | 78 | Raises: 79 | AccessTokenRefreshError: When the refresh fails. 80 | """ 81 | uri = uritemplate.expand(META, {'scope': self.scope}) 82 | response, content = http_request(uri) 83 | if response.status == 200: 84 | try: 85 | d = simplejson.loads(content) 86 | except StandardError, e: 87 | raise AccessTokenRefreshError(str(e)) 88 | self.access_token = d['accessToken'] 89 | else: 90 | raise AccessTokenRefreshError(content) 91 | -------------------------------------------------------------------------------- /libs/oauth2client/keyring_storage.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A keyring based Storage. 16 | 17 | A Storage for Credentials that uses the keyring module. 18 | """ 19 | 20 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' 21 | 22 | import keyring 23 | import threading 24 | 25 | from client import Storage as BaseStorage 26 | from client import Credentials 27 | 28 | 29 | class Storage(BaseStorage): 30 | """Store and retrieve a single credential to and from the keyring. 31 | 32 | To use this module you must have the keyring module installed. See 33 | . This is an optional module and is not 34 | installed with oauth2client by default because it does not work on all the 35 | platforms that oauth2client supports, such as Google App Engine. 36 | 37 | The keyring module is a cross-platform 38 | library for access the keyring capabilities of the local system. The user will 39 | be prompted for their keyring password when this module is used, and the 40 | manner in which the user is prompted will vary per platform. 41 | 42 | Usage: 43 | from oauth2client.keyring_storage import Storage 44 | 45 | s = Storage('name_of_application', 'user1') 46 | credentials = s.get() 47 | 48 | """ 49 | 50 | def __init__(self, service_name, user_name): 51 | """Constructor. 52 | 53 | Args: 54 | service_name: string, The name of the service under which the credentials 55 | are stored. 56 | user_name: string, The name of the user to store credentials for. 57 | """ 58 | self._service_name = service_name 59 | self._user_name = user_name 60 | self._lock = threading.Lock() 61 | 62 | def acquire_lock(self): 63 | """Acquires any lock necessary to access this Storage. 64 | 65 | This lock is not reentrant.""" 66 | self._lock.acquire() 67 | 68 | def release_lock(self): 69 | """Release the Storage lock. 70 | 71 | Trying to release a lock that isn't held will result in a 72 | RuntimeError. 73 | """ 74 | self._lock.release() 75 | 76 | def locked_get(self): 77 | """Retrieve Credential from file. 78 | 79 | Returns: 80 | oauth2client.client.Credentials 81 | """ 82 | credentials = None 83 | content = keyring.get_password(self._service_name, self._user_name) 84 | 85 | if content is not None: 86 | try: 87 | credentials = Credentials.new_from_json(content) 88 | credentials.set_store(self) 89 | except ValueError: 90 | pass 91 | 92 | return credentials 93 | 94 | def locked_put(self, credentials): 95 | """Write Credentials to file. 96 | 97 | Args: 98 | credentials: Credentials, the credentials to store. 99 | """ 100 | keyring.set_password(self._service_name, self._user_name, 101 | credentials.to_json()) 102 | 103 | def locked_delete(self): 104 | """Delete Credentials file. 105 | 106 | Args: 107 | credentials: Credentials, the credentials to store. 108 | """ 109 | keyring.set_password(self._service_name, self._user_name, '') 110 | -------------------------------------------------------------------------------- /libs/oauth2client/old_run.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """This module holds the old run() function which is deprecated, the 16 | tools.run_flow() function should be used in its place.""" 17 | 18 | 19 | import logging 20 | import socket 21 | import sys 22 | import webbrowser 23 | 24 | import gflags 25 | 26 | from oauth2client import client 27 | from oauth2client import util 28 | from tools import ClientRedirectHandler 29 | from tools import ClientRedirectServer 30 | 31 | 32 | FLAGS = gflags.FLAGS 33 | 34 | gflags.DEFINE_boolean('auth_local_webserver', True, 35 | ('Run a local web server to handle redirects during ' 36 | 'OAuth authorization.')) 37 | 38 | gflags.DEFINE_string('auth_host_name', 'localhost', 39 | ('Host name to use when running a local web server to ' 40 | 'handle redirects during OAuth authorization.')) 41 | 42 | gflags.DEFINE_multi_int('auth_host_port', [8080, 8090], 43 | ('Port to use when running a local web server to ' 44 | 'handle redirects during OAuth authorization.')) 45 | 46 | 47 | @util.positional(2) 48 | def run(flow, storage, http=None): 49 | """Core code for a command-line application. 50 | 51 | The run() function is called from your application and runs through all 52 | the steps to obtain credentials. It takes a Flow argument and attempts to 53 | open an authorization server page in the user's default web browser. The 54 | server asks the user to grant your application access to the user's data. 55 | If the user grants access, the run() function returns new credentials. The 56 | new credentials are also stored in the Storage argument, which updates the 57 | file associated with the Storage object. 58 | 59 | It presumes it is run from a command-line application and supports the 60 | following flags: 61 | 62 | --auth_host_name: Host name to use when running a local web server 63 | to handle redirects during OAuth authorization. 64 | (default: 'localhost') 65 | 66 | --auth_host_port: Port to use when running a local web server to handle 67 | redirects during OAuth authorization.; 68 | repeat this option to specify a list of values 69 | (default: '[8080, 8090]') 70 | (an integer) 71 | 72 | --[no]auth_local_webserver: Run a local web server to handle redirects 73 | during OAuth authorization. 74 | (default: 'true') 75 | 76 | Since it uses flags make sure to initialize the gflags module before 77 | calling run(). 78 | 79 | Args: 80 | flow: Flow, an OAuth 2.0 Flow to step through. 81 | storage: Storage, a Storage to store the credential in. 82 | http: An instance of httplib2.Http.request 83 | or something that acts like it. 84 | 85 | Returns: 86 | Credentials, the obtained credential. 87 | """ 88 | logging.warning('This function, oauth2client.tools.run(), and the use of ' 89 | 'the gflags library are deprecated and will be removed in a future ' 90 | 'version of the library.') 91 | if FLAGS.auth_local_webserver: 92 | success = False 93 | port_number = 0 94 | for port in FLAGS.auth_host_port: 95 | port_number = port 96 | try: 97 | httpd = ClientRedirectServer((FLAGS.auth_host_name, port), 98 | ClientRedirectHandler) 99 | except socket.error, e: 100 | pass 101 | else: 102 | success = True 103 | break 104 | FLAGS.auth_local_webserver = success 105 | if not success: 106 | print 'Failed to start a local webserver listening on either port 8080' 107 | print 'or port 9090. Please check your firewall settings and locally' 108 | print 'running programs that may be blocking or using those ports.' 109 | print 110 | print 'Falling back to --noauth_local_webserver and continuing with', 111 | print 'authorization.' 112 | print 113 | 114 | if FLAGS.auth_local_webserver: 115 | oauth_callback = 'http://%s:%s/' % (FLAGS.auth_host_name, port_number) 116 | else: 117 | oauth_callback = client.OOB_CALLBACK_URN 118 | flow.redirect_uri = oauth_callback 119 | authorize_url = flow.step1_get_authorize_url() 120 | 121 | if FLAGS.auth_local_webserver: 122 | webbrowser.open(authorize_url, new=1, autoraise=True) 123 | print 'Your browser has been opened to visit:' 124 | print 125 | print ' ' + authorize_url 126 | print 127 | print 'If your browser is on a different machine then exit and re-run' 128 | print 'this application with the command-line parameter ' 129 | print 130 | print ' --noauth_local_webserver' 131 | print 132 | else: 133 | print 'Go to the following link in your browser:' 134 | print 135 | print ' ' + authorize_url 136 | print 137 | 138 | code = None 139 | if FLAGS.auth_local_webserver: 140 | httpd.handle_request() 141 | if 'error' in httpd.query_params: 142 | sys.exit('Authentication request was rejected.') 143 | if 'code' in httpd.query_params: 144 | code = httpd.query_params['code'] 145 | else: 146 | print 'Failed to find "code" in the query parameters of the redirect.' 147 | sys.exit('Try running with --noauth_local_webserver.') 148 | else: 149 | code = raw_input('Enter verification code: ').strip() 150 | 151 | try: 152 | credential = flow.step2_exchange(code, http=http) 153 | except client.FlowExchangeError, e: 154 | sys.exit('Authentication has failed: %s' % e) 155 | 156 | storage.put(credential) 157 | credential.set_store(storage) 158 | print 'Authentication successful.' 159 | 160 | return credential 161 | -------------------------------------------------------------------------------- /libs/oauth2client/xsrfutil.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.5 2 | # 3 | # Copyright 2010 the Melange authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Helper methods for creating & verifying XSRF tokens.""" 18 | 19 | __authors__ = [ 20 | '"Doug Coker" ', 21 | '"Joe Gregorio" ', 22 | ] 23 | 24 | 25 | import base64 26 | import hmac 27 | import os # for urandom 28 | import time 29 | 30 | from oauth2client import util 31 | 32 | 33 | # Delimiter character 34 | DELIMITER = ':' 35 | 36 | # 1 hour in seconds 37 | DEFAULT_TIMEOUT_SECS = 1*60*60 38 | 39 | @util.positional(2) 40 | def generate_token(key, user_id, action_id="", when=None): 41 | """Generates a URL-safe token for the given user, action, time tuple. 42 | 43 | Args: 44 | key: secret key to use. 45 | user_id: the user ID of the authenticated user. 46 | action_id: a string identifier of the action they requested 47 | authorization for. 48 | when: the time in seconds since the epoch at which the user was 49 | authorized for this action. If not set the current time is used. 50 | 51 | Returns: 52 | A string XSRF protection token. 53 | """ 54 | when = when or int(time.time()) 55 | digester = hmac.new(key) 56 | digester.update(str(user_id)) 57 | digester.update(DELIMITER) 58 | digester.update(action_id) 59 | digester.update(DELIMITER) 60 | digester.update(str(when)) 61 | digest = digester.digest() 62 | 63 | token = base64.urlsafe_b64encode('%s%s%d' % (digest, 64 | DELIMITER, 65 | when)) 66 | return token 67 | 68 | 69 | @util.positional(3) 70 | def validate_token(key, token, user_id, action_id="", current_time=None): 71 | """Validates that the given token authorizes the user for the action. 72 | 73 | Tokens are invalid if the time of issue is too old or if the token 74 | does not match what generateToken outputs (i.e. the token was forged). 75 | 76 | Args: 77 | key: secret key to use. 78 | token: a string of the token generated by generateToken. 79 | user_id: the user ID of the authenticated user. 80 | action_id: a string identifier of the action they requested 81 | authorization for. 82 | 83 | Returns: 84 | A boolean - True if the user is authorized for the action, False 85 | otherwise. 86 | """ 87 | if not token: 88 | return False 89 | try: 90 | decoded = base64.urlsafe_b64decode(str(token)) 91 | token_time = long(decoded.split(DELIMITER)[-1]) 92 | except (TypeError, ValueError): 93 | return False 94 | if current_time is None: 95 | current_time = time.time() 96 | # If the token is too old it's not valid. 97 | if current_time - token_time > DEFAULT_TIMEOUT_SECS: 98 | return False 99 | 100 | # The given token should match the generated one with the same time. 101 | expected_token = generate_token(key, user_id, action_id=action_id, 102 | when=token_time) 103 | if len(token) != len(expected_token): 104 | return False 105 | 106 | # Perform constant time comparison to avoid timing attacks 107 | different = 0 108 | for x, y in zip(token, expected_token): 109 | different |= ord(x) ^ ord(y) 110 | if different: 111 | return False 112 | 113 | return True 114 | -------------------------------------------------------------------------------- /libs/requests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # __ 4 | # /__) _ _ _ _ _/ _ 5 | # / ( (- (/ (/ (- _) / _) 6 | # / 7 | 8 | """ 9 | requests HTTP library 10 | ~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | Requests is an HTTP library, written in Python, for human beings. Basic GET 13 | usage: 14 | 15 | >>> import requests 16 | >>> r = requests.get('http://python.org') 17 | >>> r.status_code 18 | 200 19 | >>> 'Python is a programming language' in r.content 20 | True 21 | 22 | ... or POST: 23 | 24 | >>> payload = dict(key1='value1', key2='value2') 25 | >>> r = requests.post("http://httpbin.org/post", data=payload) 26 | >>> print r.text 27 | { 28 | ... 29 | "form": { 30 | "key2": "value2", 31 | "key1": "value1" 32 | }, 33 | ... 34 | } 35 | 36 | The other HTTP methods are supported - see `requests.api`. Full documentation 37 | is at . 38 | 39 | :copyright: (c) 2013 by Kenneth Reitz. 40 | :license: Apache 2.0, see LICENSE for more details. 41 | 42 | """ 43 | 44 | __title__ = 'requests' 45 | __version__ = '2.0.0' 46 | __build__ = 0x020000 47 | __author__ = 'Kenneth Reitz' 48 | __license__ = 'Apache 2.0' 49 | __copyright__ = 'Copyright 2013 Kenneth Reitz' 50 | 51 | # Attempt to enable urllib3's SNI support, if possible 52 | try: 53 | from .packages.urllib3.contrib import pyopenssl 54 | pyopenssl.inject_into_urllib3() 55 | except ImportError: 56 | pass 57 | 58 | from . import utils 59 | from .models import Request, Response, PreparedRequest 60 | from .api import request, get, head, post, patch, put, delete, options 61 | from .sessions import session, Session 62 | from .status_codes import codes 63 | from .exceptions import ( 64 | RequestException, Timeout, URLRequired, 65 | TooManyRedirects, HTTPError, ConnectionError 66 | ) 67 | 68 | # Set default logging handler to avoid "No handler found" warnings. 69 | import logging 70 | try: # Python 2.7+ 71 | from logging import NullHandler 72 | except ImportError: 73 | class NullHandler(logging.Handler): 74 | def emit(self, record): 75 | pass 76 | 77 | logging.getLogger(__name__).addHandler(NullHandler()) 78 | -------------------------------------------------------------------------------- /libs/requests/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.api 5 | ~~~~~~~~~~~~ 6 | 7 | This module implements the Requests API. 8 | 9 | :copyright: (c) 2012 by Kenneth Reitz. 10 | :license: Apache2, see LICENSE for more details. 11 | 12 | """ 13 | 14 | from . import sessions 15 | 16 | 17 | def request(method, url, **kwargs): 18 | """Constructs and sends a :class:`Request `. 19 | Returns :class:`Response ` object. 20 | 21 | :param method: method for the new :class:`Request` object. 22 | :param url: URL for the new :class:`Request` object. 23 | :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. 24 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 25 | :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. 26 | :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. 27 | :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload. 28 | :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. 29 | :param timeout: (optional) Float describing the timeout of the request. 30 | :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. 31 | :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. 32 | :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. 33 | :param stream: (optional) if ``False``, the response content will be immediately downloaded. 34 | :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. 35 | 36 | Usage:: 37 | 38 | >>> import requests 39 | >>> req = requests.request('GET', 'http://httpbin.org/get') 40 | 41 | """ 42 | 43 | session = sessions.Session() 44 | return session.request(method=method, url=url, **kwargs) 45 | 46 | 47 | def get(url, **kwargs): 48 | """Sends a GET request. Returns :class:`Response` object. 49 | 50 | :param url: URL for the new :class:`Request` object. 51 | :param \*\*kwargs: Optional arguments that ``request`` takes. 52 | """ 53 | 54 | kwargs.setdefault('allow_redirects', True) 55 | return request('get', url, **kwargs) 56 | 57 | 58 | def options(url, **kwargs): 59 | """Sends a OPTIONS request. Returns :class:`Response` object. 60 | 61 | :param url: URL for the new :class:`Request` object. 62 | :param \*\*kwargs: Optional arguments that ``request`` takes. 63 | """ 64 | 65 | kwargs.setdefault('allow_redirects', True) 66 | return request('options', url, **kwargs) 67 | 68 | 69 | def head(url, **kwargs): 70 | """Sends a HEAD request. Returns :class:`Response` object. 71 | 72 | :param url: URL for the new :class:`Request` object. 73 | :param \*\*kwargs: Optional arguments that ``request`` takes. 74 | """ 75 | 76 | kwargs.setdefault('allow_redirects', False) 77 | return request('head', url, **kwargs) 78 | 79 | 80 | def post(url, data=None, **kwargs): 81 | """Sends a POST request. Returns :class:`Response` object. 82 | 83 | :param url: URL for the new :class:`Request` object. 84 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 85 | :param \*\*kwargs: Optional arguments that ``request`` takes. 86 | """ 87 | 88 | return request('post', url, data=data, **kwargs) 89 | 90 | 91 | def put(url, data=None, **kwargs): 92 | """Sends a PUT request. Returns :class:`Response` object. 93 | 94 | :param url: URL for the new :class:`Request` object. 95 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 96 | :param \*\*kwargs: Optional arguments that ``request`` takes. 97 | """ 98 | 99 | return request('put', url, data=data, **kwargs) 100 | 101 | 102 | def patch(url, data=None, **kwargs): 103 | """Sends a PATCH request. Returns :class:`Response` object. 104 | 105 | :param url: URL for the new :class:`Request` object. 106 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 107 | :param \*\*kwargs: Optional arguments that ``request`` takes. 108 | """ 109 | 110 | return request('patch', url, data=data, **kwargs) 111 | 112 | 113 | def delete(url, **kwargs): 114 | """Sends a DELETE request. Returns :class:`Response` object. 115 | 116 | :param url: URL for the new :class:`Request` object. 117 | :param \*\*kwargs: Optional arguments that ``request`` takes. 118 | """ 119 | 120 | return request('delete', url, **kwargs) 121 | -------------------------------------------------------------------------------- /libs/requests/auth.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.auth 5 | ~~~~~~~~~~~~~ 6 | 7 | This module contains the authentication handlers for Requests. 8 | """ 9 | 10 | import os 11 | import re 12 | import time 13 | import hashlib 14 | import logging 15 | 16 | from base64 import b64encode 17 | 18 | from .compat import urlparse, str 19 | from .utils import parse_dict_header 20 | 21 | log = logging.getLogger(__name__) 22 | 23 | CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' 24 | CONTENT_TYPE_MULTI_PART = 'multipart/form-data' 25 | 26 | 27 | def _basic_auth_str(username, password): 28 | """Returns a Basic Auth string.""" 29 | 30 | return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1') 31 | 32 | 33 | class AuthBase(object): 34 | """Base class that all auth implementations derive from""" 35 | 36 | def __call__(self, r): 37 | raise NotImplementedError('Auth hooks must be callable.') 38 | 39 | 40 | class HTTPBasicAuth(AuthBase): 41 | """Attaches HTTP Basic Authentication to the given Request object.""" 42 | def __init__(self, username, password): 43 | self.username = username 44 | self.password = password 45 | 46 | def __call__(self, r): 47 | r.headers['Authorization'] = _basic_auth_str(self.username, self.password) 48 | return r 49 | 50 | 51 | class HTTPProxyAuth(HTTPBasicAuth): 52 | """Attaches HTTP Proxy Authentication to a given Request object.""" 53 | def __call__(self, r): 54 | r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password) 55 | return r 56 | 57 | 58 | class HTTPDigestAuth(AuthBase): 59 | """Attaches HTTP Digest Authentication to the given Request object.""" 60 | def __init__(self, username, password): 61 | self.username = username 62 | self.password = password 63 | self.last_nonce = '' 64 | self.nonce_count = 0 65 | self.chal = {} 66 | 67 | def build_digest_header(self, method, url): 68 | 69 | realm = self.chal['realm'] 70 | nonce = self.chal['nonce'] 71 | qop = self.chal.get('qop') 72 | algorithm = self.chal.get('algorithm') 73 | opaque = self.chal.get('opaque') 74 | 75 | if algorithm is None: 76 | _algorithm = 'MD5' 77 | else: 78 | _algorithm = algorithm.upper() 79 | # lambdas assume digest modules are imported at the top level 80 | if _algorithm == 'MD5': 81 | def md5_utf8(x): 82 | if isinstance(x, str): 83 | x = x.encode('utf-8') 84 | return hashlib.md5(x).hexdigest() 85 | hash_utf8 = md5_utf8 86 | elif _algorithm == 'SHA': 87 | def sha_utf8(x): 88 | if isinstance(x, str): 89 | x = x.encode('utf-8') 90 | return hashlib.sha1(x).hexdigest() 91 | hash_utf8 = sha_utf8 92 | # XXX MD5-sess 93 | KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) 94 | 95 | if hash_utf8 is None: 96 | return None 97 | 98 | # XXX not implemented yet 99 | entdig = None 100 | p_parsed = urlparse(url) 101 | path = p_parsed.path 102 | if p_parsed.query: 103 | path += '?' + p_parsed.query 104 | 105 | A1 = '%s:%s:%s' % (self.username, realm, self.password) 106 | A2 = '%s:%s' % (method, path) 107 | 108 | if qop is None: 109 | respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2))) 110 | elif qop == 'auth' or 'auth' in qop.split(','): 111 | if nonce == self.last_nonce: 112 | self.nonce_count += 1 113 | else: 114 | self.nonce_count = 1 115 | 116 | ncvalue = '%08x' % self.nonce_count 117 | s = str(self.nonce_count).encode('utf-8') 118 | s += nonce.encode('utf-8') 119 | s += time.ctime().encode('utf-8') 120 | s += os.urandom(8) 121 | 122 | cnonce = (hashlib.sha1(s).hexdigest()[:16]) 123 | noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2)) 124 | respdig = KD(hash_utf8(A1), noncebit) 125 | else: 126 | # XXX handle auth-int. 127 | return None 128 | 129 | self.last_nonce = nonce 130 | 131 | # XXX should the partial digests be encoded too? 132 | base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ 133 | 'response="%s"' % (self.username, realm, nonce, path, respdig) 134 | if opaque: 135 | base += ', opaque="%s"' % opaque 136 | if algorithm: 137 | base += ', algorithm="%s"' % algorithm 138 | if entdig: 139 | base += ', digest="%s"' % entdig 140 | if qop: 141 | base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce) 142 | 143 | return 'Digest %s' % (base) 144 | 145 | def handle_401(self, r, **kwargs): 146 | """Takes the given response and tries digest-auth, if needed.""" 147 | 148 | num_401_calls = getattr(self, 'num_401_calls', 1) 149 | s_auth = r.headers.get('www-authenticate', '') 150 | 151 | if 'digest' in s_auth.lower() and num_401_calls < 2: 152 | 153 | setattr(self, 'num_401_calls', num_401_calls + 1) 154 | pat = re.compile(r'digest ', flags=re.IGNORECASE) 155 | self.chal = parse_dict_header(pat.sub('', s_auth, count=1)) 156 | 157 | # Consume content and release the original connection 158 | # to allow our new request to reuse the same one. 159 | r.content 160 | r.raw.release_conn() 161 | prep = r.request.copy() 162 | prep.prepare_cookies(r.cookies) 163 | 164 | prep.headers['Authorization'] = self.build_digest_header( 165 | prep.method, prep.url) 166 | _r = r.connection.send(prep, **kwargs) 167 | _r.history.append(r) 168 | _r.request = prep 169 | 170 | return _r 171 | 172 | setattr(self, 'num_401_calls', 1) 173 | return r 174 | 175 | def __call__(self, r): 176 | # If we have a saved nonce, skip the 401 177 | if self.last_nonce: 178 | r.headers['Authorization'] = self.build_digest_header(r.method, r.url) 179 | r.register_hook('response', self.handle_401) 180 | return r 181 | -------------------------------------------------------------------------------- /libs/requests/certs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | certs.py 6 | ~~~~~~~~ 7 | 8 | This module returns the preferred default CA certificate bundle. 9 | 10 | If you are packaging Requests, e.g., for a Linux distribution or a managed 11 | environment, you can change the definition of where() to return a separately 12 | packaged CA bundle. 13 | """ 14 | 15 | import os.path 16 | 17 | 18 | def where(): 19 | """Return the preferred certificate bundle.""" 20 | # vendored bundle inside Requests 21 | return os.path.join(os.path.dirname(__file__), 'cacert.pem') 22 | 23 | if __name__ == '__main__': 24 | print(where()) 25 | -------------------------------------------------------------------------------- /libs/requests/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | pythoncompat 5 | """ 6 | 7 | from .packages import charade as chardet 8 | 9 | import sys 10 | 11 | # ------- 12 | # Pythons 13 | # ------- 14 | 15 | # Syntax sugar. 16 | _ver = sys.version_info 17 | 18 | #: Python 2.x? 19 | is_py2 = (_ver[0] == 2) 20 | 21 | #: Python 3.x? 22 | is_py3 = (_ver[0] == 3) 23 | 24 | #: Python 3.0.x 25 | is_py30 = (is_py3 and _ver[1] == 0) 26 | 27 | #: Python 3.1.x 28 | is_py31 = (is_py3 and _ver[1] == 1) 29 | 30 | #: Python 3.2.x 31 | is_py32 = (is_py3 and _ver[1] == 2) 32 | 33 | #: Python 3.3.x 34 | is_py33 = (is_py3 and _ver[1] == 3) 35 | 36 | #: Python 3.4.x 37 | is_py34 = (is_py3 and _ver[1] == 4) 38 | 39 | #: Python 2.7.x 40 | is_py27 = (is_py2 and _ver[1] == 7) 41 | 42 | #: Python 2.6.x 43 | is_py26 = (is_py2 and _ver[1] == 6) 44 | 45 | #: Python 2.5.x 46 | is_py25 = (is_py2 and _ver[1] == 5) 47 | 48 | #: Python 2.4.x 49 | is_py24 = (is_py2 and _ver[1] == 4) # I'm assuming this is not by choice. 50 | 51 | 52 | # --------- 53 | # Platforms 54 | # --------- 55 | 56 | 57 | # Syntax sugar. 58 | _ver = sys.version.lower() 59 | 60 | is_pypy = ('pypy' in _ver) 61 | is_jython = ('jython' in _ver) 62 | is_ironpython = ('iron' in _ver) 63 | 64 | # Assume CPython, if nothing else. 65 | is_cpython = not any((is_pypy, is_jython, is_ironpython)) 66 | 67 | # Windows-based system. 68 | is_windows = 'win32' in str(sys.platform).lower() 69 | 70 | # Standard Linux 2+ system. 71 | is_linux = ('linux' in str(sys.platform).lower()) 72 | is_osx = ('darwin' in str(sys.platform).lower()) 73 | is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess. 74 | is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess. 75 | 76 | try: 77 | import simplejson as json 78 | except ImportError: 79 | import json 80 | 81 | # --------- 82 | # Specifics 83 | # --------- 84 | 85 | if is_py2: 86 | from urllib import quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, proxy_bypass 87 | from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag 88 | from urllib2 import parse_http_list 89 | import cookielib 90 | from Cookie import Morsel 91 | from StringIO import StringIO 92 | from .packages.urllib3.packages.ordered_dict import OrderedDict 93 | from httplib import IncompleteRead 94 | 95 | builtin_str = str 96 | bytes = str 97 | str = unicode 98 | basestring = basestring 99 | numeric_types = (int, long, float) 100 | 101 | 102 | elif is_py3: 103 | from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag 104 | from urllib.request import parse_http_list, getproxies, proxy_bypass 105 | from http import cookiejar as cookielib 106 | from http.cookies import Morsel 107 | from io import StringIO 108 | from collections import OrderedDict 109 | from http.client import IncompleteRead 110 | 111 | builtin_str = str 112 | str = str 113 | bytes = bytes 114 | basestring = (str, bytes) 115 | numeric_types = (int, float) 116 | -------------------------------------------------------------------------------- /libs/requests/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.exceptions 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | This module contains the set of Requests' exceptions. 8 | 9 | """ 10 | 11 | 12 | class RequestException(IOError): 13 | """There was an ambiguous exception that occurred while handling your 14 | request.""" 15 | 16 | 17 | class HTTPError(RequestException): 18 | """An HTTP error occurred.""" 19 | 20 | def __init__(self, *args, **kwargs): 21 | """ Initializes HTTPError with optional `response` object. """ 22 | self.response = kwargs.pop('response', None) 23 | super(HTTPError, self).__init__(*args, **kwargs) 24 | 25 | 26 | class ConnectionError(RequestException): 27 | """A Connection error occurred.""" 28 | 29 | 30 | class SSLError(ConnectionError): 31 | """An SSL error occurred.""" 32 | 33 | 34 | class Timeout(RequestException): 35 | """The request timed out.""" 36 | 37 | 38 | class URLRequired(RequestException): 39 | """A valid URL is required to make a request.""" 40 | 41 | 42 | class TooManyRedirects(RequestException): 43 | """Too many redirects.""" 44 | 45 | 46 | class MissingSchema(RequestException, ValueError): 47 | """The URL schema (e.g. http or https) is missing.""" 48 | 49 | 50 | class InvalidSchema(RequestException, ValueError): 51 | """See defaults.py for valid schemas.""" 52 | 53 | 54 | class InvalidURL(RequestException, ValueError): 55 | """ The URL provided was somehow invalid. """ 56 | 57 | 58 | class ChunkedEncodingError(RequestException): 59 | """The server declared chunked encoding but sent an invalid chunk.""" 60 | -------------------------------------------------------------------------------- /libs/requests/hooks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.hooks 5 | ~~~~~~~~~~~~~~ 6 | 7 | This module provides the capabilities for the Requests hooks system. 8 | 9 | Available hooks: 10 | 11 | ``response``: 12 | The response generated from a Request. 13 | 14 | """ 15 | 16 | 17 | HOOKS = ['response'] 18 | 19 | 20 | def default_hooks(): 21 | hooks = {} 22 | for event in HOOKS: 23 | hooks[event] = [] 24 | return hooks 25 | 26 | # TODO: response is the only one 27 | 28 | 29 | def dispatch_hook(key, hooks, hook_data, **kwargs): 30 | """Dispatches a hook dictionary on a given piece of data.""" 31 | 32 | hooks = hooks or dict() 33 | 34 | if key in hooks: 35 | hooks = hooks.get(key) 36 | 37 | if hasattr(hooks, '__call__'): 38 | hooks = [hooks] 39 | 40 | for hook in hooks: 41 | _hook_data = hook(hook_data, **kwargs) 42 | if _hook_data is not None: 43 | hook_data = _hook_data 44 | 45 | return hook_data 46 | -------------------------------------------------------------------------------- /libs/requests/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import urllib3 4 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/__init__.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # This library is free software; you can redistribute it and/or 3 | # modify it under the terms of the GNU Lesser General Public 4 | # License as published by the Free Software Foundation; either 5 | # version 2.1 of the License, or (at your option) any later version. 6 | # 7 | # This library is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 | # Lesser General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU Lesser General Public 13 | # License along with this library; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 15 | # 02110-1301 USA 16 | ######################### END LICENSE BLOCK ######################### 17 | 18 | __version__ = "1.0.3" 19 | from sys import version_info 20 | 21 | 22 | def detect(aBuf): 23 | if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or 24 | (version_info >= (3, 0) and not isinstance(aBuf, bytes))): 25 | raise ValueError('Expected a bytes object, not a unicode object') 26 | 27 | from . import universaldetector 28 | u = universaldetector.UniversalDetector() 29 | u.reset() 30 | u.feed(aBuf) 31 | u.close() 32 | return u.result 33 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/big5prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import Big5DistributionAnalysis 31 | from .mbcssm import Big5SMModel 32 | 33 | 34 | class Big5Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(Big5SMModel) 38 | self._mDistributionAnalyzer = Big5DistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "Big5" 43 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/charsetgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | import sys 30 | from .charsetprober import CharSetProber 31 | 32 | 33 | class CharSetGroupProber(CharSetProber): 34 | def __init__(self): 35 | CharSetProber.__init__(self) 36 | self._mActiveNum = 0 37 | self._mProbers = [] 38 | self._mBestGuessProber = None 39 | 40 | def reset(self): 41 | CharSetProber.reset(self) 42 | self._mActiveNum = 0 43 | for prober in self._mProbers: 44 | if prober: 45 | prober.reset() 46 | prober.active = True 47 | self._mActiveNum += 1 48 | self._mBestGuessProber = None 49 | 50 | def get_charset_name(self): 51 | if not self._mBestGuessProber: 52 | self.get_confidence() 53 | if not self._mBestGuessProber: 54 | return None 55 | # self._mBestGuessProber = self._mProbers[0] 56 | return self._mBestGuessProber.get_charset_name() 57 | 58 | def feed(self, aBuf): 59 | for prober in self._mProbers: 60 | if not prober: 61 | continue 62 | if not prober.active: 63 | continue 64 | st = prober.feed(aBuf) 65 | if not st: 66 | continue 67 | if st == constants.eFoundIt: 68 | self._mBestGuessProber = prober 69 | return self.get_state() 70 | elif st == constants.eNotMe: 71 | prober.active = False 72 | self._mActiveNum -= 1 73 | if self._mActiveNum <= 0: 74 | self._mState = constants.eNotMe 75 | return self.get_state() 76 | return self.get_state() 77 | 78 | def get_confidence(self): 79 | st = self.get_state() 80 | if st == constants.eFoundIt: 81 | return 0.99 82 | elif st == constants.eNotMe: 83 | return 0.01 84 | bestConf = 0.0 85 | self._mBestGuessProber = None 86 | for prober in self._mProbers: 87 | if not prober: 88 | continue 89 | if not prober.active: 90 | if constants._debug: 91 | sys.stderr.write(prober.get_charset_name() 92 | + ' not active\n') 93 | continue 94 | cf = prober.get_confidence() 95 | if constants._debug: 96 | sys.stderr.write('%s confidence = %s\n' % 97 | (prober.get_charset_name(), cf)) 98 | if bestConf < cf: 99 | bestConf = cf 100 | self._mBestGuessProber = prober 101 | if not self._mBestGuessProber: 102 | return 0.0 103 | return bestConf 104 | # else: 105 | # self._mBestGuessProber = self._mProbers[0] 106 | # return self._mBestGuessProber.get_confidence() 107 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/charsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from . import constants 30 | import re 31 | 32 | 33 | class CharSetProber: 34 | def __init__(self): 35 | pass 36 | 37 | def reset(self): 38 | self._mState = constants.eDetecting 39 | 40 | def get_charset_name(self): 41 | return None 42 | 43 | def feed(self, aBuf): 44 | pass 45 | 46 | def get_state(self): 47 | return self._mState 48 | 49 | def get_confidence(self): 50 | return 0.0 51 | 52 | def filter_high_bit_only(self, aBuf): 53 | aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf) 54 | return aBuf 55 | 56 | def filter_without_english_letters(self, aBuf): 57 | aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf) 58 | return aBuf 59 | 60 | def filter_with_english_letters(self, aBuf): 61 | # TODO 62 | return aBuf 63 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/codingstatemachine.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .constants import eStart 29 | from .compat import wrap_ord 30 | 31 | 32 | class CodingStateMachine: 33 | def __init__(self, sm): 34 | self._mModel = sm 35 | self._mCurrentBytePos = 0 36 | self._mCurrentCharLen = 0 37 | self.reset() 38 | 39 | def reset(self): 40 | self._mCurrentState = eStart 41 | 42 | def next_state(self, c): 43 | # for each byte we get its class 44 | # if it is first byte, we also get byte length 45 | # PY3K: aBuf is a byte stream, so c is an int, not a byte 46 | byteCls = self._mModel['classTable'][wrap_ord(c)] 47 | if self._mCurrentState == eStart: 48 | self._mCurrentBytePos = 0 49 | self._mCurrentCharLen = self._mModel['charLenTable'][byteCls] 50 | # from byte's class and stateTable, we get its next state 51 | curr_state = (self._mCurrentState * self._mModel['classFactor'] 52 | + byteCls) 53 | self._mCurrentState = self._mModel['stateTable'][curr_state] 54 | self._mCurrentBytePos += 1 55 | return self._mCurrentState 56 | 57 | def get_current_charlen(self): 58 | return self._mCurrentCharLen 59 | 60 | def get_coding_state_machine(self): 61 | return self._mModel['name'] 62 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/compat.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # Contributor(s): 3 | # Ian Cordasco - port to Python 4 | # 5 | # This library is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU Lesser General Public 7 | # License as published by the Free Software Foundation; either 8 | # version 2.1 of the License, or (at your option) any later version. 9 | # 10 | # This library is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public 16 | # License along with this library; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 18 | # 02110-1301 USA 19 | ######################### END LICENSE BLOCK ######################### 20 | 21 | import sys 22 | 23 | 24 | if sys.version_info < (3, 0): 25 | base_str = (str, unicode) 26 | else: 27 | base_str = (bytes, str) 28 | 29 | 30 | def wrap_ord(a): 31 | if sys.version_info < (3, 0) and isinstance(a, base_str): 32 | return ord(a) 33 | else: 34 | return a 35 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/constants.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | _debug = 0 30 | 31 | eDetecting = 0 32 | eFoundIt = 1 33 | eNotMe = 2 34 | 35 | eStart = 0 36 | eError = 1 37 | eItsMe = 2 38 | 39 | SHORTCUT_THRESHOLD = 0.95 40 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/cp949prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import CP949SMModel 32 | 33 | 34 | class CP949Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(CP949SMModel) 38 | # NOTE: CP949 is a superset of EUC-KR, so the distribution should be 39 | # not different. 40 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 41 | self.reset() 42 | 43 | def get_charset_name(self): 44 | return "CP949" 45 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/escprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, 30 | ISO2022KRSMModel) 31 | from .charsetprober import CharSetProber 32 | from .codingstatemachine import CodingStateMachine 33 | from .compat import wrap_ord 34 | 35 | 36 | class EscCharSetProber(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = [ 40 | CodingStateMachine(HZSMModel), 41 | CodingStateMachine(ISO2022CNSMModel), 42 | CodingStateMachine(ISO2022JPSMModel), 43 | CodingStateMachine(ISO2022KRSMModel) 44 | ] 45 | self.reset() 46 | 47 | def reset(self): 48 | CharSetProber.reset(self) 49 | for codingSM in self._mCodingSM: 50 | if not codingSM: 51 | continue 52 | codingSM.active = True 53 | codingSM.reset() 54 | self._mActiveSM = len(self._mCodingSM) 55 | self._mDetectedCharset = None 56 | 57 | def get_charset_name(self): 58 | return self._mDetectedCharset 59 | 60 | def get_confidence(self): 61 | if self._mDetectedCharset: 62 | return 0.99 63 | else: 64 | return 0.00 65 | 66 | def feed(self, aBuf): 67 | for c in aBuf: 68 | # PY3K: aBuf is a byte array, so c is an int, not a byte 69 | for codingSM in self._mCodingSM: 70 | if not codingSM: 71 | continue 72 | if not codingSM.active: 73 | continue 74 | codingState = codingSM.next_state(wrap_ord(c)) 75 | if codingState == constants.eError: 76 | codingSM.active = False 77 | self._mActiveSM -= 1 78 | if self._mActiveSM <= 0: 79 | self._mState = constants.eNotMe 80 | return self.get_state() 81 | elif codingState == constants.eItsMe: 82 | self._mState = constants.eFoundIt 83 | self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8 84 | return self.get_state() 85 | 86 | return self.get_state() 87 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/eucjpprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from . import constants 30 | from .mbcharsetprober import MultiByteCharSetProber 31 | from .codingstatemachine import CodingStateMachine 32 | from .chardistribution import EUCJPDistributionAnalysis 33 | from .jpcntx import EUCJPContextAnalysis 34 | from .mbcssm import EUCJPSMModel 35 | 36 | 37 | class EUCJPProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(EUCJPSMModel) 41 | self._mDistributionAnalyzer = EUCJPDistributionAnalysis() 42 | self._mContextAnalyzer = EUCJPContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "EUC-JP" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mContextAnalyzer.feed(self._mLastChar, charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen) 75 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 76 | charLen) 77 | 78 | self._mLastChar[0] = aBuf[aLen - 1] 79 | 80 | if self.get_state() == constants.eDetecting: 81 | if (self._mContextAnalyzer.got_enough_data() and 82 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 83 | self._mState = constants.eFoundIt 84 | 85 | return self.get_state() 86 | 87 | def get_confidence(self): 88 | contxtCf = self._mContextAnalyzer.get_confidence() 89 | distribCf = self._mDistributionAnalyzer.get_confidence() 90 | return max(contxtCf, distribCf) 91 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/euckrprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import EUCKRSMModel 32 | 33 | 34 | class EUCKRProber(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(EUCKRSMModel) 38 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "EUC-KR" 43 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/euctwprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCTWDistributionAnalysis 31 | from .mbcssm import EUCTWSMModel 32 | 33 | class EUCTWProber(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(EUCTWSMModel) 37 | self._mDistributionAnalyzer = EUCTWDistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "EUC-TW" 42 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/gb2312prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import GB2312DistributionAnalysis 31 | from .mbcssm import GB2312SMModel 32 | 33 | class GB2312Prober(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(GB2312SMModel) 37 | self._mDistributionAnalyzer = GB2312DistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "GB2312" 42 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/latin1prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetprober import CharSetProber 30 | from .constants import eNotMe 31 | from .compat import wrap_ord 32 | 33 | FREQ_CAT_NUM = 4 34 | 35 | UDF = 0 # undefined 36 | OTH = 1 # other 37 | ASC = 2 # ascii capital letter 38 | ASS = 3 # ascii small letter 39 | ACV = 4 # accent capital vowel 40 | ACO = 5 # accent capital other 41 | ASV = 6 # accent small vowel 42 | ASO = 7 # accent small other 43 | CLASS_NUM = 8 # total classes 44 | 45 | Latin1_CharToClass = ( 46 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 47 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F 48 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 49 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F 50 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 51 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F 52 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 53 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F 54 | OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 55 | ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F 56 | ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 57 | ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F 58 | OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 59 | ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F 60 | ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 61 | ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F 62 | OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 63 | OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F 64 | UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 65 | OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F 66 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 67 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF 68 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 69 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF 70 | ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 71 | ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF 72 | ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 73 | ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF 74 | ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 75 | ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF 76 | ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 77 | ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF 78 | ) 79 | 80 | # 0 : illegal 81 | # 1 : very unlikely 82 | # 2 : normal 83 | # 3 : very likely 84 | Latin1ClassModel = ( 85 | # UDF OTH ASC ASS ACV ACO ASV ASO 86 | 0, 0, 0, 0, 0, 0, 0, 0, # UDF 87 | 0, 3, 3, 3, 3, 3, 3, 3, # OTH 88 | 0, 3, 3, 3, 3, 3, 3, 3, # ASC 89 | 0, 3, 3, 3, 1, 1, 3, 3, # ASS 90 | 0, 3, 3, 3, 1, 2, 1, 2, # ACV 91 | 0, 3, 3, 3, 3, 3, 3, 3, # ACO 92 | 0, 3, 1, 3, 1, 1, 1, 3, # ASV 93 | 0, 3, 1, 3, 1, 1, 3, 3, # ASO 94 | ) 95 | 96 | 97 | class Latin1Prober(CharSetProber): 98 | def __init__(self): 99 | CharSetProber.__init__(self) 100 | self.reset() 101 | 102 | def reset(self): 103 | self._mLastCharClass = OTH 104 | self._mFreqCounter = [0] * FREQ_CAT_NUM 105 | CharSetProber.reset(self) 106 | 107 | def get_charset_name(self): 108 | return "windows-1252" 109 | 110 | def feed(self, aBuf): 111 | aBuf = self.filter_with_english_letters(aBuf) 112 | for c in aBuf: 113 | charClass = Latin1_CharToClass[wrap_ord(c)] 114 | freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) 115 | + charClass] 116 | if freq == 0: 117 | self._mState = eNotMe 118 | break 119 | self._mFreqCounter[freq] += 1 120 | self._mLastCharClass = charClass 121 | 122 | return self.get_state() 123 | 124 | def get_confidence(self): 125 | if self.get_state() == eNotMe: 126 | return 0.01 127 | 128 | total = sum(self._mFreqCounter) 129 | if total < 0.01: 130 | confidence = 0.0 131 | else: 132 | confidence = ((self._mFreqCounter[3] / total) 133 | - (self._mFreqCounter[1] * 20.0 / total)) 134 | if confidence < 0.0: 135 | confidence = 0.0 136 | # lower the confidence of latin1 so that other more accurate 137 | # detector can take priority. 138 | confidence = confidence * 0.5 139 | return confidence 140 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/mbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | import sys 31 | from . import constants 32 | from .charsetprober import CharSetProber 33 | 34 | 35 | class MultiByteCharSetProber(CharSetProber): 36 | def __init__(self): 37 | CharSetProber.__init__(self) 38 | self._mDistributionAnalyzer = None 39 | self._mCodingSM = None 40 | self._mLastChar = [0, 0] 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | if self._mCodingSM: 45 | self._mCodingSM.reset() 46 | if self._mDistributionAnalyzer: 47 | self._mDistributionAnalyzer.reset() 48 | self._mLastChar = [0, 0] 49 | 50 | def get_charset_name(self): 51 | pass 52 | 53 | def feed(self, aBuf): 54 | aLen = len(aBuf) 55 | for i in range(0, aLen): 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 72 | else: 73 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 74 | charLen) 75 | 76 | self._mLastChar[0] = aBuf[aLen - 1] 77 | 78 | if self.get_state() == constants.eDetecting: 79 | if (self._mDistributionAnalyzer.got_enough_data() and 80 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 81 | self._mState = constants.eFoundIt 82 | 83 | return self.get_state() 84 | 85 | def get_confidence(self): 86 | return self._mDistributionAnalyzer.get_confidence() 87 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/mbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | from .charsetgroupprober import CharSetGroupProber 31 | from .utf8prober import UTF8Prober 32 | from .sjisprober import SJISProber 33 | from .eucjpprober import EUCJPProber 34 | from .gb2312prober import GB2312Prober 35 | from .euckrprober import EUCKRProber 36 | from .cp949prober import CP949Prober 37 | from .big5prober import Big5Prober 38 | from .euctwprober import EUCTWProber 39 | 40 | 41 | class MBCSGroupProber(CharSetGroupProber): 42 | def __init__(self): 43 | CharSetGroupProber.__init__(self) 44 | self._mProbers = [ 45 | UTF8Prober(), 46 | SJISProber(), 47 | EUCJPProber(), 48 | GB2312Prober(), 49 | EUCKRProber(), 50 | CP949Prober(), 51 | Big5Prober(), 52 | EUCTWProber() 53 | ] 54 | self.reset() 55 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/sbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | import sys 30 | from . import constants 31 | from .charsetprober import CharSetProber 32 | from .compat import wrap_ord 33 | 34 | SAMPLE_SIZE = 64 35 | SB_ENOUGH_REL_THRESHOLD = 1024 36 | POSITIVE_SHORTCUT_THRESHOLD = 0.95 37 | NEGATIVE_SHORTCUT_THRESHOLD = 0.05 38 | SYMBOL_CAT_ORDER = 250 39 | NUMBER_OF_SEQ_CAT = 4 40 | POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1 41 | #NEGATIVE_CAT = 0 42 | 43 | 44 | class SingleByteCharSetProber(CharSetProber): 45 | def __init__(self, model, reversed=False, nameProber=None): 46 | CharSetProber.__init__(self) 47 | self._mModel = model 48 | # TRUE if we need to reverse every pair in the model lookup 49 | self._mReversed = reversed 50 | # Optional auxiliary prober for name decision 51 | self._mNameProber = nameProber 52 | self.reset() 53 | 54 | def reset(self): 55 | CharSetProber.reset(self) 56 | # char order of last character 57 | self._mLastOrder = 255 58 | self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT 59 | self._mTotalSeqs = 0 60 | self._mTotalChar = 0 61 | # characters that fall in our sampling range 62 | self._mFreqChar = 0 63 | 64 | def get_charset_name(self): 65 | if self._mNameProber: 66 | return self._mNameProber.get_charset_name() 67 | else: 68 | return self._mModel['charsetName'] 69 | 70 | def feed(self, aBuf): 71 | if not self._mModel['keepEnglishLetter']: 72 | aBuf = self.filter_without_english_letters(aBuf) 73 | aLen = len(aBuf) 74 | if not aLen: 75 | return self.get_state() 76 | for c in aBuf: 77 | order = self._mModel['charToOrderMap'][wrap_ord(c)] 78 | if order < SYMBOL_CAT_ORDER: 79 | self._mTotalChar += 1 80 | if order < SAMPLE_SIZE: 81 | self._mFreqChar += 1 82 | if self._mLastOrder < SAMPLE_SIZE: 83 | self._mTotalSeqs += 1 84 | if not self._mReversed: 85 | i = (self._mLastOrder * SAMPLE_SIZE) + order 86 | model = self._mModel['precedenceMatrix'][i] 87 | else: # reverse the order of the letters in the lookup 88 | i = (order * SAMPLE_SIZE) + self._mLastOrder 89 | model = self._mModel['precedenceMatrix'][i] 90 | self._mSeqCounters[model] += 1 91 | self._mLastOrder = order 92 | 93 | if self.get_state() == constants.eDetecting: 94 | if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD: 95 | cf = self.get_confidence() 96 | if cf > POSITIVE_SHORTCUT_THRESHOLD: 97 | if constants._debug: 98 | sys.stderr.write('%s confidence = %s, we have a' 99 | 'winner\n' % 100 | (self._mModel['charsetName'], cf)) 101 | self._mState = constants.eFoundIt 102 | elif cf < NEGATIVE_SHORTCUT_THRESHOLD: 103 | if constants._debug: 104 | sys.stderr.write('%s confidence = %s, below negative' 105 | 'shortcut threshhold %s\n' % 106 | (self._mModel['charsetName'], cf, 107 | NEGATIVE_SHORTCUT_THRESHOLD)) 108 | self._mState = constants.eNotMe 109 | 110 | return self.get_state() 111 | 112 | def get_confidence(self): 113 | r = 0.01 114 | if self._mTotalSeqs > 0: 115 | r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs 116 | / self._mModel['mTypicalPositiveRatio']) 117 | r = r * self._mFreqChar / self._mTotalChar 118 | if r >= 1.0: 119 | r = 0.99 120 | return r 121 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/sbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetgroupprober import CharSetGroupProber 30 | from .sbcharsetprober import SingleByteCharSetProber 31 | from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel, 32 | Latin5CyrillicModel, MacCyrillicModel, 33 | Ibm866Model, Ibm855Model) 34 | from .langgreekmodel import Latin7GreekModel, Win1253GreekModel 35 | from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel 36 | from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel 37 | from .langthaimodel import TIS620ThaiModel 38 | from .langhebrewmodel import Win1255HebrewModel 39 | from .hebrewprober import HebrewProber 40 | 41 | 42 | class SBCSGroupProber(CharSetGroupProber): 43 | def __init__(self): 44 | CharSetGroupProber.__init__(self) 45 | self._mProbers = [ 46 | SingleByteCharSetProber(Win1251CyrillicModel), 47 | SingleByteCharSetProber(Koi8rModel), 48 | SingleByteCharSetProber(Latin5CyrillicModel), 49 | SingleByteCharSetProber(MacCyrillicModel), 50 | SingleByteCharSetProber(Ibm866Model), 51 | SingleByteCharSetProber(Ibm855Model), 52 | SingleByteCharSetProber(Latin7GreekModel), 53 | SingleByteCharSetProber(Win1253GreekModel), 54 | SingleByteCharSetProber(Latin5BulgarianModel), 55 | SingleByteCharSetProber(Win1251BulgarianModel), 56 | SingleByteCharSetProber(Latin2HungarianModel), 57 | SingleByteCharSetProber(Win1250HungarianModel), 58 | SingleByteCharSetProber(TIS620ThaiModel), 59 | ] 60 | hebrewProber = HebrewProber() 61 | logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, 62 | False, hebrewProber) 63 | visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True, 64 | hebrewProber) 65 | hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber) 66 | self._mProbers.extend([hebrewProber, logicalHebrewProber, 67 | visualHebrewProber]) 68 | 69 | self.reset() 70 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/sjisprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from .mbcharsetprober import MultiByteCharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .chardistribution import SJISDistributionAnalysis 32 | from .jpcntx import SJISContextAnalysis 33 | from .mbcssm import SJISSMModel 34 | from . import constants 35 | 36 | 37 | class SJISProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(SJISSMModel) 41 | self._mDistributionAnalyzer = SJISDistributionAnalysis() 42 | self._mContextAnalyzer = SJISContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "SHIFT_JIS" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | codingState = self._mCodingSM.next_state(aBuf[i]) 56 | if codingState == constants.eError: 57 | if constants._debug: 58 | sys.stderr.write(self.get_charset_name() 59 | + ' prober hit error at byte ' + str(i) 60 | + '\n') 61 | self._mState = constants.eNotMe 62 | break 63 | elif codingState == constants.eItsMe: 64 | self._mState = constants.eFoundIt 65 | break 66 | elif codingState == constants.eStart: 67 | charLen = self._mCodingSM.get_current_charlen() 68 | if i == 0: 69 | self._mLastChar[1] = aBuf[0] 70 | self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:], 71 | charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3 75 | - charLen], charLen) 76 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 77 | charLen) 78 | 79 | self._mLastChar[0] = aBuf[aLen - 1] 80 | 81 | if self.get_state() == constants.eDetecting: 82 | if (self._mContextAnalyzer.got_enough_data() and 83 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 84 | self._mState = constants.eFoundIt 85 | 86 | return self.get_state() 87 | 88 | def get_confidence(self): 89 | contxtCf = self._mContextAnalyzer.get_confidence() 90 | distribCf = self._mDistributionAnalyzer.get_confidence() 91 | return max(contxtCf, distribCf) 92 | -------------------------------------------------------------------------------- /libs/requests/packages/charade/utf8prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .charsetprober import CharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .mbcssm import UTF8SMModel 32 | 33 | ONE_CHAR_PROB = 0.5 34 | 35 | 36 | class UTF8Prober(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = CodingStateMachine(UTF8SMModel) 40 | self.reset() 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | self._mCodingSM.reset() 45 | self._mNumOfMBChar = 0 46 | 47 | def get_charset_name(self): 48 | return "utf-8" 49 | 50 | def feed(self, aBuf): 51 | for c in aBuf: 52 | codingState = self._mCodingSM.next_state(c) 53 | if codingState == constants.eError: 54 | self._mState = constants.eNotMe 55 | break 56 | elif codingState == constants.eItsMe: 57 | self._mState = constants.eFoundIt 58 | break 59 | elif codingState == constants.eStart: 60 | if self._mCodingSM.get_current_charlen() >= 2: 61 | self._mNumOfMBChar += 1 62 | 63 | if self.get_state() == constants.eDetecting: 64 | if self.get_confidence() > constants.SHORTCUT_THRESHOLD: 65 | self._mState = constants.eFoundIt 66 | 67 | return self.get_state() 68 | 69 | def get_confidence(self): 70 | unlike = 0.99 71 | if self._mNumOfMBChar < 6: 72 | for i in range(0, self._mNumOfMBChar): 73 | unlike = unlike * ONE_CHAR_PROB 74 | return 1.0 - unlike 75 | else: 76 | return unlike 77 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/__init__.py: -------------------------------------------------------------------------------- 1 | # urllib3/__init__.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """ 8 | urllib3 - Thread-safe connection pooling and re-using. 9 | """ 10 | 11 | __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' 12 | __license__ = 'MIT' 13 | __version__ = 'dev' 14 | 15 | 16 | from .connectionpool import ( 17 | HTTPConnectionPool, 18 | HTTPSConnectionPool, 19 | connection_from_url 20 | ) 21 | 22 | from . import exceptions 23 | from .filepost import encode_multipart_formdata 24 | from .poolmanager import PoolManager, ProxyManager, proxy_from_url 25 | from .response import HTTPResponse 26 | from .util import make_headers, get_host, Timeout 27 | 28 | 29 | # Set default logging handler to avoid "No handler found" warnings. 30 | import logging 31 | try: # Python 2.7+ 32 | from logging import NullHandler 33 | except ImportError: 34 | class NullHandler(logging.Handler): 35 | def emit(self, record): 36 | pass 37 | 38 | logging.getLogger(__name__).addHandler(NullHandler()) 39 | 40 | def add_stderr_logger(level=logging.DEBUG): 41 | """ 42 | Helper for quickly adding a StreamHandler to the logger. Useful for 43 | debugging. 44 | 45 | Returns the handler after adding it. 46 | """ 47 | # This method needs to be in this __init__.py to get the __name__ correct 48 | # even if urllib3 is vendored within another package. 49 | logger = logging.getLogger(__name__) 50 | handler = logging.StreamHandler() 51 | handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) 52 | logger.addHandler(handler) 53 | logger.setLevel(level) 54 | logger.debug('Added an stderr logging handler to logger: %s' % __name__) 55 | return handler 56 | 57 | # ... Clean up. 58 | del NullHandler 59 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/_collections.py: -------------------------------------------------------------------------------- 1 | # urllib3/_collections.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | from collections import MutableMapping 8 | from threading import RLock 9 | 10 | try: # Python 2.7+ 11 | from collections import OrderedDict 12 | except ImportError: 13 | from .packages.ordered_dict import OrderedDict 14 | 15 | 16 | __all__ = ['RecentlyUsedContainer'] 17 | 18 | 19 | _Null = object() 20 | 21 | 22 | class RecentlyUsedContainer(MutableMapping): 23 | """ 24 | Provides a thread-safe dict-like container which maintains up to 25 | ``maxsize`` keys while throwing away the least-recently-used keys beyond 26 | ``maxsize``. 27 | 28 | :param maxsize: 29 | Maximum number of recent elements to retain. 30 | 31 | :param dispose_func: 32 | Every time an item is evicted from the container, 33 | ``dispose_func(value)`` is called. Callback which will get called 34 | """ 35 | 36 | ContainerCls = OrderedDict 37 | 38 | def __init__(self, maxsize=10, dispose_func=None): 39 | self._maxsize = maxsize 40 | self.dispose_func = dispose_func 41 | 42 | self._container = self.ContainerCls() 43 | self.lock = RLock() 44 | 45 | def __getitem__(self, key): 46 | # Re-insert the item, moving it to the end of the eviction line. 47 | with self.lock: 48 | item = self._container.pop(key) 49 | self._container[key] = item 50 | return item 51 | 52 | def __setitem__(self, key, value): 53 | evicted_value = _Null 54 | with self.lock: 55 | # Possibly evict the existing value of 'key' 56 | evicted_value = self._container.get(key, _Null) 57 | self._container[key] = value 58 | 59 | # If we didn't evict an existing value, we might have to evict the 60 | # least recently used item from the beginning of the container. 61 | if len(self._container) > self._maxsize: 62 | _key, evicted_value = self._container.popitem(last=False) 63 | 64 | if self.dispose_func and evicted_value is not _Null: 65 | self.dispose_func(evicted_value) 66 | 67 | def __delitem__(self, key): 68 | with self.lock: 69 | value = self._container.pop(key) 70 | 71 | if self.dispose_func: 72 | self.dispose_func(value) 73 | 74 | def __len__(self): 75 | with self.lock: 76 | return len(self._container) 77 | 78 | def __iter__(self): 79 | raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.') 80 | 81 | def clear(self): 82 | with self.lock: 83 | # Copy pointers to all values, then wipe the mapping 84 | # under Python 2, this copies the list of values twice :-| 85 | values = list(self._container.values()) 86 | self._container.clear() 87 | 88 | if self.dispose_func: 89 | for value in values: 90 | self.dispose_func(value) 91 | 92 | def keys(self): 93 | with self.lock: 94 | return self._container.keys() 95 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/libs/requests/packages/urllib3/contrib/__init__.py -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/contrib/ntlmpool.py: -------------------------------------------------------------------------------- 1 | # urllib3/contrib/ntlmpool.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """ 8 | NTLM authenticating pool, contributed by erikcederstran 9 | 10 | Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 11 | """ 12 | 13 | try: 14 | from http.client import HTTPSConnection 15 | except ImportError: 16 | from httplib import HTTPSConnection 17 | from logging import getLogger 18 | from ntlm import ntlm 19 | 20 | from urllib3 import HTTPSConnectionPool 21 | 22 | 23 | log = getLogger(__name__) 24 | 25 | 26 | class NTLMConnectionPool(HTTPSConnectionPool): 27 | """ 28 | Implements an NTLM authentication version of an urllib3 connection pool 29 | """ 30 | 31 | scheme = 'https' 32 | 33 | def __init__(self, user, pw, authurl, *args, **kwargs): 34 | """ 35 | authurl is a random URL on the server that is protected by NTLM. 36 | user is the Windows user, probably in the DOMAIN\\username format. 37 | pw is the password for the user. 38 | """ 39 | super(NTLMConnectionPool, self).__init__(*args, **kwargs) 40 | self.authurl = authurl 41 | self.rawuser = user 42 | user_parts = user.split('\\', 1) 43 | self.domain = user_parts[0].upper() 44 | self.user = user_parts[1] 45 | self.pw = pw 46 | 47 | def _new_conn(self): 48 | # Performs the NTLM handshake that secures the connection. The socket 49 | # must be kept open while requests are performed. 50 | self.num_connections += 1 51 | log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % 52 | (self.num_connections, self.host, self.authurl)) 53 | 54 | headers = {} 55 | headers['Connection'] = 'Keep-Alive' 56 | req_header = 'Authorization' 57 | resp_header = 'www-authenticate' 58 | 59 | conn = HTTPSConnection(host=self.host, port=self.port) 60 | 61 | # Send negotiation message 62 | headers[req_header] = ( 63 | 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) 64 | log.debug('Request headers: %s' % headers) 65 | conn.request('GET', self.authurl, None, headers) 66 | res = conn.getresponse() 67 | reshdr = dict(res.getheaders()) 68 | log.debug('Response status: %s %s' % (res.status, res.reason)) 69 | log.debug('Response headers: %s' % reshdr) 70 | log.debug('Response data: %s [...]' % res.read(100)) 71 | 72 | # Remove the reference to the socket, so that it can not be closed by 73 | # the response object (we want to keep the socket open) 74 | res.fp = None 75 | 76 | # Server should respond with a challenge message 77 | auth_header_values = reshdr[resp_header].split(', ') 78 | auth_header_value = None 79 | for s in auth_header_values: 80 | if s[:5] == 'NTLM ': 81 | auth_header_value = s[5:] 82 | if auth_header_value is None: 83 | raise Exception('Unexpected %s response header: %s' % 84 | (resp_header, reshdr[resp_header])) 85 | 86 | # Send authentication message 87 | ServerChallenge, NegotiateFlags = \ 88 | ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) 89 | auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, 90 | self.user, 91 | self.domain, 92 | self.pw, 93 | NegotiateFlags) 94 | headers[req_header] = 'NTLM %s' % auth_msg 95 | log.debug('Request headers: %s' % headers) 96 | conn.request('GET', self.authurl, None, headers) 97 | res = conn.getresponse() 98 | log.debug('Response status: %s %s' % (res.status, res.reason)) 99 | log.debug('Response headers: %s' % dict(res.getheaders())) 100 | log.debug('Response data: %s [...]' % res.read()[:100]) 101 | if res.status != 200: 102 | if res.status == 401: 103 | raise Exception('Server rejected request: wrong ' 104 | 'username or password') 105 | raise Exception('Wrong server response: %s %s' % 106 | (res.status, res.reason)) 107 | 108 | res.fp = None 109 | log.debug('Connection established') 110 | return conn 111 | 112 | def urlopen(self, method, url, body=None, headers=None, retries=3, 113 | redirect=True, assert_same_host=True): 114 | if headers is None: 115 | headers = {} 116 | headers['Connection'] = 'Keep-Alive' 117 | return super(NTLMConnectionPool, self).urlopen(method, url, body, 118 | headers, retries, 119 | redirect, 120 | assert_same_host) 121 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/exceptions.py: -------------------------------------------------------------------------------- 1 | # urllib3/exceptions.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | 8 | ## Base Exceptions 9 | 10 | class HTTPError(Exception): 11 | "Base exception used by this module." 12 | pass 13 | 14 | 15 | class PoolError(HTTPError): 16 | "Base exception for errors caused within a pool." 17 | def __init__(self, pool, message): 18 | self.pool = pool 19 | HTTPError.__init__(self, "%s: %s" % (pool, message)) 20 | 21 | def __reduce__(self): 22 | # For pickling purposes. 23 | return self.__class__, (None, None) 24 | 25 | 26 | class RequestError(PoolError): 27 | "Base exception for PoolErrors that have associated URLs." 28 | def __init__(self, pool, url, message): 29 | self.url = url 30 | PoolError.__init__(self, pool, message) 31 | 32 | def __reduce__(self): 33 | # For pickling purposes. 34 | return self.__class__, (None, self.url, None) 35 | 36 | 37 | class SSLError(HTTPError): 38 | "Raised when SSL certificate fails in an HTTPS connection." 39 | pass 40 | 41 | 42 | class ProxyError(HTTPError): 43 | "Raised when the connection to a proxy fails." 44 | pass 45 | 46 | 47 | class DecodeError(HTTPError): 48 | "Raised when automatic decoding based on Content-Type fails." 49 | pass 50 | 51 | 52 | ## Leaf Exceptions 53 | 54 | class MaxRetryError(RequestError): 55 | "Raised when the maximum number of retries is exceeded." 56 | 57 | def __init__(self, pool, url, reason=None): 58 | self.reason = reason 59 | 60 | message = "Max retries exceeded with url: %s" % url 61 | if reason: 62 | message += " (Caused by %s: %s)" % (type(reason), reason) 63 | else: 64 | message += " (Caused by redirect)" 65 | 66 | RequestError.__init__(self, pool, url, message) 67 | 68 | 69 | class HostChangedError(RequestError): 70 | "Raised when an existing pool gets a request for a foreign host." 71 | 72 | def __init__(self, pool, url, retries=3): 73 | message = "Tried to open a foreign host with url: %s" % url 74 | RequestError.__init__(self, pool, url, message) 75 | self.retries = retries 76 | 77 | 78 | class TimeoutStateError(HTTPError): 79 | """ Raised when passing an invalid state to a timeout """ 80 | pass 81 | 82 | 83 | class TimeoutError(HTTPError): 84 | """ Raised when a socket timeout error occurs. 85 | 86 | Catching this error will catch both :exc:`ReadTimeoutErrors 87 | ` and :exc:`ConnectTimeoutErrors `. 88 | """ 89 | pass 90 | 91 | 92 | class ReadTimeoutError(TimeoutError, RequestError): 93 | "Raised when a socket timeout occurs while receiving data from a server" 94 | pass 95 | 96 | 97 | # This timeout error does not have a URL attached and needs to inherit from the 98 | # base HTTPError 99 | class ConnectTimeoutError(TimeoutError): 100 | "Raised when a socket timeout occurs while connecting to a server" 101 | pass 102 | 103 | 104 | class EmptyPoolError(PoolError): 105 | "Raised when a pool runs out of connections and no more are allowed." 106 | pass 107 | 108 | 109 | class ClosedPoolError(PoolError): 110 | "Raised when a request enters a pool after the pool has been closed." 111 | pass 112 | 113 | 114 | class LocationParseError(ValueError, HTTPError): 115 | "Raised when get_host or similar fails to parse the URL input." 116 | 117 | def __init__(self, location): 118 | message = "Failed to parse: %s" % location 119 | HTTPError.__init__(self, message) 120 | 121 | self.location = location 122 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/filepost.py: -------------------------------------------------------------------------------- 1 | # urllib3/filepost.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | import codecs 8 | import mimetypes 9 | 10 | from uuid import uuid4 11 | from io import BytesIO 12 | 13 | from .packages import six 14 | from .packages.six import b 15 | from .fields import RequestField 16 | 17 | writer = codecs.lookup('utf-8')[3] 18 | 19 | 20 | def choose_boundary(): 21 | """ 22 | Our embarassingly-simple replacement for mimetools.choose_boundary. 23 | """ 24 | return uuid4().hex 25 | 26 | 27 | def iter_field_objects(fields): 28 | """ 29 | Iterate over fields. 30 | 31 | Supports list of (k, v) tuples and dicts, and lists of 32 | :class:`~urllib3.fields.RequestField`. 33 | 34 | """ 35 | if isinstance(fields, dict): 36 | i = six.iteritems(fields) 37 | else: 38 | i = iter(fields) 39 | 40 | for field in i: 41 | if isinstance(field, RequestField): 42 | yield field 43 | else: 44 | yield RequestField.from_tuples(*field) 45 | 46 | 47 | def iter_fields(fields): 48 | """ 49 | Iterate over fields. 50 | 51 | .. deprecated :: 52 | 53 | The addition of `~urllib3.fields.RequestField` makes this function 54 | obsolete. Instead, use :func:`iter_field_objects`, which returns 55 | `~urllib3.fields.RequestField` objects, instead. 56 | 57 | Supports list of (k, v) tuples and dicts. 58 | 59 | """ 60 | if isinstance(fields, dict): 61 | return ((k, v) for k, v in six.iteritems(fields)) 62 | 63 | return ((k, v) for k, v in fields) 64 | 65 | 66 | def encode_multipart_formdata(fields, boundary=None): 67 | """ 68 | Encode a dictionary of ``fields`` using the multipart/form-data MIME format. 69 | 70 | :param fields: 71 | Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). 72 | 73 | :param boundary: 74 | If not specified, then a random boundary will be generated using 75 | :func:`mimetools.choose_boundary`. 76 | """ 77 | body = BytesIO() 78 | if boundary is None: 79 | boundary = choose_boundary() 80 | 81 | for field in iter_field_objects(fields): 82 | body.write(b('--%s\r\n' % (boundary))) 83 | 84 | writer(body).write(field.render_headers()) 85 | data = field.data 86 | 87 | if isinstance(data, int): 88 | data = str(data) # Backwards compatibility 89 | 90 | if isinstance(data, six.text_type): 91 | writer(body).write(data) 92 | else: 93 | body.write(data) 94 | 95 | body.write(b'\r\n') 96 | 97 | body.write(b('--%s--\r\n' % (boundary))) 98 | 99 | content_type = str('multipart/form-data; boundary=%s' % boundary) 100 | 101 | return body.getvalue(), content_type 102 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import ssl_match_hostname 4 | 5 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py: -------------------------------------------------------------------------------- 1 | """The match_hostname() function from Python 3.2, essential when using SSL.""" 2 | 3 | import re 4 | 5 | __version__ = '3.2.2' 6 | 7 | class CertificateError(ValueError): 8 | pass 9 | 10 | def _dnsname_match(dn, hostname, max_wildcards=1): 11 | """Matching according to RFC 6125, section 6.4.3 12 | 13 | http://tools.ietf.org/html/rfc6125#section-6.4.3 14 | """ 15 | pats = [] 16 | if not dn: 17 | return False 18 | 19 | parts = dn.split(r'.') 20 | leftmost = parts[0] 21 | 22 | wildcards = leftmost.count('*') 23 | if wildcards > max_wildcards: 24 | # Issue #17980: avoid denials of service by refusing more 25 | # than one wildcard per fragment. A survery of established 26 | # policy among SSL implementations showed it to be a 27 | # reasonable choice. 28 | raise CertificateError( 29 | "too many wildcards in certificate DNS name: " + repr(dn)) 30 | 31 | # speed up common case w/o wildcards 32 | if not wildcards: 33 | return dn.lower() == hostname.lower() 34 | 35 | # RFC 6125, section 6.4.3, subitem 1. 36 | # The client SHOULD NOT attempt to match a presented identifier in which 37 | # the wildcard character comprises a label other than the left-most label. 38 | if leftmost == '*': 39 | # When '*' is a fragment by itself, it matches a non-empty dotless 40 | # fragment. 41 | pats.append('[^.]+') 42 | elif leftmost.startswith('xn--') or hostname.startswith('xn--'): 43 | # RFC 6125, section 6.4.3, subitem 3. 44 | # The client SHOULD NOT attempt to match a presented identifier 45 | # where the wildcard character is embedded within an A-label or 46 | # U-label of an internationalized domain name. 47 | pats.append(re.escape(leftmost)) 48 | else: 49 | # Otherwise, '*' matches any dotless string, e.g. www* 50 | pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) 51 | 52 | # add the remaining fragments, ignore any wildcards 53 | for frag in parts[1:]: 54 | pats.append(re.escape(frag)) 55 | 56 | pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) 57 | return pat.match(hostname) 58 | 59 | 60 | def match_hostname(cert, hostname): 61 | """Verify that *cert* (in decoded format as returned by 62 | SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 63 | rules are followed, but IP addresses are not accepted for *hostname*. 64 | 65 | CertificateError is raised on failure. On success, the function 66 | returns nothing. 67 | """ 68 | if not cert: 69 | raise ValueError("empty or no certificate") 70 | dnsnames = [] 71 | san = cert.get('subjectAltName', ()) 72 | for key, value in san: 73 | if key == 'DNS': 74 | if _dnsname_match(value, hostname): 75 | return 76 | dnsnames.append(value) 77 | if not dnsnames: 78 | # The subject is only checked when there is no dNSName entry 79 | # in subjectAltName 80 | for sub in cert.get('subject', ()): 81 | for key, value in sub: 82 | # XXX according to RFC 2818, the most specific Common Name 83 | # must be used. 84 | if key == 'commonName': 85 | if _dnsname_match(value, hostname): 86 | return 87 | dnsnames.append(value) 88 | if len(dnsnames) > 1: 89 | raise CertificateError("hostname %r " 90 | "doesn't match either of %s" 91 | % (hostname, ', '.join(map(repr, dnsnames)))) 92 | elif len(dnsnames) == 1: 93 | raise CertificateError("hostname %r " 94 | "doesn't match %r" 95 | % (hostname, dnsnames[0])) 96 | else: 97 | raise CertificateError("no appropriate commonName or " 98 | "subjectAltName fields were found") 99 | -------------------------------------------------------------------------------- /libs/requests/packages/urllib3/request.py: -------------------------------------------------------------------------------- 1 | # urllib3/request.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | try: 8 | from urllib.parse import urlencode 9 | except ImportError: 10 | from urllib import urlencode 11 | 12 | from .filepost import encode_multipart_formdata 13 | 14 | 15 | __all__ = ['RequestMethods'] 16 | 17 | 18 | class RequestMethods(object): 19 | """ 20 | Convenience mixin for classes who implement a :meth:`urlopen` method, such 21 | as :class:`~urllib3.connectionpool.HTTPConnectionPool` and 22 | :class:`~urllib3.poolmanager.PoolManager`. 23 | 24 | Provides behavior for making common types of HTTP request methods and 25 | decides which type of request field encoding to use. 26 | 27 | Specifically, 28 | 29 | :meth:`.request_encode_url` is for sending requests whose fields are encoded 30 | in the URL (such as GET, HEAD, DELETE). 31 | 32 | :meth:`.request_encode_body` is for sending requests whose fields are 33 | encoded in the *body* of the request using multipart or www-form-urlencoded 34 | (such as for POST, PUT, PATCH). 35 | 36 | :meth:`.request` is for making any kind of request, it will look up the 37 | appropriate encoding format and use one of the above two methods to make 38 | the request. 39 | 40 | Initializer parameters: 41 | 42 | :param headers: 43 | Headers to include with all requests, unless other headers are given 44 | explicitly. 45 | """ 46 | 47 | _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) 48 | _encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE']) 49 | 50 | def __init__(self, headers=None): 51 | self.headers = headers or {} 52 | 53 | def urlopen(self, method, url, body=None, headers=None, 54 | encode_multipart=True, multipart_boundary=None, 55 | **kw): # Abstract 56 | raise NotImplemented("Classes extending RequestMethods must implement " 57 | "their own ``urlopen`` method.") 58 | 59 | def request(self, method, url, fields=None, headers=None, **urlopen_kw): 60 | """ 61 | Make a request using :meth:`urlopen` with the appropriate encoding of 62 | ``fields`` based on the ``method`` used. 63 | 64 | This is a convenience method that requires the least amount of manual 65 | effort. It can be used in most situations, while still having the option 66 | to drop down to more specific methods when necessary, such as 67 | :meth:`request_encode_url`, :meth:`request_encode_body`, 68 | or even the lowest level :meth:`urlopen`. 69 | """ 70 | method = method.upper() 71 | 72 | if method in self._encode_url_methods: 73 | return self.request_encode_url(method, url, fields=fields, 74 | headers=headers, 75 | **urlopen_kw) 76 | else: 77 | return self.request_encode_body(method, url, fields=fields, 78 | headers=headers, 79 | **urlopen_kw) 80 | 81 | def request_encode_url(self, method, url, fields=None, **urlopen_kw): 82 | """ 83 | Make a request using :meth:`urlopen` with the ``fields`` encoded in 84 | the url. This is useful for request methods like GET, HEAD, DELETE, etc. 85 | """ 86 | if fields: 87 | url += '?' + urlencode(fields) 88 | return self.urlopen(method, url, **urlopen_kw) 89 | 90 | def request_encode_body(self, method, url, fields=None, headers=None, 91 | encode_multipart=True, multipart_boundary=None, 92 | **urlopen_kw): 93 | """ 94 | Make a request using :meth:`urlopen` with the ``fields`` encoded in 95 | the body. This is useful for request methods like POST, PUT, PATCH, etc. 96 | 97 | When ``encode_multipart=True`` (default), then 98 | :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the 99 | payload with the appropriate content type. Otherwise 100 | :meth:`urllib.urlencode` is used with the 101 | 'application/x-www-form-urlencoded' content type. 102 | 103 | Multipart encoding must be used when posting files, and it's reasonably 104 | safe to use it in other times too. However, it may break request signing, 105 | such as with OAuth. 106 | 107 | Supports an optional ``fields`` parameter of key/value strings AND 108 | key/filetuple. A filetuple is a (filename, data, MIME type) tuple where 109 | the MIME type is optional. For example: :: 110 | 111 | fields = { 112 | 'foo': 'bar', 113 | 'fakefile': ('foofile.txt', 'contents of foofile'), 114 | 'realfile': ('barfile.txt', open('realfile').read()), 115 | 'typedfile': ('bazfile.bin', open('bazfile').read(), 116 | 'image/jpeg'), 117 | 'nonamefile': 'contents of nonamefile field', 118 | } 119 | 120 | When uploading a file, providing a filename (the first parameter of the 121 | tuple) is optional but recommended to best mimick behavior of browsers. 122 | 123 | Note that if ``headers`` are supplied, the 'Content-Type' header will be 124 | overwritten because it depends on the dynamic random boundary string 125 | which is used to compose the body of the request. The random boundary 126 | string can be explicitly set with the ``multipart_boundary`` parameter. 127 | """ 128 | if encode_multipart: 129 | body, content_type = encode_multipart_formdata(fields or {}, 130 | boundary=multipart_boundary) 131 | else: 132 | body, content_type = (urlencode(fields or {}), 133 | 'application/x-www-form-urlencoded') 134 | 135 | if headers is None: 136 | headers = self.headers 137 | 138 | headers_ = {'Content-Type': content_type} 139 | headers_.update(headers) 140 | 141 | return self.urlopen(method, url, body=body, headers=headers_, 142 | **urlopen_kw) 143 | -------------------------------------------------------------------------------- /libs/requests/status_codes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .structures import LookupDict 4 | 5 | _codes = { 6 | 7 | # Informational. 8 | 100: ('continue',), 9 | 101: ('switching_protocols',), 10 | 102: ('processing',), 11 | 103: ('checkpoint',), 12 | 122: ('uri_too_long', 'request_uri_too_long'), 13 | 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'), 14 | 201: ('created',), 15 | 202: ('accepted',), 16 | 203: ('non_authoritative_info', 'non_authoritative_information'), 17 | 204: ('no_content',), 18 | 205: ('reset_content', 'reset'), 19 | 206: ('partial_content', 'partial'), 20 | 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), 21 | 208: ('already_reported',), 22 | 226: ('im_used',), 23 | 24 | # Redirection. 25 | 300: ('multiple_choices',), 26 | 301: ('moved_permanently', 'moved', '\\o-'), 27 | 302: ('found',), 28 | 303: ('see_other', 'other'), 29 | 304: ('not_modified',), 30 | 305: ('use_proxy',), 31 | 306: ('switch_proxy',), 32 | 307: ('temporary_redirect', 'temporary_moved', 'temporary'), 33 | 308: ('resume_incomplete', 'resume'), 34 | 35 | # Client Error. 36 | 400: ('bad_request', 'bad'), 37 | 401: ('unauthorized',), 38 | 402: ('payment_required', 'payment'), 39 | 403: ('forbidden',), 40 | 404: ('not_found', '-o-'), 41 | 405: ('method_not_allowed', 'not_allowed'), 42 | 406: ('not_acceptable',), 43 | 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), 44 | 408: ('request_timeout', 'timeout'), 45 | 409: ('conflict',), 46 | 410: ('gone',), 47 | 411: ('length_required',), 48 | 412: ('precondition_failed', 'precondition'), 49 | 413: ('request_entity_too_large',), 50 | 414: ('request_uri_too_large',), 51 | 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), 52 | 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 53 | 417: ('expectation_failed',), 54 | 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), 55 | 422: ('unprocessable_entity', 'unprocessable'), 56 | 423: ('locked',), 57 | 424: ('failed_dependency', 'dependency'), 58 | 425: ('unordered_collection', 'unordered'), 59 | 426: ('upgrade_required', 'upgrade'), 60 | 428: ('precondition_required', 'precondition'), 61 | 429: ('too_many_requests', 'too_many'), 62 | 431: ('header_fields_too_large', 'fields_too_large'), 63 | 444: ('no_response', 'none'), 64 | 449: ('retry_with', 'retry'), 65 | 450: ('blocked_by_windows_parental_controls', 'parental_controls'), 66 | 451: ('unavailable_for_legal_reasons', 'legal_reasons'), 67 | 499: ('client_closed_request',), 68 | 69 | # Server Error. 70 | 500: ('internal_server_error', 'server_error', '/o\\', '✗'), 71 | 501: ('not_implemented',), 72 | 502: ('bad_gateway',), 73 | 503: ('service_unavailable', 'unavailable'), 74 | 504: ('gateway_timeout',), 75 | 505: ('http_version_not_supported', 'http_version'), 76 | 506: ('variant_also_negotiates',), 77 | 507: ('insufficient_storage',), 78 | 509: ('bandwidth_limit_exceeded', 'bandwidth'), 79 | 510: ('not_extended',), 80 | } 81 | 82 | codes = LookupDict(name='status_codes') 83 | 84 | for (code, titles) in list(_codes.items()): 85 | for title in titles: 86 | setattr(codes, title, code) 87 | if not title.startswith('\\'): 88 | setattr(codes, title.upper(), code) 89 | -------------------------------------------------------------------------------- /libs/requests/structures.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.structures 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | Data structures that power Requests. 8 | 9 | """ 10 | 11 | import os 12 | import collections 13 | from itertools import islice 14 | 15 | 16 | class IteratorProxy(object): 17 | """docstring for IteratorProxy""" 18 | def __init__(self, i): 19 | self.i = i 20 | # self.i = chain.from_iterable(i) 21 | 22 | def __iter__(self): 23 | return self.i 24 | 25 | def __len__(self): 26 | if hasattr(self.i, '__len__'): 27 | return len(self.i) 28 | if hasattr(self.i, 'len'): 29 | return self.i.len 30 | if hasattr(self.i, 'fileno'): 31 | return os.fstat(self.i.fileno()).st_size 32 | 33 | def read(self, n): 34 | return "".join(islice(self.i, None, n)) 35 | 36 | 37 | class CaseInsensitiveDict(collections.MutableMapping): 38 | """ 39 | A case-insensitive ``dict``-like object. 40 | 41 | Implements all methods and operations of 42 | ``collections.MutableMapping`` as well as dict's ``copy``. Also 43 | provides ``lower_items``. 44 | 45 | All keys are expected to be strings. The structure remembers the 46 | case of the last key to be set, and ``iter(instance)``, 47 | ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` 48 | will contain case-sensitive keys. However, querying and contains 49 | testing is case insensitive: 50 | 51 | cid = CaseInsensitiveDict() 52 | cid['Accept'] = 'application/json' 53 | cid['aCCEPT'] == 'application/json' # True 54 | list(cid) == ['Accept'] # True 55 | 56 | For example, ``headers['content-encoding']`` will return the 57 | value of a ``'Content-Encoding'`` response header, regardless 58 | of how the header name was originally stored. 59 | 60 | If the constructor, ``.update``, or equality comparison 61 | operations are given keys that have equal ``.lower()``s, the 62 | behavior is undefined. 63 | 64 | """ 65 | def __init__(self, data=None, **kwargs): 66 | self._store = dict() 67 | if data is None: 68 | data = {} 69 | self.update(data, **kwargs) 70 | 71 | def __setitem__(self, key, value): 72 | # Use the lowercased key for lookups, but store the actual 73 | # key alongside the value. 74 | self._store[key.lower()] = (key, value) 75 | 76 | def __getitem__(self, key): 77 | return self._store[key.lower()][1] 78 | 79 | def __delitem__(self, key): 80 | del self._store[key.lower()] 81 | 82 | def __iter__(self): 83 | return (casedkey for casedkey, mappedvalue in self._store.values()) 84 | 85 | def __len__(self): 86 | return len(self._store) 87 | 88 | def lower_items(self): 89 | """Like iteritems(), but with all lowercase keys.""" 90 | return ( 91 | (lowerkey, keyval[1]) 92 | for (lowerkey, keyval) 93 | in self._store.items() 94 | ) 95 | 96 | def __eq__(self, other): 97 | if isinstance(other, collections.Mapping): 98 | other = CaseInsensitiveDict(other) 99 | else: 100 | return NotImplemented 101 | # Compare insensitively 102 | return dict(self.lower_items()) == dict(other.lower_items()) 103 | 104 | # Copy is required 105 | def copy(self): 106 | return CaseInsensitiveDict(self._store.values()) 107 | 108 | def __repr__(self): 109 | return '%s(%r)' % (self.__class__.__name__, dict(self.items())) 110 | 111 | 112 | class LookupDict(dict): 113 | """Dictionary lookup object.""" 114 | 115 | def __init__(self, name=None): 116 | self.name = name 117 | super(LookupDict, self).__init__() 118 | 119 | def __repr__(self): 120 | return '' % (self.name) 121 | 122 | def __getitem__(self, key): 123 | # We allow fall-through here, so values default to None 124 | 125 | return self.__dict__.get(key, None) 126 | 127 | def get(self, key, default=None): 128 | return self.__dict__.get(key, default) 129 | -------------------------------------------------------------------------------- /libs/uritemplate/__init__.py: -------------------------------------------------------------------------------- 1 | # Early, and incomplete implementation of -04. 2 | # 3 | import re 4 | import urllib 5 | 6 | RESERVED = ":/?#[]@!$&'()*+,;=" 7 | OPERATOR = "+./;?|!@" 8 | EXPLODE = "*+" 9 | MODIFIER = ":^" 10 | TEMPLATE = re.compile(r"{(?P[\+\./;\?|!@])?(?P[^}]+)}", re.UNICODE) 11 | VAR = re.compile(r"^(?P[^=\+\*:\^]+)((?P[\+\*])|(?P[:\^]-?[0-9]+))?(=(?P.*))?$", re.UNICODE) 12 | 13 | def _tostring(varname, value, explode, operator, safe=""): 14 | if type(value) == type([]): 15 | if explode == "+": 16 | return ",".join([varname + "." + urllib.quote(x, safe) for x in value]) 17 | else: 18 | return ",".join([urllib.quote(x, safe) for x in value]) 19 | if type(value) == type({}): 20 | keys = value.keys() 21 | keys.sort() 22 | if explode == "+": 23 | return ",".join([varname + "." + urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys]) 24 | else: 25 | return ",".join([urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys]) 26 | else: 27 | return urllib.quote(value, safe) 28 | 29 | 30 | def _tostring_path(varname, value, explode, operator, safe=""): 31 | joiner = operator 32 | if type(value) == type([]): 33 | if explode == "+": 34 | return joiner.join([varname + "." + urllib.quote(x, safe) for x in value]) 35 | elif explode == "*": 36 | return joiner.join([urllib.quote(x, safe) for x in value]) 37 | else: 38 | return ",".join([urllib.quote(x, safe) for x in value]) 39 | elif type(value) == type({}): 40 | keys = value.keys() 41 | keys.sort() 42 | if explode == "+": 43 | return joiner.join([varname + "." + urllib.quote(key, safe) + joiner + urllib.quote(value[key], safe) for key in keys]) 44 | elif explode == "*": 45 | return joiner.join([urllib.quote(key, safe) + joiner + urllib.quote(value[key], safe) for key in keys]) 46 | else: 47 | return ",".join([urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys]) 48 | else: 49 | if value: 50 | return urllib.quote(value, safe) 51 | else: 52 | return "" 53 | 54 | def _tostring_query(varname, value, explode, operator, safe=""): 55 | joiner = operator 56 | varprefix = "" 57 | if operator == "?": 58 | joiner = "&" 59 | varprefix = varname + "=" 60 | if type(value) == type([]): 61 | if 0 == len(value): 62 | return "" 63 | if explode == "+": 64 | return joiner.join([varname + "=" + urllib.quote(x, safe) for x in value]) 65 | elif explode == "*": 66 | return joiner.join([urllib.quote(x, safe) for x in value]) 67 | else: 68 | return varprefix + ",".join([urllib.quote(x, safe) for x in value]) 69 | elif type(value) == type({}): 70 | if 0 == len(value): 71 | return "" 72 | keys = value.keys() 73 | keys.sort() 74 | if explode == "+": 75 | return joiner.join([varname + "." + urllib.quote(key, safe) + "=" + urllib.quote(value[key], safe) for key in keys]) 76 | elif explode == "*": 77 | return joiner.join([urllib.quote(key, safe) + "=" + urllib.quote(value[key], safe) for key in keys]) 78 | else: 79 | return varprefix + ",".join([urllib.quote(key, safe) + "," + urllib.quote(value[key], safe) for key in keys]) 80 | else: 81 | if value: 82 | return varname + "=" + urllib.quote(value, safe) 83 | else: 84 | return varname 85 | 86 | TOSTRING = { 87 | "" : _tostring, 88 | "+": _tostring, 89 | ";": _tostring_query, 90 | "?": _tostring_query, 91 | "/": _tostring_path, 92 | ".": _tostring_path, 93 | } 94 | 95 | 96 | def expand(template, vars): 97 | def _sub(match): 98 | groupdict = match.groupdict() 99 | operator = groupdict.get('operator') 100 | if operator is None: 101 | operator = '' 102 | varlist = groupdict.get('varlist') 103 | 104 | safe = "@" 105 | if operator == '+': 106 | safe = RESERVED 107 | varspecs = varlist.split(",") 108 | varnames = [] 109 | defaults = {} 110 | for varspec in varspecs: 111 | m = VAR.search(varspec) 112 | groupdict = m.groupdict() 113 | varname = groupdict.get('varname') 114 | explode = groupdict.get('explode') 115 | partial = groupdict.get('partial') 116 | default = groupdict.get('default') 117 | if default: 118 | defaults[varname] = default 119 | varnames.append((varname, explode, partial)) 120 | 121 | retval = [] 122 | joiner = operator 123 | prefix = operator 124 | if operator == "+": 125 | prefix = "" 126 | joiner = "," 127 | if operator == "?": 128 | joiner = "&" 129 | if operator == "": 130 | joiner = "," 131 | for varname, explode, partial in varnames: 132 | if varname in vars: 133 | value = vars[varname] 134 | #if not value and (type(value) == type({}) or type(value) == type([])) and varname in defaults: 135 | if not value and value != "" and varname in defaults: 136 | value = defaults[varname] 137 | elif varname in defaults: 138 | value = defaults[varname] 139 | else: 140 | continue 141 | retval.append(TOSTRING[operator](varname, value, explode, operator, safe=safe)) 142 | if "".join(retval): 143 | return prefix + joiner.join(retval) 144 | else: 145 | return "" 146 | 147 | return TEMPLATE.sub(_sub, template) 148 | -------------------------------------------------------------------------------- /logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=consoleHandler,fileHandler 6 | 7 | [formatters] 8 | keys=simpleFormatter 9 | 10 | [logger_root] 11 | level=DEBUG 12 | handlers=consoleHandler,fileHandler 13 | 14 | [handler_consoleHandler] 15 | class=StreamHandler 16 | level=DEBUG 17 | formatter=simpleFormatter 18 | args=(sys.stdout,) 19 | 20 | [handler_fileHandler] 21 | class=FileHandler 22 | level=DEBUG 23 | formatter=simpleFormatter 24 | args=('./output.txt',) 25 | 26 | [formatter_simpleFormatter] 27 | format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 28 | datefmt= 29 | -------------------------------------------------------------------------------- /queue.yaml: -------------------------------------------------------------------------------- 1 | # Set the total storage limit for all queues to 100MB 2 | total_storage_limit: 100M 3 | queue: 4 | - name: default 5 | rate: 6/m 6 | retry_parameters: 7 | task_retry_limit: 5 8 | task_age_limit: 1d 9 | bucket_size: 10 10 | max_concurrent_requests: 5 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cffi==1.2.1 2 | config==0.3.9 3 | cryptography==1.0.2 4 | enum34==1.0.4 5 | gnippy==0.4.1 6 | google-api-python-client==1.4.1 7 | httplib2==0.9.1 8 | idna==2.0 9 | ipaddress==1.0.14 10 | Jinja2==2.7.3 11 | MarkupSafe==0.23 12 | oauth2client==1.4.11 13 | oauthlib==0.7.2 14 | pyasn1==0.1.9 15 | pyasn1-modules==0.0.5 16 | pycparser==2.14 17 | pycrypto==2.6.1 18 | pyOpenSSL==0.15.1 19 | PyYAML==3.11 20 | requests==2.7.0 21 | requests-oauthlib==0.5.0 22 | rsa==3.1.4 23 | simplejson==3.7.3 24 | six==1.9.0 25 | tweepy==3.3.0 26 | uritemplate==0.6 27 | wheel==0.24.0 28 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/screenshot.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import shutil 3 | from config import Config 4 | from jinja2 import Template 5 | from utils import Utils 6 | 7 | f = file("./config") 8 | config = Config(f) 9 | 10 | schema = None 11 | if (config.MODE not in ["gnip", "twitter"]): 12 | print "Invalid Mode. Mode can be 'gnip' or 'twitter' only"; 13 | exit() 14 | 15 | container_template = Utils.read_file("./container.yaml.template") 16 | container_template = Template(container_template).render(config) 17 | 18 | container_file = open("./container.yaml", 'w') 19 | container_file.write(container_template) 20 | 21 | 22 | -------------------------------------------------------------------------------- /static/css/c3.css: -------------------------------------------------------------------------------- 1 | /*-- Chart --*/ 2 | 3 | .c3 svg { 4 | font: 10px sans-serif; 5 | } 6 | .c3 path, .c3 line { 7 | fill: none; 8 | stroke: #000; 9 | } 10 | .c3 text { 11 | -webkit-user-select: none; 12 | -moz-user-select: none; 13 | user-select: none; 14 | } 15 | 16 | .c3-legend-item-tile, 17 | .c3-xgrid-focus, 18 | .c3-ygrid, 19 | .c3-event-rect, 20 | .c3-bars path { 21 | shape-rendering: crispEdges; 22 | } 23 | 24 | .c3-chart-arc path { 25 | stroke: #fff; 26 | 27 | } 28 | .c3-chart-arc text { 29 | fill: #fff; 30 | font-size: 13px; 31 | } 32 | 33 | /*-- Axis --*/ 34 | 35 | .c3-axis-x .tick { 36 | } 37 | .c3-axis-x-label { 38 | } 39 | 40 | .c3-axis-y .tick { 41 | } 42 | .c3-axis-y-label { 43 | } 44 | 45 | .c3-axis-y2 .tick { 46 | } 47 | .c3-axis-y2-label { 48 | } 49 | 50 | /*-- Grid --*/ 51 | 52 | .c3-grid line { 53 | stroke: #aaa; 54 | } 55 | .c3-grid text { 56 | fill: #aaa; 57 | } 58 | .c3-xgrid, .c3-ygrid { 59 | stroke-dasharray: 3 3; 60 | } 61 | .c3-xgrid-focus { 62 | } 63 | 64 | /*-- Text on Chart --*/ 65 | 66 | .c3-text { 67 | } 68 | 69 | .c3-text.c3-empty { 70 | fill: #808080; 71 | font-size: 2em; 72 | } 73 | 74 | /*-- Line --*/ 75 | 76 | .c3-line { 77 | stroke-width: 1px; 78 | } 79 | /*-- Point --*/ 80 | 81 | .c3-circle._expanded_ { 82 | stroke-width: 1px; 83 | stroke: white; 84 | } 85 | .c3-selected-circle { 86 | fill: white; 87 | stroke-width: 2px; 88 | } 89 | 90 | /*-- Bar --*/ 91 | 92 | .c3-bar { 93 | stroke-width: 0; 94 | } 95 | .c3-bar._expanded_ { 96 | fill-opacity: 0.75; 97 | } 98 | 99 | /*-- Arc --*/ 100 | 101 | .c3-chart-arcs-title { 102 | dominant-baseline: middle; 103 | font-size: 1.3em; 104 | } 105 | 106 | /*-- Focus --*/ 107 | 108 | .c3-target.c3-focused { 109 | opacity: 1; 110 | } 111 | .c3-target.c3-focused path.c3-line, .c3-target.c3-focused path.c3-step { 112 | stroke-width: 2px; 113 | } 114 | .c3-target.c3-defocused { 115 | opacity: 0.3 !important; 116 | } 117 | 118 | 119 | /*-- Region --*/ 120 | 121 | .c3-region { 122 | fill: steelblue; 123 | fill-opacity: .1; 124 | } 125 | 126 | /*-- Brush --*/ 127 | 128 | .c3-brush .extent { 129 | fill-opacity: .1; 130 | } 131 | 132 | /*-- Select - Drag --*/ 133 | 134 | .c3-dragarea { 135 | } 136 | 137 | /*-- Legend --*/ 138 | 139 | .c3-legend-item { 140 | font-size: 12px; 141 | } 142 | .c3-legend-item-hidden { 143 | opacity: 0.15; 144 | } 145 | 146 | .c3-legend-background { 147 | opacity: 0.75; 148 | fill: white; 149 | stroke: lightgray; 150 | stroke-width: 1 151 | } 152 | 153 | /*-- Tooltip --*/ 154 | 155 | .c3-tooltip-container { 156 | z-index: 10; 157 | } 158 | .c3-tooltip { 159 | border-collapse:collapse; 160 | border-spacing:0; 161 | background-color:#fff; 162 | empty-cells:show; 163 | -webkit-box-shadow: 7px 7px 12px -9px rgb(119,119,119); 164 | -moz-box-shadow: 7px 7px 12px -9px rgb(119,119,119); 165 | box-shadow: 7px 7px 12px -9px rgb(119,119,119); 166 | opacity: 0.9; 167 | } 168 | .c3-tooltip tr { 169 | border:1px solid #CCC; 170 | } 171 | .c3-tooltip th { 172 | background-color: #aaa; 173 | font-size:14px; 174 | padding:2px 5px; 175 | text-align:left; 176 | color:#FFF; 177 | } 178 | .c3-tooltip td { 179 | font-size:13px; 180 | padding: 3px 6px; 181 | background-color:#fff; 182 | border-left:1px dotted #999; 183 | } 184 | .c3-tooltip td > span { 185 | display: inline-block; 186 | width: 10px; 187 | height: 10px; 188 | margin-right: 6px; 189 | } 190 | .c3-tooltip td.value{ 191 | text-align: right; 192 | } 193 | 194 | .c3-area { 195 | stroke-width: 0; 196 | opacity: 0.2; 197 | } 198 | 199 | .c3-chart-arcs .c3-chart-arcs-background { 200 | fill: #e0e0e0; 201 | stroke: none; 202 | } 203 | .c3-chart-arcs .c3-chart-arcs-gauge-unit { 204 | fill: #000; 205 | font-size: 16px; 206 | } 207 | .c3-chart-arcs .c3-chart-arcs-gauge-max { 208 | fill: #777; 209 | } 210 | .c3-chart-arcs .c3-chart-arcs-gauge-min { 211 | fill: #777; 212 | } 213 | 214 | .c3-chart-arc .c3-gauge-value { 215 | fill: #000; 216 | /* font-size: 28px !important;*/ 217 | } 218 | -------------------------------------------------------------------------------- /static/css/c3.min.css: -------------------------------------------------------------------------------- 1 | .c3 svg{font:10px sans-serif}.c3 line,.c3 path{fill:none;stroke:#000}.c3 text{-webkit-user-select:none;-moz-user-select:none;user-select:none}.c3-bars path,.c3-event-rect,.c3-legend-item-tile,.c3-xgrid-focus,.c3-ygrid{shape-rendering:crispEdges}.c3-chart-arc path{stroke:#fff}.c3-chart-arc text{fill:#fff;font-size:13px}.c3-grid line{stroke:#aaa}.c3-grid text{fill:#aaa}.c3-xgrid,.c3-ygrid{stroke-dasharray:3 3}.c3-text.c3-empty{fill:gray;font-size:2em}.c3-line{stroke-width:1px}.c3-circle._expanded_{stroke-width:1px;stroke:#fff}.c3-selected-circle{fill:#fff;stroke-width:2px}.c3-bar{stroke-width:0}.c3-bar._expanded_{fill-opacity:.75}.c3-chart-arcs-title{dominant-baseline:middle;font-size:1.3em}.c3-target.c3-focused{opacity:1}.c3-target.c3-focused path.c3-line,.c3-target.c3-focused path.c3-step{stroke-width:2px}.c3-target.c3-defocused{opacity:.3!important}.c3-region{fill:#4682b4;fill-opacity:.1}.c3-brush .extent{fill-opacity:.1}.c3-legend-item{font-size:12px}.c3-legend-item-hidden{opacity:.15}.c3-legend-background{opacity:.75;fill:#fff;stroke:#d3d3d3;stroke-width:1}.c3-tooltip-container{z-index:10}.c3-tooltip{border-collapse:collapse;border-spacing:0;background-color:#fff;empty-cells:show;-webkit-box-shadow:7px 7px 12px -9px #777;-moz-box-shadow:7px 7px 12px -9px #777;box-shadow:7px 7px 12px -9px #777;opacity:.9}.c3-tooltip tr{border:1px solid #CCC}.c3-tooltip th{background-color:#aaa;font-size:14px;padding:2px 5px;text-align:left;color:#FFF}.c3-tooltip td{font-size:13px;padding:3px 6px;background-color:#fff;border-left:1px dotted #999}.c3-tooltip td>span{display:inline-block;width:10px;height:10px;margin-right:6px}.c3-tooltip td.value{text-align:right}.c3-area{stroke-width:0;opacity:.2}.c3-chart-arcs .c3-chart-arcs-background{fill:#e0e0e0;stroke:none}.c3-chart-arcs .c3-chart-arcs-gauge-unit{fill:#000;font-size:16px}.c3-chart-arcs .c3-chart-arcs-gauge-max,.c3-chart-arcs .c3-chart-arcs-gauge-min{fill:#777}.c3-chart-arc .c3-gauge-value{fill:#000} -------------------------------------------------------------------------------- /static/css/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 0px; 3 | } 4 | 5 | h3, h4, h5 { 6 | border: none; 7 | border-bottom-width: 0px; 8 | } 9 | 10 | h1, .h1, h2, .h2, h3, .h3 { 11 | margin-top: 6px; 12 | margin-bottom: 3px; 13 | } 14 | 15 | #chart { 16 | margin-top: 40px; 17 | } 18 | 19 | #canvas { 20 | text-align: center; 21 | margin-top: 40px; 22 | margin-bottom: 40px; 23 | } 24 | 25 | .logo { 26 | margin-top: 0px; 27 | vertical-align: bottom; 28 | max-height: 120px; 29 | max-width: 120px; 30 | } 31 | 32 | .navbar-default { 33 | margin-top: 0px; 34 | } 35 | 36 | .heart { 37 | color: red; 38 | font-size: 1.2em; 39 | font-weight: bold; 40 | } 41 | 42 | .input_holder { 43 | margin-top: 20px; 44 | } 45 | 46 | .clearfix { 47 | zoom: 1; 48 | } 49 | 50 | .loading { 51 | height: 30px; 52 | } 53 | 54 | #rule_import_text { 55 | margin-top: 12px; 56 | } -------------------------------------------------------------------------------- /static/css/theme/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | A custom Bootstrap 3.2 'Google Plus style' theme 3 | from http://bootply.com 4 | 5 | This CSS code should follow the 'bootstrap.css' 6 | in your HTML file. 7 | 8 | license: MIT 9 | author: bootply.com 10 | */ 11 | 12 | @import url(http://fonts.googleapis.com/css?family=Roboto:400); 13 | body { 14 | xbackground-color:#e0e0e0; 15 | -webkit-font-smoothing: antialiased; 16 | font: normal 14px Roboto,arial,sans-serif; 17 | } 18 | .navbar-default {background-color:#f4f4f4;margin-top:50px;border-width:0;z-index:5;} 19 | .navbar-default .navbar-nav > .active > a,.navbar-default .navbar-nav > li:hover > a {border:0 solid #4285f4;border-bottom-width:2px;font-weight:800;background-color:transparent;} 20 | .navbar-default .dropdown-menu {background-color:#ffffff;} 21 | .navbar-default .dropdown-menu li > a {padding-left:30px;} 22 | 23 | .header {background-color:#ffffff;border-width:0;} 24 | .header .navbar-collapse {background-color:#ffffff;} 25 | .btn,.form-control,.panel,.list-group,.well {border-radius:1px;box-shadow:0 0 0;} 26 | .form-control {border-color:#d7d7d7;} 27 | .btn-primary {border-color:transparent;} 28 | .btn-primary,.label-primary,.list-group-item.active, .list-group-item.active:hover, .list-group-item.active:focus {background-color:#4285f4;} 29 | .btn-plus {background-color:#ffffff;border-width:1px;border-color:#dddddd;box-shadow:1px 1px 0 #999999;border-radius:3px;color:#666666;text-shadow:0 0 1px #bbbbbb;} 30 | .well,.panel {border-color:#d2d2d2;box-shadow:0 1px 0 #cfcfcf;border-radius:3px;} 31 | .btn-success,.label-success,.progress-bar-success{background-color:#65b045;} 32 | .btn-info,.label-info,.progress-bar-info{background-color:#a0c3ff;border-color:#a0c3ff;} 33 | .btn-danger,.label-danger,.progress-bar-danger{background-color:#dd4b39;} 34 | .btn-warning,.label-warning,.progress-bar-warning{background-color:#f4b400;color:#444444;} 35 | 36 | hr {border-color:#ececec;} 37 | button { 38 | outline: 0; 39 | } 40 | textarea { 41 | resize: none; 42 | outline: 0; 43 | } 44 | .panel .btn i,.btn span{ 45 | color:#666666; 46 | } 47 | .panel .panel-heading { 48 | background-color:#ffffff; 49 | font-weight:700; 50 | font-size:16px; 51 | color:#262626; 52 | border-color:#ffffff; 53 | } 54 | .panel .panel-heading a { 55 | font-weight:400; 56 | font-size:11px; 57 | } 58 | .panel .panel-default { 59 | border-color:#cccccc; 60 | } 61 | .panel .panel-thumbnail { 62 | padding:0; 63 | } 64 | .panel .img-circle { 65 | width:50px; 66 | height:50px; 67 | } 68 | .list-group-item:first-child,.list-group-item:last-child { 69 | border-radius:0; 70 | } 71 | h3,h4,h5 { 72 | border:0 solid #efefef; 73 | border-bottom-width:1px; 74 | padding-bottom:10px; 75 | } 76 | .modal-dialog { 77 | width: 450px; 78 | } 79 | .modal-footer { 80 | border-width:0; 81 | } 82 | .dropdown-menu { 83 | background-color:#f4f4f4; 84 | border-color:#f0f0f0; 85 | border-radius:0; 86 | margin-top:-1px; 87 | } 88 | /* end theme */ 89 | 90 | /* template layout*/ 91 | #subnav { 92 | position:fixed; 93 | width:100%; 94 | } 95 | 96 | @media (max-width: 768px) { 97 | #subnav { 98 | padding-top: 6px; 99 | } 100 | } 101 | 102 | #main { 103 | padding-top:120px; 104 | } -------------------------------------------------------------------------------- /static/css/tipsy.css: -------------------------------------------------------------------------------- 1 | .tipsy { font-size: 10px; position: absolute; padding: 5px; z-index: 100000; } 2 | .tipsy-inner { background-color: #000; color: #FFF; max-width: 200px; padding: 5px 8px 4px 8px; text-align: center; } 3 | 4 | /* Rounded corners */ 5 | .tipsy-inner { border-radius: 3px; -moz-border-radius: 3px; -webkit-border-radius: 3px; } 6 | 7 | /* Uncomment for shadow */ 8 | /*.tipsy-inner { box-shadow: 0 0 5px #000000; -webkit-box-shadow: 0 0 5px #000000; -moz-box-shadow: 0 0 5px #000000; }*/ 9 | 10 | .tipsy-arrow { position: absolute; width: 0; height: 0; line-height: 0; border: 5px dashed #000; } 11 | 12 | /* Rules to colour arrows */ 13 | .tipsy-arrow-n { border-bottom-color: #000; } 14 | .tipsy-arrow-s { border-top-color: #000; } 15 | .tipsy-arrow-e { border-left-color: #000; } 16 | .tipsy-arrow-w { border-right-color: #000; } 17 | 18 | .tipsy-n .tipsy-arrow { top: 0px; left: 50%; margin-left: -5px; border-bottom-style: solid; border-top: none; border-left-color: transparent; border-right-color: transparent; } 19 | .tipsy-nw .tipsy-arrow { top: 0; left: 10px; border-bottom-style: solid; border-top: none; border-left-color: transparent; border-right-color: transparent;} 20 | .tipsy-ne .tipsy-arrow { top: 0; right: 10px; border-bottom-style: solid; border-top: none; border-left-color: transparent; border-right-color: transparent;} 21 | .tipsy-s .tipsy-arrow { bottom: 0; left: 50%; margin-left: -5px; border-top-style: solid; border-bottom: none; border-left-color: transparent; border-right-color: transparent; } 22 | .tipsy-sw .tipsy-arrow { bottom: 0; left: 10px; border-top-style: solid; border-bottom: none; border-left-color: transparent; border-right-color: transparent; } 23 | .tipsy-se .tipsy-arrow { bottom: 0; right: 10px; border-top-style: solid; border-bottom: none; border-left-color: transparent; border-right-color: transparent; } 24 | .tipsy-e .tipsy-arrow { right: 0; top: 50%; margin-top: -5px; border-left-style: solid; border-right: none; border-top-color: transparent; border-bottom-color: transparent; } 25 | .tipsy-w .tipsy-arrow { left: 0; top: 50%; margin-top: -5px; border-right-style: solid; border-left: none; border-top-color: transparent; border-bottom-color: transparent; } 26 | -------------------------------------------------------------------------------- /static/img/bird.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/bird.ico -------------------------------------------------------------------------------- /static/img/bird_large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/bird_large.png -------------------------------------------------------------------------------- /static/img/config_fields.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/config_fields.png -------------------------------------------------------------------------------- /static/img/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/loading.gif -------------------------------------------------------------------------------- /static/img/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/screenshot.png -------------------------------------------------------------------------------- /static/img/settings_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/settings_1.png -------------------------------------------------------------------------------- /static/img/settings_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xdevplatform/twitter-for-bigquery/8dcbcd28f813587e8f53e4436ad3e7e61f1c0a37/static/img/settings_2.png -------------------------------------------------------------------------------- /templates/admin.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block bodycontent %} 3 | 4 |
5 | 6 | View project 7 | 8 | 9 | View instances 10 | 11 | 12 | View task queue 13 | 14 | 15 | View logs 16 | 17 |
18 | 19 |

Administration

20 | 21 |

Configure GNIP and Twitter settings.

22 | 23 |
24 | 25 |
26 |

Gnip

27 |
28 | 29 |
30 | 31 |
32 |
33 |
34 | 35 |
36 | 37 |
38 |
39 |
40 | 41 |
42 | 43 |
44 |
45 |

BigQuery

46 |
47 | 48 |
49 | 50 |
51 |
52 |
53 | 54 |
55 | 56 |
57 |
58 |
59 | 60 |
61 | 62 |
63 |
64 |
65 | 66 |
67 | File on Record 68 |
69 |
70 | 71 |
72 |
73 |
74 |
75 | 76 |
77 |
78 | 79 | 80 | 89 | {% endblock %} 90 | -------------------------------------------------------------------------------- /templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Twitter ♥ BigQuery 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 52 | 53 |
54 | 55 |
56 |
57 |
58 | 59 |

Something went wrong.

60 |
61 |
62 |
63 | 64 | {% block bodycontent %} 65 | Replace this 66 | {% endblock %} 67 | 68 |
69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /templates/chart_partial.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | 8 | 9 |
10 | 11 |
12 | 13 |
14 | 15 |

Table

16 | 19 | 20 |
21 | 22 |
23 | 24 |

Field

25 | 31 | 32 |
33 | 34 |
35 | 36 |

Chart

37 | 42 | 43 |
44 | 45 |
46 | 47 |

Time

48 | 54 | 55 |
56 | 57 |
58 | 59 |
60 | 61 |
62 |

 

63 | 64 |
65 | 66 | 67 |
68 | 69 | 70 | 71 |
72 | 73 |
74 | 75 |
Please select a data source and click "Submit".
76 | 77 |
78 | -------------------------------------------------------------------------------- /templates/home.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block bodycontent %} 3 | 4 | {% include "chart_partial.html" %} 5 | 6 |
7 | 8 |
9 |
10 |
11 | 16 |
17 | 18 |
19 |
20 | 21 |
22 | 23 |
24 | 25 | 34 | {% endblock %} 35 | -------------------------------------------------------------------------------- /templates/rule_add_partial.html: -------------------------------------------------------------------------------- 1 | 2 | 58 | -------------------------------------------------------------------------------- /templates/rule_list.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block bodycontent %} 3 | 4 |
5 | 8 |
9 | 10 |

PowerTrack Rules

11 | 12 |

Add or remove rules to your stream to update data piped to BigQuery.

13 | 14 |
15 | 16 |
17 | 18 |
19 | 20 | 21 | 22 | 23 | 24 | 37 | 38 | 39 | 40 | 41 |
RuleDatasetHPT JSON 25 | 26 |
27 | 30 | 34 |
35 | 36 |
42 | 43 | 44 | 45 | {% include "rule_add_partial.html" %} 46 | 47 | {% raw %} 48 | 64 | {% endraw %} 65 | 66 | 76 | 77 | {% endblock %} -------------------------------------------------------------------------------- /templates/table_detail.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block bodycontent %} 3 | 4 |
5 | View in BigQuery 6 | Delete 7 |
8 | 9 |

10 | Dataset: {{tableReference.datasetId}}.{{tableReference.tableId}} 11 |

12 | 13 |

14 | 15 |
16 |
17 |
{{tableReference.projectId}}
18 |
19 |
{{creationTime}}
20 |
21 | 22 |
23 |
24 |
Gnip
25 |
26 |
{{lastModifiedTime}}
27 |
28 | 29 |
30 | 31 | 32 | 33 | 34 |
35 | 38 | 42 |
43 | 44 | 60 | 61 | 62 |
63 |
64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 |
RuleDatasetHPT JSON
77 | 78 |
79 | 80 |
81 | 82 |
83 | 84 |
85 |
86 | 87 | 91 | 92 |
93 |
94 | 95 |
96 |
97 | 98 |
99 |
100 |

Users

101 |
102 |
103 |

See unique users who have tweeted in this data set, and optionally import their tweets via new Gnip rules.

104 | Show unique users 105 | Add users to data set 106 | 107 |

108 | 109 |

.

110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 |
UserTweet Count
121 | 122 |
123 |
124 | 125 |
126 | 127 |
128 | 129 |
130 |
131 |

Data

132 |
133 |
134 |

Delete all data from this table.

135 | Delete data 136 | 137 |
138 |
139 | 140 |
141 |
142 | 143 |
144 |
145 | 146 |
147 | 148 | 149 |
150 | 151 |
152 | 153 | {% include "chart_partial.html" %} 154 | 155 |
156 |
157 | 158 | 159 | {% include "rule_add_partial.html" %} 160 | 161 | {% raw %} 162 | 163 | 179 | 180 | 190 | 191 | {% endraw %} 192 | 193 | 203 | 204 | {% endblock %} -------------------------------------------------------------------------------- /templates/table_list.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block bodycontent %} 3 | 4 |
5 | 8 |
9 | 10 |

BigQuery Tables

11 | 12 |

View and manage BigQuery tables in your project.

13 | 14 |
15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
TableProjectTypeRulesActions
31 | 32 |
33 | 34 | 35 | 107 | 108 | {% raw %} 109 | 131 | {% endraw %} 132 | 133 | 143 | 144 | {% endblock %} 145 | --------------------------------------------------------------------------------