├── LICENSE ├── MANIFEST.in ├── NYTimesArticleAPI ├── __init__.py ├── nytapi.py ├── search_api.py └── tests.py ├── README.md ├── README.rst ├── ez_setup.py ├── setup.cfg └── setup.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Matt Morrison 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.* 3 | -------------------------------------------------------------------------------- /NYTimesArticleAPI/__init__.py: -------------------------------------------------------------------------------- 1 | from search_api import * 2 | 3 | __version__ = "1.0.0" 4 | __author__ = "Matt Morrison (@MattDMo)" 5 | __all__ = ["articleAPI"] 6 | 7 | if __name__ == "__main__": 8 | print("This module cannot be run on its own. Please use by running ", 9 | "\"from NYTimesArticleAPI import articleAPI\"") 10 | exit(0) 11 | -------------------------------------------------------------------------------- /NYTimesArticleAPI/nytapi.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | API_ROOT = 'http://api.nytimes.com/svc/search/v2/articlesearch.' 4 | 5 | API_SIGNUP_PAGE = 'http://developer.nytimes.com/docs/reference/keys' 6 | 7 | class NoAPIKeyException(Exception): 8 | def __init__(self, value): 9 | self.value = value 10 | def __str__(self): 11 | return repr(self.value) 12 | 13 | class articleAPI(object): 14 | def __init__(self, key = None): 15 | """ 16 | Initializes the articleAPI class with a developer key. Raises an exception if a key is not given. 17 | 18 | Request a key at http://developer.nytimes.com/docs/reference/keys 19 | 20 | :param key: New York Times Developer Key 21 | 22 | """ 23 | self.key = key 24 | self.response_format = 'json' 25 | 26 | if self.key is None: 27 | raise NoAPIKeyException('Warning: Missing API Key. Please visit ' + API_SIGNUP_PAGE + ' to register for a key.') 28 | 29 | def _utf8_encode(self, d): 30 | """ 31 | Ensures all values are encoded in UTF-8 and converts them to lowercase 32 | 33 | """ 34 | for k, v in d.items(): 35 | if isinstance(v, str): 36 | d[k] = v.encode('utf8').lower() 37 | if isinstance(v, list): 38 | for index,item in enumerate(v): 39 | item = item.encode('utf8').lower() 40 | v[index] = item 41 | if isinstance(v, dict): 42 | d[k] = self._utf8_encode(v) 43 | 44 | return d 45 | 46 | def _bool_encode(self, d): 47 | """ 48 | Converts bool values to lowercase strings 49 | 50 | """ 51 | for k, v in d.items(): 52 | if isinstance(v, bool): 53 | d[k] = str(v).lower() 54 | 55 | return d 56 | 57 | def _options(self, **kwargs): 58 | """ 59 | Formats search parameters/values for use with API 60 | 61 | :param \*\*kwargs: search parameters/values 62 | 63 | """ 64 | def _format_fq(d): 65 | for k,v in d.items(): 66 | if isinstance(v, list): 67 | d[k] = ' '.join(map(lambda x: '"' + x + '"', v)) 68 | else: 69 | d[k] = '"' + v + '"' 70 | values = [] 71 | for k,v in d.items(): 72 | value = '%s:(%s)' % (k,v) 73 | values.append(value) 74 | values = ' AND '.join(values) 75 | return values 76 | 77 | kwargs = self._utf8_encode(kwargs) 78 | kwargs = self._bool_encode(kwargs) 79 | 80 | values = '' 81 | 82 | for k, v in kwargs.items(): 83 | if k is 'fq' and isinstance(v, dict): 84 | v = _format_fq(v) 85 | elif isinstance(v, list): 86 | v = ','.join(v) 87 | values += '%s=%s&' % (k, v) 88 | 89 | return values 90 | 91 | def search(self, 92 | response_format = None, 93 | key = None, 94 | **kwargs): 95 | """ 96 | Calls the API and returns a dictionary of the search results 97 | 98 | :param response_format: the format that the API uses for its response, 99 | includes JSON (.json) and JSONP (.jsonp). 100 | Defaults to '.json'. 101 | 102 | :param key: a developer key. Defaults to key given when the articleAPI class was initialized. 103 | 104 | """ 105 | if response_format is None: 106 | response_format = self.response_format 107 | if key is None: 108 | key = self.key 109 | 110 | url = '%s%s?%sapi-key=%s' % ( 111 | API_ROOT, response_format, self._options(**kwargs), key 112 | ) 113 | 114 | r = requests.get(url) 115 | return r.json() 116 | -------------------------------------------------------------------------------- /NYTimesArticleAPI/search_api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | API_ROOT = 'http://api.nytimes.com/svc/search/v2/articlesearch.' 4 | 5 | API_SIGNUP_PAGE = 'http://developer.nytimes.com/docs/reference/keys' 6 | 7 | class NoAPIKeyException(Exception): 8 | def __init__(self, value): 9 | self.value = value 10 | def __str__(self): 11 | return repr(self.value) 12 | 13 | class articleAPI(object): 14 | def __init__(self, key = None): 15 | """ 16 | Initializes the articleAPI class with a developer key. Raises an exception if a key is not given. 17 | 18 | Request a key at http://developer.nytimes.com/docs/reference/keys 19 | 20 | :param key: New York Times Developer Key 21 | 22 | """ 23 | self.key = key 24 | self.response_format = 'json' 25 | 26 | if self.key is None: 27 | raise NoAPIKeyException('Warning: Missing API Key. Please visit ' + API_SIGNUP_PAGE + ' to register for a key.') 28 | 29 | def _bool_encode(self, d): 30 | """ 31 | Converts boolean values to lowercase strings 32 | 33 | """ 34 | for k, v in d.items(): 35 | if isinstance(v, bool): 36 | d[k] = str(v).lower() 37 | 38 | return d 39 | 40 | def _options(self, **kwargs): 41 | """ 42 | Formats search parameters/values for use with API 43 | 44 | :param \*\*kwargs: search parameters/values 45 | 46 | """ 47 | def _format_fq(d): 48 | for k,v in d.items(): 49 | if isinstance(v, list): 50 | d[k] = ' '.join(map(lambda x: '"' + x + '"', v)) 51 | else: 52 | d[k] = '"' + v + '"' 53 | values = [] 54 | for k,v in d.items(): 55 | value = '%s:(%s)' % (k,v) 56 | values.append(value) 57 | values = ' AND '.join(values) 58 | return values 59 | 60 | kwargs = self._bool_encode(kwargs) 61 | 62 | values = '' 63 | 64 | for k, v in kwargs.items(): 65 | if k is 'fq' and isinstance(v, dict): 66 | v = _format_fq(v) 67 | elif isinstance(v, list): 68 | v = ','.join(v) 69 | values += '%s=%s&' % (k, v) 70 | 71 | return values 72 | 73 | def search(self, 74 | response_format = None, 75 | key = None, 76 | **kwargs): 77 | """ 78 | Calls the API and returns a dictionary of the search results 79 | 80 | :param response_format: the format that the API uses for its response, 81 | includes JSON (.json) and JSONP (.jsonp). 82 | Defaults to '.json'. 83 | 84 | :param key: a developer key. Defaults to key given when the articleAPI class was initialized. 85 | 86 | """ 87 | if response_format is None: 88 | response_format = self.response_format 89 | if key is None: 90 | key = self.key 91 | 92 | url = '%s%s?%sapi-key=%s' % ( 93 | API_ROOT, response_format, self._options(**kwargs), key 94 | ) 95 | 96 | self.req = requests.get(url) 97 | return self.req.json() 98 | -------------------------------------------------------------------------------- /NYTimesArticleAPI/tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | 4 | 5 | class NYTimesArticleAPITests(unittest.TestCase): 6 | def __init__(self): 7 | self.api_key = os.environ["NYT_API_KEY"] 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NYTimesArticleAPI 2 | 3 | `NYTimesArticleAPI` is a Python wrapper for the [New York Times Article Search API][1]. Based on the excellent [`requests`][13] package, it provides full support for all of the API's search parameters, and also allows access to the request object itself for debugging purposes. 4 | 5 | 6 | ## Installation 7 | 8 | With pip: 9 | 10 | $ pip install NYTimesArticleAPI 11 | 12 | 13 | ## Dependencies 14 | 15 | NYTimesArticleAPI requires the [`requests`][2] and [`setuptools`][3] packages. 16 | 17 | 18 | ## Usage 19 | 20 | Simply import and initialize the API with your developer key: 21 | 22 | ```python 23 | >>> from NYTimesArticleAPI import articleAPI 24 | >>> api = articleAPI("YourAPIKey") 25 | ``` 26 | 27 | Then call the `search` function with your desired search parameters/values: 28 | 29 | ```python 30 | >>> articles = api.search(q="Obama", 31 | fq={"headline": "Obama", 32 | "source": ["Reuters", 33 | "AP", 34 | "The New York Times"]}, 35 | begin_date="20161001", # this can also be an int 36 | facet_field=["source", "day_of_week"], 37 | facet_filter=True) 38 | ``` 39 | 40 | The search function returns a Python dictionary of the search results. 41 | 42 | You can specify multiple filters by using a dictionary:: 43 | 44 | ```python 45 | >>> fq = {"headline": "Obama", "source": ["Reuters", "AP", "The New York Times"]} 46 | >>> articles = api.search(q="Obama", fq=fq) 47 | ``` 48 | 49 | And multiple values by using a list:: 50 | 51 | ```python 52 | >>> facet_field = ["source", "day_of_week"] 53 | >>> articles = api.search(q="Obama", facet_field=facet_field) 54 | ``` 55 | 56 | More examples: 57 | 58 | ```python 59 | # simple search 60 | >>> articles = api.search(q="Obama") 61 | # search between specific dates 62 | >>> articles = api.search(q="Obama", begin_date="20161001", end_date="20161020", page=2) 63 | # access most recent request object 64 | >>> headers = api.req.headers 65 | ``` 66 | 67 | For a complete overview of the available search parameters, please refer to the [NYTimes Article Search API Documentation][4]. 68 | 69 | 70 | ## History 71 | 72 | This package was originally written by [Evan Sherlock][5] as [`nytimesarticle`][6]. It has since been forked and updated by [Matt Morrison][7], and subsequently released as [`NyTimesArticleAPI`][8], with contributions from [Gerald Spencer][9] and [Andrew Han][10]. 73 | 74 | 75 | ## License 76 | 77 | © 2016 Matt Morrison . 78 | 79 | This is free software. It is licensed under the [MIT License][11]. Feel free to use this in your own work. However, if you modify and/or redistribute it, please attribute me in some way, and distribute your work under this or a similar license. A shout-out or a beer would be appreciated. 80 | 81 | You can support development of this project on [Gratipay][12]. 82 | 83 | 84 | [1]: https://developer.nytimes.com/article_search_v2.json 85 | [2]: https://pypi.python.org/pypi/requests 86 | [3]: https://pypi.python.org/pypi/setuptools 87 | [4]: http://developer.nytimes.com/docs/read/article_search_api_v2 88 | [5]: https://github.com/evansherlock 89 | [6]: https://github.com/evansherlock/nytimesarticle 90 | [7]: https://github.com/MattDMo 91 | [8]: https://pypi.python.org/pypi/NYTimesArticleAPI 92 | [9]: https://github.com/Geethree 93 | [10]: https://github.com/handrew 94 | [11]: http://opensource.org/licenses/MIT 95 | [12]: https://www.gratipay.com/on/github/MattDMo/ 96 | [13]: http://docs.python-requests.org 97 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | NYTimesArticleAPI 2 | ================= 3 | 4 | ``NYTimesArticleAPI`` is a fully-functional Python wrapper for the `New 5 | York Times Article Search 6 | API `__. 7 | 8 | Installation 9 | ------------ 10 | 11 | With pip: 12 | 13 | :: 14 | 15 | $ pip install NYTimesArticleAPI 16 | 17 | Dependencies 18 | ------------ 19 | 20 | NYTimesArticleAPI requires the 21 | ```requests`` `__ and 22 | ```setuptools`` `__ packages. 23 | 24 | Usage 25 | ----- 26 | 27 | Simply import and initialize the API with your developer key: 28 | 29 | .. code:: python 30 | 31 | >>> from NYTimesArticleAPI import articleAPI 32 | >>> api = articleAPI("YourAPIKey") 33 | 34 | Then call the ``search`` function with your desired search 35 | parameters/values: 36 | 37 | .. code:: python 38 | 39 | >>> articles = api.search(q="Obama", 40 | fq={"headline": "Obama", 41 | "source": ["Reuters", 42 | "AP", 43 | "The New York Times"]}, 44 | begin_date="20161023", # this can also be an int 45 | facet_field=["source", "day_of_week"], 46 | facet_filter=True) 47 | 48 | The search function returns a Python dictionary of the search results. 49 | 50 | You can specify multiple filters by using a dictionary:: 51 | 52 | .. code:: python 53 | 54 | >>> fq = {"headline": "Obama", "source": ["Reuters", "AP", "The New York Times"]} 55 | 56 | And multiple values by using a list:: 57 | 58 | .. code:: python 59 | 60 | >>> facet_field = ["source", "day_of_week"] 61 | 62 | More examples: 63 | 64 | .. code:: python 65 | 66 | >>> articles = api.search(q="Obama") 67 | 68 | >>> articles = api.search(q="Obama", begin_date="20111231", page=2) 69 | 70 | For a complete overview of the available search parameters, please refer 71 | to the `NYTimes Article Search API 72 | Documentation `__. 73 | 74 | History 75 | ------- 76 | 77 | This package was originally written by `Evan 78 | Sherlock `__ as 79 | ```nytimesarticle`` `__. 80 | It has since been forked and updated by `Matt 81 | Morrison `__, and subsequently released as 82 | ```NyTimesArticleAPI`` `__, 83 | with contributions from `Gerald Spencer `__ 84 | and `Andrew Han `__. 85 | 86 | License 87 | ------- 88 | 89 | © 2016 Matt Morrison mattdmo@pigimal.com. 90 | 91 | This is free software. It is licensed under the `MIT 92 | License `__. Feel free to use this 93 | in your own work. However, if you modify and/or redistribute it, please 94 | attribute me in some way, and distribute your work under this or a 95 | similar license. A shout-out or a beer would be appreciated. 96 | -------------------------------------------------------------------------------- /ez_setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Setuptools bootstrapping installer. 5 | 6 | Maintained at https://github.com/pypa/setuptools/tree/bootstrap. 7 | 8 | Run this script to install or upgrade setuptools. 9 | """ 10 | 11 | import os 12 | import shutil 13 | import sys 14 | import tempfile 15 | import zipfile 16 | import optparse 17 | import subprocess 18 | import platform 19 | import textwrap 20 | import contextlib 21 | import json 22 | import codecs 23 | 24 | from distutils import log 25 | 26 | try: 27 | from urllib.request import urlopen 28 | from urllib.parse import urljoin 29 | except ImportError: 30 | from urllib2 import urlopen 31 | from urlparse import urljoin 32 | 33 | try: 34 | from site import USER_SITE 35 | except ImportError: 36 | USER_SITE = None 37 | 38 | LATEST = object() 39 | DEFAULT_VERSION = LATEST 40 | DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/" 41 | DEFAULT_SAVE_DIR = os.curdir 42 | 43 | 44 | def _python_cmd(*args): 45 | """ 46 | Execute a command. 47 | 48 | Return True if the command succeeded. 49 | """ 50 | args = (sys.executable,) + args 51 | return subprocess.call(args) == 0 52 | 53 | 54 | def _install(archive_filename, install_args=()): 55 | """Install Setuptools.""" 56 | with archive_context(archive_filename): 57 | # installing 58 | log.warn('Installing Setuptools') 59 | if not _python_cmd('setup.py', 'install', *install_args): 60 | log.warn('Something went wrong during the installation.') 61 | log.warn('See the error message above.') 62 | # exitcode will be 2 63 | return 2 64 | 65 | 66 | def _build_egg(egg, archive_filename, to_dir): 67 | """Build Setuptools egg.""" 68 | with archive_context(archive_filename): 69 | # building an egg 70 | log.warn('Building a Setuptools egg in %s', to_dir) 71 | _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) 72 | # returning the result 73 | log.warn(egg) 74 | if not os.path.exists(egg): 75 | raise IOError('Could not build the egg.') 76 | 77 | 78 | class ContextualZipFile(zipfile.ZipFile): 79 | 80 | """Supplement ZipFile class to support context manager for Python 2.6.""" 81 | 82 | def __enter__(self): 83 | return self 84 | 85 | def __exit__(self, type, value, traceback): 86 | self.close() 87 | 88 | def __new__(cls, *args, **kwargs): 89 | """Construct a ZipFile or ContextualZipFile as appropriate.""" 90 | if hasattr(zipfile.ZipFile, '__exit__'): 91 | return zipfile.ZipFile(*args, **kwargs) 92 | return super(ContextualZipFile, cls).__new__(cls) 93 | 94 | 95 | @contextlib.contextmanager 96 | def archive_context(filename): 97 | """ 98 | Unzip filename to a temporary directory, set to the cwd. 99 | 100 | The unzipped target is cleaned up after. 101 | """ 102 | tmpdir = tempfile.mkdtemp() 103 | log.warn('Extracting in %s', tmpdir) 104 | old_wd = os.getcwd() 105 | try: 106 | os.chdir(tmpdir) 107 | with ContextualZipFile(filename) as archive: 108 | archive.extractall() 109 | 110 | # going in the directory 111 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 112 | os.chdir(subdir) 113 | log.warn('Now working in %s', subdir) 114 | yield 115 | 116 | finally: 117 | os.chdir(old_wd) 118 | shutil.rmtree(tmpdir) 119 | 120 | 121 | def _do_download(version, download_base, to_dir, download_delay): 122 | """Download Setuptools.""" 123 | py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys) 124 | tp = 'setuptools-{version}-{py_desig}.egg' 125 | egg = os.path.join(to_dir, tp.format(**locals())) 126 | if not os.path.exists(egg): 127 | archive = download_setuptools(version, download_base, 128 | to_dir, download_delay) 129 | _build_egg(egg, archive, to_dir) 130 | sys.path.insert(0, egg) 131 | 132 | # Remove previously-imported pkg_resources if present (see 133 | # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). 134 | if 'pkg_resources' in sys.modules: 135 | _unload_pkg_resources() 136 | 137 | import setuptools 138 | setuptools.bootstrap_install_from = egg 139 | 140 | 141 | def use_setuptools( 142 | version=DEFAULT_VERSION, download_base=DEFAULT_URL, 143 | to_dir=DEFAULT_SAVE_DIR, download_delay=15): 144 | """ 145 | Ensure that a setuptools version is installed. 146 | 147 | Return None. Raise SystemExit if the requested version 148 | or later cannot be installed. 149 | """ 150 | version = _resolve_version(version) 151 | to_dir = os.path.abspath(to_dir) 152 | 153 | # prior to importing, capture the module state for 154 | # representative modules. 155 | rep_modules = 'pkg_resources', 'setuptools' 156 | imported = set(sys.modules).intersection(rep_modules) 157 | 158 | try: 159 | import pkg_resources 160 | pkg_resources.require("setuptools>=" + version) 161 | # a suitable version is already installed 162 | return 163 | except ImportError: 164 | # pkg_resources not available; setuptools is not installed; download 165 | pass 166 | except pkg_resources.DistributionNotFound: 167 | # no version of setuptools was found; allow download 168 | pass 169 | except pkg_resources.VersionConflict as VC_err: 170 | if imported: 171 | _conflict_bail(VC_err, version) 172 | 173 | # otherwise, unload pkg_resources to allow the downloaded version to 174 | # take precedence. 175 | del pkg_resources 176 | _unload_pkg_resources() 177 | 178 | return _do_download(version, download_base, to_dir, download_delay) 179 | 180 | 181 | def _conflict_bail(VC_err, version): 182 | """ 183 | Setuptools was imported prior to invocation, so it is 184 | unsafe to unload it. Bail out. 185 | """ 186 | conflict_tmpl = textwrap.dedent(""" 187 | The required version of setuptools (>={version}) is not available, 188 | and can't be installed while this script is running. Please 189 | install a more recent version first, using 190 | 'easy_install -U setuptools'. 191 | 192 | (Currently using {VC_err.args[0]!r}) 193 | """) 194 | msg = conflict_tmpl.format(**locals()) 195 | sys.stderr.write(msg) 196 | sys.exit(2) 197 | 198 | 199 | def _unload_pkg_resources(): 200 | sys.meta_path = [ 201 | importer 202 | for importer in sys.meta_path 203 | if importer.__class__.__module__ != 'pkg_resources.extern' 204 | ] 205 | del_modules = [ 206 | name for name in sys.modules 207 | if name.startswith('pkg_resources') 208 | ] 209 | for mod_name in del_modules: 210 | del sys.modules[mod_name] 211 | 212 | 213 | def _clean_check(cmd, target): 214 | """ 215 | Run the command to download target. 216 | 217 | If the command fails, clean up before re-raising the error. 218 | """ 219 | try: 220 | subprocess.check_call(cmd) 221 | except subprocess.CalledProcessError: 222 | if os.access(target, os.F_OK): 223 | os.unlink(target) 224 | raise 225 | 226 | 227 | def download_file_powershell(url, target): 228 | """ 229 | Download the file at url to target using Powershell. 230 | 231 | Powershell will validate trust. 232 | Raise an exception if the command cannot complete. 233 | """ 234 | target = os.path.abspath(target) 235 | ps_cmd = ( 236 | "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " 237 | "[System.Net.CredentialCache]::DefaultCredentials; " 238 | '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' 239 | % locals() 240 | ) 241 | cmd = [ 242 | 'powershell', 243 | '-Command', 244 | ps_cmd, 245 | ] 246 | _clean_check(cmd, target) 247 | 248 | 249 | def has_powershell(): 250 | """Determine if Powershell is available.""" 251 | if platform.system() != 'Windows': 252 | return False 253 | cmd = ['powershell', '-Command', 'echo test'] 254 | with open(os.path.devnull, 'wb') as devnull: 255 | try: 256 | subprocess.check_call(cmd, stdout=devnull, stderr=devnull) 257 | except Exception: 258 | return False 259 | return True 260 | download_file_powershell.viable = has_powershell 261 | 262 | 263 | def download_file_curl(url, target): 264 | cmd = ['curl', url, '--location', '--silent', '--output', target] 265 | _clean_check(cmd, target) 266 | 267 | 268 | def has_curl(): 269 | cmd = ['curl', '--version'] 270 | with open(os.path.devnull, 'wb') as devnull: 271 | try: 272 | subprocess.check_call(cmd, stdout=devnull, stderr=devnull) 273 | except Exception: 274 | return False 275 | return True 276 | download_file_curl.viable = has_curl 277 | 278 | 279 | def download_file_wget(url, target): 280 | cmd = ['wget', url, '--quiet', '--output-document', target] 281 | _clean_check(cmd, target) 282 | 283 | 284 | def has_wget(): 285 | cmd = ['wget', '--version'] 286 | with open(os.path.devnull, 'wb') as devnull: 287 | try: 288 | subprocess.check_call(cmd, stdout=devnull, stderr=devnull) 289 | except Exception: 290 | return False 291 | return True 292 | download_file_wget.viable = has_wget 293 | 294 | 295 | def download_file_insecure(url, target): 296 | """Use Python to download the file, without connection authentication.""" 297 | src = urlopen(url) 298 | try: 299 | # Read all the data in one block. 300 | data = src.read() 301 | finally: 302 | src.close() 303 | 304 | # Write all the data in one block to avoid creating a partial file. 305 | with open(target, "wb") as dst: 306 | dst.write(data) 307 | download_file_insecure.viable = lambda: True 308 | 309 | 310 | def get_best_downloader(): 311 | downloaders = ( 312 | download_file_powershell, 313 | download_file_curl, 314 | download_file_wget, 315 | download_file_insecure, 316 | ) 317 | viable_downloaders = (dl for dl in downloaders if dl.viable()) 318 | return next(viable_downloaders, None) 319 | 320 | 321 | def download_setuptools( 322 | version=DEFAULT_VERSION, download_base=DEFAULT_URL, 323 | to_dir=DEFAULT_SAVE_DIR, delay=15, 324 | downloader_factory=get_best_downloader): 325 | """ 326 | Download setuptools from a specified location and return its filename. 327 | 328 | `version` should be a valid setuptools version number that is available 329 | as an sdist for download under the `download_base` URL (which should end 330 | with a '/'). `to_dir` is the directory where the egg will be downloaded. 331 | `delay` is the number of seconds to pause before an actual download 332 | attempt. 333 | 334 | ``downloader_factory`` should be a function taking no arguments and 335 | returning a function for downloading a URL to a target. 336 | """ 337 | version = _resolve_version(version) 338 | # making sure we use the absolute path 339 | to_dir = os.path.abspath(to_dir) 340 | zip_name = "setuptools-%s.zip" % version 341 | url = download_base + zip_name 342 | saveto = os.path.join(to_dir, zip_name) 343 | if not os.path.exists(saveto): # Avoid repeated downloads 344 | log.warn("Downloading %s", url) 345 | downloader = downloader_factory() 346 | downloader(url, saveto) 347 | return os.path.realpath(saveto) 348 | 349 | 350 | def _resolve_version(version): 351 | """ 352 | Resolve LATEST version 353 | """ 354 | if version is not LATEST: 355 | return version 356 | 357 | meta_url = urljoin(DEFAULT_URL, '/pypi/setuptools/json') 358 | resp = urlopen(meta_url) 359 | with contextlib.closing(resp): 360 | try: 361 | charset = resp.info().get_content_charset() 362 | except Exception: 363 | # Python 2 compat; assume UTF-8 364 | charset = 'UTF-8' 365 | reader = codecs.getreader(charset) 366 | doc = json.load(reader(resp)) 367 | 368 | return str(doc['info']['version']) 369 | 370 | 371 | def _build_install_args(options): 372 | """ 373 | Build the arguments to 'python setup.py install' on the setuptools package. 374 | 375 | Returns list of command line arguments. 376 | """ 377 | return ['--user'] if options.user_install else [] 378 | 379 | 380 | def _parse_args(): 381 | """Parse the command line for options.""" 382 | parser = optparse.OptionParser() 383 | parser.add_option( 384 | '--user', dest='user_install', action='store_true', default=False, 385 | help='install in user site package') 386 | parser.add_option( 387 | '--download-base', dest='download_base', metavar="URL", 388 | default=DEFAULT_URL, 389 | help='alternative URL from where to download the setuptools package') 390 | parser.add_option( 391 | '--insecure', dest='downloader_factory', action='store_const', 392 | const=lambda: download_file_insecure, default=get_best_downloader, 393 | help='Use internal, non-validating downloader' 394 | ) 395 | parser.add_option( 396 | '--version', help="Specify which version to download", 397 | default=DEFAULT_VERSION, 398 | ) 399 | parser.add_option( 400 | '--to-dir', 401 | help="Directory to save (and re-use) package", 402 | default=DEFAULT_SAVE_DIR, 403 | ) 404 | options, args = parser.parse_args() 405 | # positional arguments are ignored 406 | return options 407 | 408 | 409 | def _download_args(options): 410 | """Return args for download_setuptools function from cmdline args.""" 411 | return dict( 412 | version=options.version, 413 | download_base=options.download_base, 414 | downloader_factory=options.downloader_factory, 415 | to_dir=options.to_dir, 416 | ) 417 | 418 | 419 | def main(): 420 | """Install or upgrade setuptools and EasyInstall.""" 421 | options = _parse_args() 422 | archive = download_setuptools(**_download_args(options)) 423 | return _install(archive, _build_install_args(options)) 424 | 425 | if __name__ == '__main__': 426 | sys.exit(main()) 427 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | # build a py2.py3-none-any wheel 3 | universal=1 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import ez_setup 2 | ez_setup.use_setuptools() # in case users don't have setuptools installed 3 | 4 | from setuptools import setup 5 | 6 | import NYTimesArticleAPI 7 | 8 | setup( 9 | name="NYTimesArticleAPI", 10 | description="Python wrapper for the New York Times Article Search API", 11 | version=NYTimesArticleAPI.__version__, 12 | author=NYTimesArticleAPI.__author__, 13 | author_email="mattdmo@pigimal.com", 14 | url="https://github.com/MattDMo/NYTimesArticleAPI", 15 | license="MIT", 16 | long_description=open("README.rst").read(), 17 | classifiers=[ 18 | "Development Status :: 4 - Beta", 19 | "Intended Audience :: Developers", 20 | "License :: OSI Approved :: MIT License", 21 | "Operating System :: OS Independent", 22 | "Programming Language :: Python :: 2", 23 | "Programming Language :: Python :: 2.7", 24 | "Programming Language :: Python :: 3", 25 | "Programming Language :: Python :: 3.2", 26 | "Programming Language :: Python :: 3.3", 27 | "Programming Language :: Python :: 3.4", 28 | "Programming Language :: Python :: 3.5", 29 | "Programming Language :: Python :: 3.6", 30 | "Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary", 31 | "Topic :: Office/Business :: News/Diary", 32 | "Topic :: Software Development :: Libraries :: Python Modules", 33 | ], 34 | install_requires=[ 35 | "requests >= 2.7.0", 36 | ], 37 | packages=[ 38 | "NYTimesArticleAPI", 39 | ], 40 | keywords="nytimes new york times api article search nyt journalism news", 41 | test_suite="NYTimesArticleAPI.tests.NYTimesArticleAPITests", 42 | zip_safe=True, 43 | ) 44 | --------------------------------------------------------------------------------