├── .cvsignore ├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .hgignore ├── CHANGES.txt ├── LICENSE.txt ├── MailingLogger.py ├── Makefile ├── README.rst ├── TODO.txt ├── __init__.py ├── admin.py ├── authadapters.py ├── cgi-server.py ├── config.ini.template ├── config.py ├── constants.py ├── disposable_email_blacklist.conf ├── doc ├── discriminators.txt ├── discussion.anthony.20021024.txt ├── dist.py.patch ├── dist.tex.patch ├── download.ideas.txt ├── feedback.anthony.20021024.txt ├── fm.txt ├── freshmeat.trove.discriminators.txt ├── freshmeat.trove.txt ├── metadata-1.2-pep.txt ├── mirroring.txt ├── naming.txt ├── operational_manual.txt ├── pep-0301.txt ├── pep.editor.feedback.20021102.txt ├── rest.doc ├── sourceforge.trove.discriminators.txt └── sourceforge.trove.txt ├── dogadapter.py ├── dumper.py ├── favicon.ico ├── find_dups.py ├── fncache.py ├── gae.py ├── legacy_passwords.py ├── mini_pkg_resources.py ├── mirrors.txt ├── mod_python.conf ├── pkgbase_schema.sql ├── pypi.cgi ├── pypi.fcgi ├── pypi.py ├── pypi.wsgi ├── register.py ├── requirements.in ├── requirements.txt ├── rpc.py ├── setup.py ├── standalone.py ├── static ├── css │ ├── docutils.css │ ├── pygments.css │ ├── pypi-screen.css │ └── pypi.css ├── error │ ├── 400.html │ ├── 429.html │ └── 500.html ├── images │ ├── PythonPoweredAnimSmall.gif │ ├── blank.gif │ ├── bullet.gif │ ├── button-on-bg.png │ ├── header-bg2.png │ ├── nav-off-bg.png │ ├── nav-on-bg.png │ ├── python-logo.png │ ├── testing-site-right.png │ ├── testing-site.png │ └── trans.gif └── styles │ ├── defaultfonts.css │ ├── largestyles.css │ ├── netscape4.css │ ├── print.css │ ├── screen-switcher-default.css │ └── styles.css ├── store.py ├── tasks.py ├── templates ├── about.pt ├── browse.pt ├── confirm.pt ├── dialog.pt ├── display.pt ├── edit_form.pt ├── files.pt ├── home.pt ├── index.pt ├── login.pt ├── message.pt ├── mirrors.pt ├── openid.pt ├── packages-rss.xml ├── password_reset.pt ├── password_reset_change.pt ├── pkg_edit.pt ├── print_the_world.pt ├── register.pt ├── register_gone.pt ├── role_form.pt ├── rss.xml ├── rss1hour.xml ├── security.pt ├── standard_template.pt └── tos.pt ├── tests └── rpc-test.py ├── thfcgi.py ├── tools ├── apache_count.py ├── apache_count_dist.py ├── apache_reader.py ├── apache_stats.py ├── daily.py ├── daily.sql ├── demodata.py ├── downloadstats ├── dumpstats ├── duplicate_users.py ├── email_renamed_users.py ├── index-trove.py ├── index.py ├── integrate-redis-stats.py ├── integratestats ├── keygen ├── migrate.py ├── mirrorlib.py ├── mksqlite.py ├── pw_reset_mailout.py ├── rsyslog-cdn.py ├── sanitize_package_names.py ├── sql-migrate-20050320-2.sql ├── sql-migrate-20050320.sql ├── sql-migrate-20050322.sql ├── sql-migrate-20050402.sql ├── sql-migrate-20070409.sql ├── sql-migrate-20070706.sql ├── sql-migrate-20070709.sql ├── sql-migrate-20070713.sql ├── sql-migrate-20070721.sql ├── sql-migrate-20080512.sql ├── sql-migrate-20081007.sql ├── sql-migrate-20090124.sql ├── sql-migrate-20090208.sql ├── sql-migrate-20090327.sql ├── sql-migrate-20090329.sql ├── sql-migrate-20090331.sql ├── sql-migrate-20090829.sql ├── sql-migrate-20090914.sql ├── sql-migrate-20091015.sql ├── sql-migrate-20091125.sql ├── sql-migrate-20091204.sql ├── sql-migrate-20091228.sql ├── sql-migrate-20100102.sql ├── sql-migrate-20100103.sql ├── sql-migrate-20100313.sql ├── sql-migrate-20100724.sql ├── sql-migrate-20100814.sql ├── sql-migrate-20110129.sql ├── sql-migrate-20110220.sql ├── sql-migrate-20110822.sql ├── sql-migrate-20110823-2.sql ├── sql-migrate-20110823.sql ├── sql-migrate-20110831.sql ├── sql-migrate-20110905.sql ├── sql-migrate-20111119.sql ├── sql-migrate-20111120.sql ├── sql-migrate-20111130.sql ├── sql-migrate-20120312.sql ├── sql-migrate-20120314.sql ├── sql-migrate-20120325.sql ├── sql-migrate-20130114.sql ├── sql-migrate-20130319-2.sql ├── sql-migrate-20130319.sql ├── sql-migrate-20130523.sql ├── sql-migrate-20130917-2.sql ├── sql-migrate-20130917.sql ├── sql-migrate-20140702.sql ├── sqlite_create.py ├── tests │ ├── __init__.py │ ├── pypi.access.log.1.bz2 │ ├── pypi.cfg │ └── test_apache_reader.py ├── touch_all_files.py ├── upgradepw.py ├── utf8convert.py ├── verify.py └── worker.py ├── trove.py └── webui.py /.cvsignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Before opening an issue for PyPI, please note the following: 2 | - First, probably do not! This codebase has been retired. 3 | - This repository does not service pypi.org or pypi.python.org any longer. 4 | - [Warehouse](https://github.com/pypa/warehouse) has replaced this codebase. 5 | - Any issues likely belong at [Warehouse](https://github.com/pypa/warehouse/issues) 6 | - We do not maintain individual packages installed from PyPI, please seek out the maintainer 7 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Before opening a Pull-Request for PyPI, please note the following: 2 | - First, probably do not! This codebase has been retired. 3 | - This repository does not service pypi.org or pypi.python.org any longer. 4 | - [Warehouse](https://github.com/pypa/warehouse) has replaced this codebase. 5 | - Any Pull Requests likely belong at [Warehouse](https://github.com/pypa/warehouse/pulls) 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | *.swp 3 | lib 4 | bin 5 | local 6 | include 7 | **.pyc 8 | config.ini 9 | uwsgi-logd.conf 10 | sshkeys_update 11 | privkey* 12 | pubkey* 13 | pypi.egg* 14 | .idea 15 | -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | 2005-12-14 2 | - added more sanity checks on input (empty description, name, version) 3 | - check supplied user email addresses for '@' and '.' 4 | - more robustness around various places 5 | - don't show package Journal to non-admin/owner/maintainer 6 | 7 | 8 | 2005-05-21 9 | - added database password option 10 | 11 | 2004-03-19 12 | - migrated from sqlite to postgres 13 | - upload of files for releases, and removal of files 14 | 15 | 16 | 2004-07-14 17 | - fixed editing of package information in package summary page (bug 989597) 18 | 19 | 20 | 2004-06-23 21 | - fixed the display of latest release (bug 977432) 22 | 23 | 24 | 2004-06-21 25 | - when displaying a package and no version is given, try to determine 26 | the latest release (bug #958515) 27 | - better error handling in display action 28 | - fixed page titles for search_form and forgotten_password_form 29 | - fixed typo in store.py which prevented indexes being created 30 | 31 | 2003-04-01 32 | - PKG-INFO download now has "obfuscated" email addresses 33 | 34 | 35 | 2003-03-02 36 | - fixed deletion of packages where there were no versions (bugs #907317 and 37 | #908118) 38 | - list only new releases in RSS and front page, not any old edit (bug 39 | #907315) 40 | 41 | 42 | 2004-03-01 43 | - add meaningful titles to pages (uses page heading) 44 | - properly unquote version numbers for release editing page (bug #855883) 45 | - allow removal of more than one release at a time 46 | - make "delete whole package" a form button 47 | - made wording of role admin link more helpful 48 | - hide all current releases when a new release is added 49 | 50 | 51 | 2004-06-20 52 | - when displaying a package and no version is given, try to determine 53 | the latest release (bug #958515) 54 | - better error handling in display action 55 | - fixed page titles for search_form and forgotten_password_form 56 | - fixed typo in store.py which prevented indexes being created 57 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Python Package Index 2 | ~~~~~~~~~~~~~~~~~~~~ 3 | 4 | Copyright (c) 2009 Python Software Foundation 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 3. The name of the author may not be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR 18 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 20 | NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 22 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 23 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | -------------------------------------------------------------------------------- /MailingLogger.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2004-2005 Simplistix Ltd 2 | # Copyright (c) 2001-2003 New Information Paradigms Ltd 3 | # 4 | # This Software is released under the MIT License: 5 | # http://www.opensource.org/licenses/mit-license.html 6 | # See license.txt for more details. 7 | 8 | import datetime 9 | import os 10 | import smtplib 11 | import socket 12 | 13 | from email.MIMEText import MIMEText 14 | from logging.handlers import SMTPHandler 15 | from logging import Formatter, LogRecord, CRITICAL 16 | 17 | now = datetime.datetime.now 18 | 19 | class SubjectFormatter(Formatter): 20 | 21 | def format(self,record): 22 | record.message = record.getMessage() 23 | if self._fmt.find('%(line)') >= 0: 24 | record.line = record.message.split('\n')[0] 25 | if self._fmt.find("%(asctime)") >= 0: 26 | record.asctime = self.formatTime(record, self.datefmt) 27 | if self._fmt.find("%(hostname)") >= 0: 28 | record.hostname = socket.gethostname() 29 | return self._fmt % record.__dict__ 30 | 31 | class MailingLogger(SMTPHandler): 32 | 33 | def __init__(self, mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None, send_empty_entries=False, flood_level=None): 34 | SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject, credentials=credentials, secure=secure) 35 | self.subject_formatter = SubjectFormatter(subject) 36 | self.send_empty_entries = send_empty_entries 37 | self.flood_level = flood_level 38 | self.hour = now().hour 39 | self.sent = 0 40 | 41 | def getSubject(self,record): 42 | return self.subject_formatter.format(record) 43 | 44 | def emit(self,record): 45 | current_time = now() 46 | current_hour = current_time.hour 47 | if current_hour > self.hour: 48 | self.hour = current_hour 49 | self.sent = 0 50 | if self.sent == self.flood_level: 51 | # send critical error 52 | record = LogRecord( 53 | name = 'flood', 54 | level = CRITICAL, 55 | pathname = '', 56 | lineno = 0, 57 | msg = """Too Many Log Entries 58 | 59 | More than %s entries have been logged that would have resulted in 60 | emails being sent. 61 | 62 | No further emails will be sent for log entries generated between 63 | %s and %i:00:00 64 | 65 | Please consult any other configured logs, such as a File Logger, 66 | that may contain important entries that have not been emailed. 67 | """ % (self.sent,current_time.strftime('%H:%M:%S'),current_hour+1), 68 | args = (), 69 | exc_info = None) 70 | if not self.send_empty_entries and not record.msg.strip(): 71 | return 72 | elif self.sent > self.flood_level: 73 | # do nothing, we've sent too many emails already 74 | return 75 | self.sent += 1 76 | 77 | # actually send the mail 78 | try: 79 | import smtplib 80 | port = self.mailport 81 | if not port: 82 | port = smtplib.SMTP_PORT 83 | smtp = smtplib.SMTP(self.mailhost, port) 84 | msg = self.format(record) 85 | email = MIMEText(msg) 86 | email['Subject']=self.getSubject(record) 87 | email['From']=self.fromaddr 88 | email['To']=', '.join(self.toaddrs) 89 | email['X-Mailer']='MailingLogger' 90 | if self.username: 91 | if self.secure is not None: 92 | smtp.starttls(*self.secure) 93 | smtp.login(self.username, self.password) 94 | smtp.sendmail(self.fromaddr, self.toaddrs, email.as_string()) 95 | smtp.quit() 96 | except: 97 | self.handleError(record) 98 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | serve: 3 | if [ ! -e cgi-bin ] ; then ln -s . cgi-bin ; fi 4 | python cgi-server.py & 5 | -------------------------------------------------------------------------------- /TODO.txt: -------------------------------------------------------------------------------- 1 | 2013 Sprints 2 | 3 | - the distutils upload command omits some meta-data (long_description) 4 | - the upload handler doesn't allow registration of a new package 5 | - distutils doesn't allow specifying https, nor does it default to https 6 | - OAuth PEM to users over HTTPS 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | Now: 15 | 16 | - patch python 2.5 to use the name "Cheese Shop" and the new URL 17 | - PEP for metadata 1.2 18 | http://www.python.org/peps/pep-0345.html 19 | - support .egg format uploads 20 | https://sourceforge.net/tracker/?func=detail&atid=513503&aid=1229137&group_id=66150 21 | - incorporate Ian Bicking's XML-RPC API 22 | http://mail.python.org/pipermail/catalog-sig/2005-May/000634.html 23 | - auto-generate download_urls for package uploads 24 | 25 | 26 | Sooner: 27 | 28 | - command-line tool to query pypi and fetch entries 29 | - table structure and query support for checking dependencies 30 | - table structure has columns for identification of the release 31 | - add a specification ID to release_*? 32 | - or just use release, name 33 | - table also has a column for each of <, <=, ==, >=, > so queries are 34 | easier to form 35 | 36 | Later: 37 | 38 | - change notification emails 39 | - "latest changes" 40 | - admin interface for user details editing (and fix theller@python.net) 41 | - "does this release supercede the existing release?" 42 | 43 | 44 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/__init__.py -------------------------------------------------------------------------------- /authadapters.py: -------------------------------------------------------------------------------- 1 | from authomatic.adapters import BaseAdapter 2 | import urlparse 3 | import Cookie 4 | from wsgiref.util import request_uri 5 | 6 | 7 | class PyPIAdapter(BaseAdapter): 8 | 9 | 10 | def __init__(self, env, config, handler, form, return_url=None): 11 | self.env = env 12 | self.config = config 13 | self.handler = handler 14 | self.form = form 15 | self.return_url = return_url 16 | 17 | @property 18 | def params(self): 19 | return dict(self.form) 20 | 21 | @property 22 | def url(self): 23 | return request_uri(self.env, include_query=False) 24 | 25 | @property 26 | def cookies(self): 27 | return dict([(k, v.value) for k, v in Cookie.SimpleCookie(self.env.get('HTTP_COOKIE', '')).items()]) 28 | 29 | def write(self, value): 30 | self.handler.set_status('200 OK') 31 | self.handler.send_header("Content-type", 'text/html') 32 | self.handler.send_header("Content-length", str(len(value))) 33 | self.handler.end_headers() 34 | self.handler.wfile.write(value) 35 | 36 | def set_header(self, key, value): 37 | self.handler.send_header(key, value) 38 | 39 | def set_status(self, status): 40 | self.handler.set_status(status) 41 | -------------------------------------------------------------------------------- /cgi-server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | 4 | from CGIHTTPServer import CGIHTTPRequestHandler 5 | import BaseHTTPServer, SimpleHTTPServer 6 | 7 | def main (): 8 | SimpleHTTPServer.test(CGIHTTPRequestHandler, BaseHTTPServer.HTTPServer) 9 | 10 | if __name__ == '__main__': 11 | main() -------------------------------------------------------------------------------- /config.ini.template: -------------------------------------------------------------------------------- 1 | [database] 2 | 3 | ;Postgres Database 4 | host = hostname 5 | port = 5432 6 | name = pypi 7 | ;user = pypi 8 | 9 | ; Redis 10 | queue_redis_url = redis://localhost:6379/0 11 | count_redis_url = redis://localhost:6379/1 12 | cache_redis_url = redis://localhost:6379/2 13 | block_redis_url = redis://localhost:6379/3 14 | xmlrpc_redis_url = redis://localhost:6379/4 15 | 16 | ; ElasticSearch 17 | ; releases_index_url = http://127.0.0.1:9200 18 | ; releases_index_name = pypi-releases 19 | 20 | ; AWS 21 | ; aws_access_key_id = 22 | ; aws_secret_access_key = 23 | 24 | ; Storage Directories 25 | ;files_bucket = 26 | files_dir = /MacDev/svn.python.org/pypi-pep345/files 27 | docs_dir = /MacDev/svn.python.org/pypi-pep345/docs 28 | 29 | ; Third-Party 30 | pubsubhubbub = http://pubsubhubbub.appspot.com/ 31 | 32 | [webui] 33 | 34 | ; PyPI config 35 | debug_mode = yes 36 | rss_file = /tmp/pypi_rss.xml 37 | packages_rss_file = /tmp/pypi_packages_rss.xml 38 | 39 | ; Email 40 | adminemail = richard@python.org 41 | replyto = richard@python.org 42 | 43 | ; Secrets 44 | key_dir = . 45 | cheesecake_password = secret 46 | ; this is the secret used to sign password reset efforts - keep it secret! 47 | ; ''.join(random.choice(string.letters + string.digits) for n in range(64)) 48 | ;reset_secret = secret 49 | 50 | ; URI Paths 51 | simple_script = /simple 52 | raw_package_prefix = /raw-packages 53 | simple_sign_script = /serversig 54 | 55 | ; URLs 56 | url = http://localhost:8000/pypi 57 | files_url = http://localhost/pypi_files 58 | pydotorg = http://www.python.org/ 59 | package_docs_url = http://pythonhosted.org/ 60 | 61 | ; Statuspage.io 62 | statuspage_id = 928bjjg42vzc 63 | 64 | [smtp] 65 | hostname = localhost:25 66 | starttls = off 67 | auth = off 68 | ;login = postmaster@localhost 69 | ;password = muchsecret 70 | 71 | [passlib] 72 | ; The first listed schemed will automatically be the default, see passlib 73 | ; documentation for a full list of options. Schemes not listed here will 74 | ; not be recognised by passlib. 75 | ; bcrypt is our current ideal 76 | ; hex_sha1 is the legacy unsalted 77 | schemes = bcrypt, hex_sha1 78 | ; Old password schemes will be automatically migrated by passlib - the default 79 | ; for PyPI is "auto" - setting it to blank will disable auto migration. See 80 | ; passlib documentation for more info. 81 | ; deprecated = 82 | 83 | [logging] 84 | file = 85 | mail_logger = off 86 | fromaddr = 87 | toaddrs = 88 | 89 | ; Not seeing this used in production 90 | ;[mirrors] 91 | ;folder = mirrors 92 | ;local-stats = local-stats 93 | ;global-stats = global-stats 94 | 95 | [sentry] 96 | dsn = 97 | 98 | [uwsgi] 99 | ;uid=pypi 100 | ;gid=pypi 101 | wsgi-file = pypi.wsgi 102 | socket = /tmp/pypi.sock 103 | ;pidfile = /var/run/devpypi/pypi.pid 104 | ;daemonize = 127.0.0.1:8224 105 | ;processes = 2 106 | harakiri = 60 107 | ;reload-on-as = 400 108 | ;max-requests = 10000 109 | master = 1 110 | post-buffering = 8192 111 | chmod-socket = 666 112 | ;disable-logging = true 113 | ;log-5xx = true 114 | 115 | ; CDN API 116 | [fastly] 117 | api_domain = https://api.fastly.com/ 118 | api_key = 119 | service_id = 120 | 121 | ; IP/User Blocking 122 | [blocking] 123 | blocked_timeout = 600 ; in seconds 124 | blocked_attempts_ip = 10 125 | blocked_attempts_user = 1000 126 | 127 | ; XMLRPC Settings 128 | [xmlrpc] 129 | max_concurrent = 10 130 | enforce = false 131 | request_log_file = /tmp/rpclog 132 | 133 | ; Authomatic 134 | [authomatic] 135 | secure = true 136 | secret = deadbeefdeadbeefdeadbeefdeadbeef 137 | 138 | ; Google OAuth 2 139 | [google] 140 | client_id = 141 | client_secret = 142 | -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | DOMAIN_BLACKLIST = [] 5 | 6 | DOMAIN_BLACKLIST_CONF = os.path.join(os.path.dirname(os.path.realpath(__file__)), 7 | 'disposable_email_blacklist.conf') 8 | try: 9 | with open(DOMAIN_BLACKLIST_CONF, 'rU') as f: 10 | DOMAIN_BLACKLIST = [line.rstrip() for line in f.readlines()] 11 | except Exception as e: 12 | pass 13 | -------------------------------------------------------------------------------- /doc/discussion.anthony.20021024.txt: -------------------------------------------------------------------------------- 1 | > the user stuff was kinda thrown in on the train this morning. had a look 2 | > at PAUSE when I got in: mostly it's about registering users and they then 3 | > own namespaces. I 4 | 'm not sure how the owning 5 | > stuff will work - trove talks about it a bit. trove is also pretty thin on 6 | > the ground in places ;) 7 | 8 | **** A message has arrived from pluto on Thu Oct 24 17:19! **** 9 | From: anthony [To: richard] 10 | for trove, I'd suggest something like a trove browser (but better 11 | than the SF one, pleeeease) that allows you to extract the data for 12 | insertion into your setup.py. 13 | Or else, categorise it via the web, and if you post a new setup.py 14 | register with no discriminators, tell it to leave the current one alone. 15 | 16 | 17 | **** A message has arrived from pluto on Thu Oct 24 17:19! **** 18 | From: anthony [To: richard] 19 | you might also want some sort of two-stage "the following things will 20 | be changed, do you want to proceed" thing. 21 | -------------------------------------------------------------------------------- /doc/dist.py.patch: -------------------------------------------------------------------------------- 1 | *** dist.py Wed Dec 4 15:41:53 2002 2 | --- dist.py-new Wed Dec 4 15:41:32 2002 3 | *************** 4 | *** 93,98 **** 5 | --- 93,100 ---- 6 | "print the long package description"), 7 | ('platforms', None, 8 | "print the list of platforms"), 9 | + ('classifiers', None, 10 | + "print the list of classifiers"), 11 | ('keywords', None, 12 | "print the list of keywords"), 13 | ] 14 | *************** 15 | *** 634,639 **** 16 | --- 636,643 ---- 17 | value = getattr(self.metadata, "get_"+opt)() 18 | if opt in ['keywords', 'platforms']: 19 | print string.join(value, ',') 20 | + elif opt == 'classifiers': 21 | + print string.join(value, '\n') 22 | else: 23 | print value 24 | any_display_options = 1 25 | *************** 26 | *** 962,968 **** 27 | "maintainer", "maintainer_email", "url", 28 | "license", "description", "long_description", 29 | "keywords", "platforms", "fullname", "contact", 30 | ! "contact_email", "licence") 31 | 32 | def __init__ (self): 33 | self.name = None 34 | --- 966,972 ---- 35 | "maintainer", "maintainer_email", "url", 36 | "license", "description", "long_description", 37 | "keywords", "platforms", "fullname", "contact", 38 | ! "contact_email", "licence", "classifiers") 39 | 40 | def __init__ (self): 41 | self.name = None 42 | *************** 43 | *** 977,982 **** 44 | --- 981,987 ---- 45 | self.long_description = None 46 | self.keywords = None 47 | self.platforms = None 48 | + self.classifiers = None 49 | 50 | def write_pkg_info (self, base_dir): 51 | """Write the PKG-INFO file into the release tree. 52 | *************** 53 | *** 1003,1008 **** 54 | --- 1008,1016 ---- 55 | for platform in self.get_platforms(): 56 | pkg_info.write('Platform: %s\n' % platform ) 57 | 58 | + for classifier in self.get_classifiers(): 59 | + pkg_info.write('Classifier: %s\n' % classifier ) 60 | + 61 | pkg_info.close() 62 | 63 | # write_pkg_info () 64 | *************** 65 | *** 1059,1064 **** 66 | --- 1067,1075 ---- 67 | def get_platforms(self): 68 | return self.platforms or ["UNKNOWN"] 69 | 70 | + def get_classifiers(self): 71 | + return self.classifiers or [] 72 | + 73 | # class DistributionMetadata 74 | 75 | 76 | -------------------------------------------------------------------------------- /doc/dist.tex.patch: -------------------------------------------------------------------------------- 1 | --- dist.tex Thu Dec 5 07:29:20 2002 2 | +++ dist.tex-new Tue Dec 24 14:15:21 2002 3 | @@ -282,7 +282,8 @@ 4 | rather than by module. This is important since the Distutils consist of 5 | a couple of dozen modules split into (so far) two packages; an explicit 6 | list of every module would be tedious to generate and difficult to 7 | -maintain. 8 | +maintain. For more information on the additional meta-data, see 9 | +section~\ref{meta-data}. 10 | 11 | Note that any pathnames (files or directories) supplied in the setup 12 | script should be written using the \UNIX{} convention, i.e. 13 | @@ -680,6 +681,75 @@ 14 | To install data files directly in the target directory, an empty 15 | string should be given as the directory. 16 | 17 | + 18 | +\subsection{Additional meta-data} 19 | +\label{meta-data} 20 | + 21 | +The setup script may include additional meta-data beyond the name and 22 | +version. This information includes: 23 | + 24 | +\begin{tableiii}{l|l|c}{code}% 25 | + {Meta-Data}{Description}{Notes} 26 | + \lineiii{name}{the name of the package}{(1)} 27 | + \lineiii{version}{the version of this release}{(1)} 28 | + \lineiii{author}{package author's name}{(2)} 29 | + \lineiii{author_email}{email address of the package author}{(2)} 30 | + \lineiii{maintainer}{package maintainer's name}{(2)} 31 | + \lineiii{maintainer_email}{email address of the package maintainer}{(2)} 32 | + \lineiii{home_page}{a URL}{(1)} 33 | + \lineiii{license}{the terms the package is released under}{} 34 | + \lineiii{description}{a short, summary description of the package}{} 35 | + \lineiii{long_description}{a longer description of the package}{} 36 | + \lineiii{keywords}{some keywords appropriate to the package}{} 37 | + \lineiii{platform}{a list of the target platforms}{} 38 | + \lineiii{classifiers}{a list of Trove classifiers}{(2)} 39 | +\end{tableiii} 40 | + 41 | +\noindent Notes: 42 | +\begin{description} 43 | +\item[(1)] these fields are required 44 | +\item[(2)] either the author or the maintainer must be nominated 45 | +\item[(3)] should not be used if your package is to be compatible with 46 | + Python versions prior to 2.2.3 or 2.3. The list is available from the 47 | + PyPI website. 48 | +\end{description} 49 | + 50 | +\option{classifiers} are specified in a python list: 51 | + 52 | +\begin{verbatim} 53 | +setup(... 54 | + classifiers = [ 55 | + 'Development Status :: 4 - Beta', 56 | + 'Environment :: Console', 57 | + 'Environment :: Web Environment', 58 | + 'Intended Audience :: End Users/Desktop', 59 | + 'Intended Audience :: Developers', 60 | + 'Intended Audience :: System Administrators', 61 | + 'License :: OSI Approved :: Python Software Foundation License', 62 | + 'Operating System :: MacOS :: MacOS X', 63 | + 'Operating System :: Microsoft :: Windows', 64 | + 'Operating System :: POSIX', 65 | + 'Programming Language :: Python', 66 | + 'Topic :: Communications :: Email', 67 | + 'Topic :: Office/Business', 68 | + 'Topic :: Software Development :: Bug Tracking', 69 | + ], 70 | + ... 71 | +) 72 | +\end{verbatim} 73 | + 74 | +If you wish to include classifiers in your \file{setup.py} file and also 75 | +wish to remain backwards-compatible with Python releases prior to 2.2.3, 76 | +then you can include the following code fragment in your \file{setup.py} 77 | +before the \code{setup()} call. 78 | + 79 | +\begin{verbatim} 80 | +# patch distutils if it can't cope with the "classifiers" keyword 81 | +if sys.version < '2.2.3': 82 | + from distutils.dist import DistributionMetadata 83 | + DistributionMetadata.classifiers = None 84 | +\end{verbatim} 85 | + 86 | 87 | \section{Writing the Setup Configuration File} 88 | \label{setup-config} 89 | @@ -1314,6 +1384,57 @@ 90 | The installer file will be written to the ``distribution directory'' 91 | --- normally \file{dist/}, but customizable with the 92 | \longprogramopt{dist-dir} option. 93 | + 94 | +\section{Registering with the Package Index} 95 | +\label{package-index} 96 | + 97 | +The Python Package Index (PyPI) holds meta-data describing distributions 98 | +packaged with distutils. The distutils command \command{register} is 99 | +used to submit your distribution's meta-data to the index. It is invoked 100 | +as follows: 101 | + 102 | +\begin{verbatim} 103 | +python setup.py register 104 | +\end{verbatim} 105 | + 106 | +Distutils will respond with the following prompt: 107 | + 108 | +\begin{verbatim} 109 | +running register 110 | +We need to know who you are, so please choose either: 111 | + 1. use your existing login, 112 | + 2. register as a new user, 113 | + 3. have the server generate a new password for you (and email it to you), or 114 | + 4. quit 115 | +Your selection [default 1]: 116 | +\end{verbatim} 117 | + 118 | +\noindent Note: if your username and password are saved locally, you will 119 | +not see this menu. 120 | + 121 | +If you have not registered with PyPI, then you will need to do so now. You 122 | +should choose option 2, and enter your details as required. Soon after 123 | +submitting your details, you will receive an email which will be used to 124 | +confirm your registration. 125 | + 126 | +Once you are registered, you may choose option 1 from the menu. You will 127 | +be prompted for your PyPI username and password, and \command{register} 128 | +will then submit your meta-data to the index. 129 | + 130 | +You may submit any number of versions of your distribution to the index. If 131 | +you alter the meta-data for a particular version, you may submit it again 132 | +and the index will be updated. 133 | + 134 | +PyPI holds a record for each (name, version) combination submitted. The 135 | +first user to submit information for a given name is designated the Owner 136 | +of that name. They may submit changes through the \command{register} 137 | +command or through the web interface. They may also designate other users 138 | +as Owners or Maintainers. Maintainers may edit the package information, but 139 | +not designate other Owners or Maintainers. 140 | + 141 | +By default PyPI will list all versions of a given package. To hide certain 142 | +versions, the Hidden property should be set to yes. This must be edited 143 | +through the web interface. 144 | 145 | 146 | \section{Examples} 147 | -------------------------------------------------------------------------------- /doc/download.ideas.txt: -------------------------------------------------------------------------------- 1 | I've included a public_key in the user schema in anticipation of handling 2 | signing. I've also got a download_url in the package schema - though a 3 | friend has recommended that I have several. I'm not so sure. I _think_ I'd 4 | rather have: 5 | 6 | - a definition of download formats for each package (filename, format) 7 | where 8 | format is rpm, deb, source gz, source zip, ... 9 | - a definition of download mirrors which give (mirror name, base URL) 10 | 11 | and then each mirror can support the same paths starting from their base 12 | URL. 13 | 14 | None of this is intended for the initial release though. 15 | 16 | 17 | -------------------------------------------------------------------------------- /doc/feedback.anthony.20021024.txt: -------------------------------------------------------------------------------- 1 | From anthony@interlink.com.au Thu Oct 24 17:08:22 2002 2 | X-Sieve: cmu-sieve 2.0 3 | Return-Path: 4 | Received: (from uucp@localhost) 5 | by crown.off.ekorp.com (8.9.3/8.9.3) id HAA08779 6 | for rjones@ekit-inc.com; Thu, 24 Oct 2002 07:10:04 GMT 7 | Received: from cirrus.netspace.net.au(203.10.110.75) 8 | via SMTP by mx3.ekorp.com, id smtpdAAAYMa4fr; Thu Oct 24 07:09:59 2002 9 | Received: from localhost.localdomain (dsl-203-113-236-58.VIC.netspace.net.au [203.113.236.58]) 10 | by cirrus.netspace.net.au (8.11.3/8.11.3) with ESMTP id g9O79w786906 11 | for ; Thu, 24 Oct 2002 17:09:58 +1000 (EST) 12 | Received: from arbhome.com.au (anthony@localhost) 13 | by localhost.localdomain (8.11.6/8.11.6) with ESMTP id g9O78MG02368 14 | for ; Thu, 24 Oct 2002 17:08:22 +1000 15 | Message-Id: <200210240708.g9O78MG02368@localhost.localdomain> 16 | X-Authentication-Warning: localhost.localdomain: anthony owned process doing -bs 17 | X-Mailer: exmh version 2.5 07/13/2001 with nmh-1.0.4 18 | X-Exmh-Isig-CompType: repl 19 | X-Exmh-Isig-Folder: inbox 20 | To: Richard Jones 21 | From: Anthony Baxter 22 | Reply-to: Anthony Baxter 23 | Subject: Re: PEP draft 24 | In-Reply-To: <200210241657.55412.rjones@ekit-inc.com> 25 | Mime-Version: 1.0 26 | Content-Type: text/plain 27 | Date: Thu, 24 Oct 2002 17:08:22 +1000 28 | Status: R 29 | X-Status: N 30 | X-KMail-EncryptionState: 31 | X-KMail-SignatureState: 32 | 33 | 34 | >>> Richard Jones wrote 35 | > PEP: XXX 36 | > Title: Distutils Enhancements 37 | > Version: $Revision$ 38 | > Last-Modified: $Date$ 39 | > Author: Richard Jones 40 | > Status: Draft 41 | > Type: Standards Track 42 | > Content-Type: text/x-rst 43 | > Created: 24-Oct-2002 44 | > Python-Version: 2.3 45 | > Post-History: 46 | > 47 | > 48 | > Abstract 49 | > ======== 50 | > 51 | > This PEP proposes several extensions to the distutils packaging 52 | > system [1]_. These enhancements include a central package index, tools 53 | > for submitting package information to the index and extensions to the 54 | > package metadata to include Trove [2]_ information. 55 | > 56 | Include Vaults of Parnassus in the "prior art" investigation. 57 | 58 | > This PEP does not address either issues of package dependency, nor 59 | > central storage of packages. 60 | centralised 61 | 62 | > existing (name, version) will result in an *update* operation. 63 | > 64 | > The web interface implement the following commands: 65 | implements 66 | > 67 | > the index view. The index will include a customisation form at the 68 | > bottom a-la Roundup. The results will be paginated, sorted 69 | Give a reference (web site) for this. 70 | 71 | > **user** 72 | > Registers a new user with the index. Requires username, password and 73 | > email address. Passwords will be stored as SHA hashes. If the 74 | > username exists: 75 | 76 | Should require a confirmation email, as with mailman &c. 77 | 78 | > Notification of changes to a package entry will be sent to all users 79 | > who have created submitted information about the package. 80 | 81 | What does this mean? 82 | 83 | > discriminators = [ 84 | > 'Development Status :: 4 - Beta', 85 | > 'Environment :: Console (Text Based)', 86 | > 'Environment :: Web Environment', 87 | > 'Intended Audience :: End Users/Desktop', 88 | > 'Intended Audience :: Developers', 89 | > 'Intended Audience :: System Administrators', 90 | > 'License :: OSI Approved :: Python License', 91 | > 'Operating System :: MacOS X', 92 | > 'Operating System :: Microsoft :: Windows', 93 | > 'Operating System :: POSIX', 94 | > 'Programming Language :: Python', 95 | > 'Topic :: Communications :: Email', 96 | > 'Topic :: Office/Business', 97 | > 'Topic :: Software Development :: Bug Tracking', 98 | 99 | Why strings like this, rather than proper structures? e.g. 100 | 101 | discriminators = { 102 | 'Development Status': '4 - Beta', 103 | 'Environment' : ('Console (Text Based)', 'Web Environment') 104 | 'Intended Audience' : ('End Users/Desktop', 'Developers', 'System Administrators'), 105 | ..... 106 | 107 | How do people know that their trove categorisation is correct (e.g. 108 | that it doesn't have speling mistaiks?) 109 | 110 | > The list of discriminator values on the module index has been snarfed 111 | > from Freshmeat, without their permission. 112 | 113 | Should check that this is ok with them. 114 | 115 | There should be some sort of simple "check categorisation" or "update 116 | categorisation" thing. 117 | 118 | Who can update a package entry? ACLs? How and what can be updated 119 | with 'register'? Only a new version? What about changing trove 120 | categorisations, homepages, or the like? What about clashes of 121 | names (I don't know if there are any out there, but it wouldn't 122 | suprise me). 123 | 124 | 125 | 126 | -------------------------------------------------------------------------------- /doc/freshmeat.trove.txt: -------------------------------------------------------------------------------- 1 | [fm #21421] (news-admins) [fmII/contact] Trove categories 2 | Date: Friday 1:12:51 am 3 | From: Patrick Lenz via RT 4 | To: rjones@ekit-inc.com 5 | 6 | rjones@ekit-inc.com wrote (Wed, Oct 23 2002 23:49:21): 7 | 8 | > I'd like to use your list of Trove categories (or discriminators in the 9 | > original Trove parlance). Do you have any objections to this? Are you the 10 | > original authors of the list, and if not, do you know who is? 11 | 12 | We originally started out with the trove category set SourceForge.net had. We 13 | both expanded that tree so we're not really in sync anymore. I don't have any 14 | objections wrt you re-using the list. 15 | -------------------------------------------------------------------------------- /doc/mirroring.txt: -------------------------------------------------------------------------------- 1 | ============== 2 | PyPI Mirroring 3 | ============== 4 | 5 | ..contents:: 6 | 7 | When mirrors are used by the PyPI server ? 8 | ========================================== 9 | 10 | The mirrors are used : 11 | 12 | - to collect the download stats once a day 13 | - to display the /mirrors page, that contains a list of mirrors 14 | 15 | How to configure the mirroring parameters 16 | ========================================= 17 | 18 | In config.ini, add a new section called `mirrors`, with three 19 | elements: 20 | 21 | - cache-root: the directory used by the script to store 22 | stats files that where downloaded from the mirrors 23 | - local-stats: the directory where the local stats files are 24 | created. 25 | - global-stats: the directory where the global stats are 26 | created. 27 | 28 | Example:: 29 | 30 | [mirrors] 31 | cache-root = /tmp/pypi/mirrors-cache 32 | local-stats = /tmp/pypi/mirroring/local-stats 33 | global-stats = /tmp/pypi/mirroringglobal-stats 34 | 35 | How to publish the stats files to the world 36 | =========================================== 37 | 38 | The two stats directories need to be added as browsable 39 | directories in Apache. 40 | 41 | Collecting stats 42 | ================ 43 | 44 | To run the stats collector, just run:: 45 | 46 | $ python tools/apache_count.py config.ini apachelog 47 | 48 | Adding a mirror 49 | =============== 50 | 51 | Command to add a mirror:: 52 | 53 | $ python admin.py addmirror http://example.com/ simple last-modified locat-stats stats mirrors 54 | 55 | 56 | This line will add a mirror that has those urls: 57 | 58 | - http://example.com/simple : index root 59 | - http://example.com/last-modified : url of the last modified date 60 | - http://example.com/local-stats : root url of the daily local stats 61 | - http://example.com/local-stats : root url of the global stats 62 | - http://example.com/local-stats : root url of the global stats 63 | 64 | 65 | -------------------------------------------------------------------------------- /doc/naming.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/doc/naming.txt -------------------------------------------------------------------------------- /doc/operational_manual.txt: -------------------------------------------------------------------------------- 1 | This document is intended for administrators of the python.org host, and 2 | contains information about operationally maintaining the PyPI system. 3 | 4 | 5 | Maintenance 6 | ----------- 7 | 8 | The cgi-bin script "pypi.cgi" has a section which should be enabled that 9 | informs the user that the cgi interface is down for maintenance. This is a 10 | simple case of changing the "if 0:" to "if 1:" above the "503" response 11 | line. 12 | 13 | 14 | Administrative tasks 15 | -------------------- 16 | 17 | The "admin.py" script will perform several admin tasks (constantly growing 18 | as the need arises): 19 | 20 | password 21 | rmpackage 22 | -------------------------------------------------------------------------------- /doc/pep.editor.feedback.20021102.txt: -------------------------------------------------------------------------------- 1 | From goodger@users.sourceforge.net Sat Nov 2 12:05:17 2002 2 | X-Sieve: cmu-sieve 2.0 3 | Return-Path: 4 | Received: (from uucp@localhost) 5 | by crown.off.ekorp.com (8.9.3/8.9.3) id BAA13375 6 | for rjones@ekit-inc.com; Sat, 2 Nov 2002 01:04:56 GMT 7 | Received: from fep04-mail.bloor.is.net.cable.rogers.com(66.185.86.74) 8 | via SMTP by mx3.ekorp.com, id smtpdAAAK2aGcA; Sat Nov 2 01:04:51 2002 9 | Received: from [65.48.132.88] by fep04-mail.bloor.is.net.cable.rogers.com 10 | (InterMail vM.5.01.05.06 201-253-122-126-106-20020509) with ESMTP 11 | id <20021102010412.IOBP4298.fep04-mail.bloor.is.net.cable.rogers.com@[65.48.132.88]>; 12 | Fri, 1 Nov 2002 20:04:12 -0500 13 | User-Agent: Microsoft-Outlook-Express-Macintosh-Edition/5.0.3 14 | Date: Fri, 01 Nov 2002 20:05:17 -0500 15 | Subject: Re: New PEP: Distutils Enhancements 16 | From: David Goodger 17 | To: Richard Jones , 18 | 19 | Message-ID: 20 | In-Reply-To: <200211011741.35931.rjones@ekit-inc.com> 21 | Mime-version: 1.0 22 | Content-type: text/plain; 23 | charset="US-ASCII" 24 | Content-transfer-encoding: 7bit 25 | X-Authentication-Info: Submitted using SMTP AUTH LOGIN at fep04-mail.bloor.is.net.cable.rogers.com from [65.48.132.88] using ID at Fri, 1 Nov 2002 20:04:12 -0500 26 | Status: R 27 | X-Status: N 28 | X-KMail-EncryptionState: 29 | X-KMail-SignatureState: 30 | 31 | (Surprise! You probably didn't know I was moonlighting as a deputy PEP 32 | editor. :) 33 | 34 | The PEP looks good, Richard. Some suggestions & questions: 35 | 36 | > Title: Distutils Enhancements 37 | 38 | Too generic IMO. How about "Package Index and Metadata for Distutils"? 39 | 40 | > The server should be hosted in the python.org domain, 41 | 42 | What server? No server mentioned before this. Instead of "The 43 | server", perhaps "A package metadata server"? 44 | 45 | > hopefully just one command-line command for more regular users. 46 | 47 | "hopefully just a one-line command for most users", perhaps? 48 | 49 | > The original PEP which proposed such a system 50 | 51 | "PEP xxx, the original PEP which proposed such a system" 52 | 53 | > The web interface implements the following commands/interfaces: 54 | 55 | For readability, I suggest inserting blank lines between definition 56 | list items. (They blob together now.) 57 | 58 | > A demonstration will be available at: 59 | > 60 | > http://www.amk.ca/cgi-bin/pypi.cgi 61 | 62 | Is it up now? I wasn't able to access it. When will it be available? 63 | Couldn't it be hosted on a permanent basis on sf.net (with **TEST** 64 | prominently displayed, of course)? The previous one at 65 | http://mechanicalcat.net:8081/ is not accessible either. 66 | 67 | > two of whom used the register command bo submit package information. 68 | ^^ 69 | 70 | "bo" should be "to". 71 | 72 | -- 73 | David Goodger Open-source projects: 74 | - Python Docutils: http://docutils.sourceforge.net/ 75 | (includes reStructuredText: http://docutils.sf.net/rst.html) 76 | - The Go Tools Project: http://gotools.sourceforge.net/ 77 | 78 | 79 | -------------------------------------------------------------------------------- /doc/sourceforge.trove.txt: -------------------------------------------------------------------------------- 1 | Re: Trove stuff from sf.net 2 | Date: Today 1:13:53 pm 3 | From: Patrick McGovern 4 | To: Richard Jones 5 | 6 | Thank you for the clarification. 7 | 8 | It's not clear to me if our trove categorization is Open Source or not. It's 9 | something I would have to ponder with our Legal folks here.... 10 | 11 | Regardless, for your application I'm granting you the right to use the Trove 12 | info. 13 | 14 | Pat- 15 | 16 | On Tue, Oct 29, 2002 at 01:02:46PM +1100, Richard Jones wrote: 17 | > Hiya, regarding alexandria-Support Requests-628367, I'd like to use the Trove 18 | > info in setting up a python modules index at python.org. The set from your 19 | > site will be made available to python programmers to classify their modules. 20 | > That's it in a nutshell. Because it's a one-off application, it's unlikely 21 | > that the source will be released - but if necessary it can. 22 | > 23 | > 24 | > Richard 25 | > 26 | 27 | -- 28 | Patrick McGovern 29 | Director, SourceForge.net pmcgovern@vasoftware.com 30 | -------------------------------------------------------------------------------- /dogadapter.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from datadog import initialize 4 | from datadog.dogstatsd import DogStatsd 5 | 6 | import config 7 | 8 | root = os.path.dirname(os.path.abspath(__file__)) 9 | conf = config.Config(os.path.join(root, "config.ini")) 10 | 11 | dogstatsd = DogStatsd(host='localhost', port=conf.datadog_dogstatsd_port, constant_tags=conf.datadog_tags) 12 | -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/favicon.ico -------------------------------------------------------------------------------- /find_dups.py: -------------------------------------------------------------------------------- 1 | import sys, os, urllib, StringIO, traceback, cgi, binascii, getopt 2 | import store, config 3 | 4 | try: 5 | config = config.Config(sys.argv[1]) 6 | except IndexError: 7 | print "Usage: find_dups.py config.ini" 8 | raise SystemExit 9 | 10 | store = store.Store(config) 11 | store.open() 12 | 13 | def owner_email(p): 14 | result = set() 15 | for r,u in store.get_package_roles(p): 16 | if r == 'Owner': 17 | result.add(store.get_user(u)['email']) 18 | return result 19 | 20 | def mail_dup(email, package1, package2): 21 | email = "martin@v.loewis.de" 22 | f = os.popen("/usr/lib/sendmail "+email, "w") 23 | f.write("To: %s\n" % email) 24 | f.write("From: martin@v.loewis.de\n") 25 | f.write("Subject: Please cleanup PyPI package names\n\n") 26 | f.write("Dear Package Owner,\n") 27 | f.write("You have currently registered the following to packages,\n") 28 | f.write("which differ only in case:\n\n%s\n%s\n\n" % (package1, package2)) 29 | f.write("As a recent policy change, we are now rejecting this kind of\n") 30 | f.write("setup. Please remove one of packages.\n\n") 31 | f.write("If you need assistance, please let me know.\n\n") 32 | f.write("Kind regards,\nMartin v. Loewis\n") 33 | f.close() 34 | 35 | def mail_distinct_dup(users, package1, package2): 36 | f = os.popen("/usr/lib/sendmail "+" ".join(users), "w") 37 | for email in users: 38 | f.write("To: %s\n" % email) 39 | f.write("From: martin@v.loewis.de\n") 40 | f.write("Subject: Please cleanup PyPI package names\n\n") 41 | f.write("Dear Package Owners,\n") 42 | f.write("You have currently registered the following to packages,\n") 43 | f.write("which differ only in case:\n\n%s\n%s\n\n" % (package1, package2)) 44 | f.write("As a recent policy change, we are now rejecting this kind of\n") 45 | f.write("setup. Please remove one of packages.\n\n") 46 | f.write("If you need assistance, please let me know.\n\n") 47 | f.write("Kind regards,\nMartin v. Loewis\n") 48 | f.close() 49 | 50 | def dup_packages(): 51 | lower = {} 52 | for name,version in store.get_packages(): 53 | lname = name.lower() 54 | if lname in lower: 55 | owner1 = owner_email(name) 56 | owner2 = owner_email(lower[lname]) 57 | owners = owner1.intersection(owner2) 58 | if owners: 59 | mail_dup(owners.pop(),name,lower[lname]) 60 | else: 61 | mail_distinct_dup(owner1.union(owner2),name,lower[lname]) 62 | lower[lname] = name 63 | 64 | def mail_unused_user(email, all, unused): 65 | #for user in unused: 66 | # if all-unused: 67 | # newuser = (all-unused).pop() 68 | # print "UPDATE journals SET submitted_by='%s' WHERE submitted_by='%s';" % (newuser,user) 69 | # print "DELETE FROM rego_otk WHERE name='%s';" % user 70 | # print "DELETE FROM users WHERE name='%s';" % user 71 | email = "martin@v.loewis.de" 72 | f = os.popen("/usr/lib/sendmail "+email, "w") 73 | f.write("To: %s\n" % email) 74 | f.write("From: martin@v.loewis.de\n") 75 | f.write("Subject: Unused PyPI account deleted\n\n") 76 | f.write("Dear PyPI user,\n") 77 | f.write("You have currently registered the following user names,\n") 78 | f.write("all for the email account %s: \n\n%s\n\n" % (email, ' '.join(all))) 79 | f.write("As this kind of setup causes problems, we would like to delete,\n") 80 | f.write("one of the accounts, or somehow merge it with the other.\n") 81 | f.write("Please let me know what kind of action I should take,\n") 82 | f.write("please respond before February 1st.\n\n") 83 | f.write("Kind regards,\nMartin v. Loewis\n") 84 | f.close() 85 | 86 | def dup_users(): 87 | by_email = {} 88 | for user in store.get_users(): 89 | email = user['email'] 90 | by_email[email] = by_email.get(email, []) + [user['name']] 91 | rest = [] 92 | print "BEGIN;" 93 | for email, users in by_email.items(): 94 | if len(users) == 1: continue 95 | users = set(users) 96 | unused = set() 97 | for user in users: 98 | if not store.user_packages(user): 99 | unused.add(user) 100 | if 1:#len(users-unused)<=1: 101 | mail_unused_user(email, users, unused) 102 | else: 103 | rest.append((email, users, unused)) 104 | print "COMMIT;" 105 | for r in rest: 106 | print r 107 | 108 | dup_users() 109 | -------------------------------------------------------------------------------- /fncache.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | modified from https://gist.githubusercontent.com/kwarrick/4247343/raw/ca961ac3429777350a0256e39944135987012834/fncache.py 4 | 5 | - Built an object rather than pure function 6 | - Modified to use redis pipelining 7 | - Modified to allow purging 8 | 9 | credit: Kevin Warrick 10 | 11 | """ 12 | 13 | import os 14 | import json 15 | import redis 16 | 17 | from functools import wraps 18 | 19 | from perfmetrics import statsd_client 20 | from perfmetrics import set_statsd_client 21 | 22 | from dogadapter import dogstatsd 23 | 24 | import config 25 | 26 | root = os.path.dirname(os.path.abspath(__file__)) 27 | conf = config.Config(os.path.join(root, "config.ini")) 28 | 29 | STATSD_URI = "statsd://127.0.0.1:8125?prefix=%s" % (conf.database_name) 30 | set_statsd_client(STATSD_URI) 31 | 32 | 33 | class RedisLru(object): 34 | """ 35 | Redis backed LRU cache for functions which return an object which 36 | can survive json.dumps() and json.loads() intact 37 | """ 38 | 39 | def __init__(self, conn, expires=86400, capacity=5000, prefix="lru", tag=None, arg_index=None, kwarg_name=None, slice_obj=slice(None)): 40 | """ 41 | conn: Redis Connection Object 42 | expires: Default key expiration time 43 | capacity: Approximate Maximum size of caching set 44 | prefix: Prefix for all keys in the cache 45 | tag: String (formattable optional) to tag keys with for purging 46 | arg_index/kwarg_name: Choose One, the tag string will be formatted with that argument 47 | slice_obj: Slice object to cut out un picklable thingz 48 | """ 49 | self.conn = conn 50 | self.expires = expires 51 | self.capacity = capacity 52 | self.prefix = prefix 53 | self.tag = tag 54 | self.arg_index = arg_index 55 | self.kwarg_name = kwarg_name 56 | self.slice = slice_obj 57 | self.statsd = statsd_client() 58 | self.dogstatsd = dogstatsd 59 | 60 | def format_key(self, func_name, tag): 61 | if tag is not None: 62 | return ':'.join([self.prefix, tag, func_name]) 63 | return ':'.join([self.prefix, 'tag', func_name]) 64 | 65 | def eject(self, func_name): 66 | self.statsd.incr('rpc-lru.eject') 67 | self.dogstatsd.increment('xmlrpc.lru.eject') 68 | count = min((self.capacity / 10) or 1, 1000) 69 | cache_keys = self.format_key(func_name, '*') 70 | if self.conn.zcard(cache_keys) >= self.capacity: 71 | eject = self.conn.zrange(cache_keys, 0, count) 72 | pipeline = self.conn.pipeline() 73 | pipeline.zremrangebyrank(cache_keys, 0, count) 74 | pipeline.hdel(cache_vals, *eject) 75 | pipeline.execute() 76 | 77 | def get(self, func_name, key, tag): 78 | value = self.conn.hget(self.format_key(func_name, tag), key) 79 | if value: 80 | self.statsd.incr('rpc-lru.hit') 81 | self.dogstatsd.increment('xmlrpc.lru.hit') 82 | value = json.loads(value) 83 | else: 84 | self.statsd.incr('rpc-lru.miss') 85 | self.dogstatsd.increment('xmlrpc.lru.miss') 86 | return value 87 | 88 | def add(self, func_name, key, value, tag): 89 | self.statsd.incr('rpc-lru.add') 90 | self.dogstatsd.increment('xmlrpc.lru.add') 91 | self.eject(func_name) 92 | pipeline = self.conn.pipeline() 93 | pipeline.hset(self.format_key(func_name, tag), key, json.dumps(value)) 94 | pipeline.expire(self.format_key(func_name, tag), self.expires) 95 | pipeline.execute() 96 | return value 97 | 98 | def purge(self, tag): 99 | self.statsd.incr('rpc-lru.purge') 100 | self.dogstatsd.increment('xmlrpc.lru.purge') 101 | keys = self.conn.scan_iter(":".join([self.prefix, tag, '*'])) 102 | pipeline = self.conn.pipeline() 103 | for key in keys: 104 | pipeline.delete(key) 105 | pipeline.execute() 106 | 107 | def decorator(self, func): 108 | @wraps(func) 109 | def wrapper(*args, **kwargs): 110 | if self.conn is None: 111 | return func(*args, **kwargs) 112 | else: 113 | try: 114 | items = args + tuple(sorted(kwargs.items())) 115 | key = json.dumps(items[self.slice]) 116 | tag = None 117 | if self.arg_index is not None and self.kwarg_name is not None: 118 | raise ValueError('only one of arg_index or kwarg_name may be specified') 119 | if self.arg_index is not None: 120 | tag = self.tag % (args[self.arg_index]) 121 | if self.kwarg_name is not None: 122 | tag = self.tag % (kwargs[self.kwarg_name]) 123 | return self.get(func.__name__, key, tag) or self.add(func.__name__, key, func(*args, **kwargs), tag) 124 | except redis.exceptions.RedisError: 125 | return func(*args, **kwargs) 126 | return wrapper 127 | -------------------------------------------------------------------------------- /gae.py: -------------------------------------------------------------------------------- 1 | # Helper for the mirror on GAE 2 | # GAE GETs an action gae_file, giving GAE host and a secret 3 | # PyPI GETs /mkupload/secret, learning path and upload session 4 | # PyPI POSTs to upload session 5 | import urllib2, httplib, threading, os, binascii, urlparse, errno 6 | 7 | POST="""\ 8 | --%(boundary)s 9 | Content-Disposition: form-data; name="secret" 10 | 11 | %(secret)s 12 | --%(boundary)s 13 | Content-Disposition: form-data; name="path" 14 | 15 | %(path)s 16 | --%(boundary)s 17 | Content-Disposition: form-data; name="%(presence)s" 18 | 19 | 1 20 | --%(boundary)s 21 | Content-Disposition: form-data; name="file"; filename="%(path)s" 22 | Content-Type: application/octet-stream 23 | 24 | %(data)s 25 | --%(boundary)s 26 | """ 27 | POST = "\r\n".join(POST.splitlines())+"\r\n" 28 | 29 | def doit(host, secret, srcdir): 30 | x = urllib2.urlopen('http://%s/mkupload/%s' % (host, secret)) 31 | if x.code != 200: 32 | return 33 | path,url = x.read().splitlines() 34 | host, session = urlparse.urlsplit(url)[1:3] 35 | 36 | try: 37 | file_path = os.path.abspath(os.path.join(srcdir, path)) 38 | if not file_path.startswith(srcdir): 39 | data = '' 40 | presence = 'deleted' 41 | else: 42 | data = open(file_path).read() 43 | presence = "present" 44 | except IOError, e: 45 | if e.errno == errno.ENOENT: 46 | # file has been deleted 47 | data = '' 48 | presence = 'deleted' 49 | else: 50 | # some other problem with file. GAE will request transfer 51 | # again 52 | return 53 | boundary = "" 54 | while boundary in data: 55 | boundary = binascii.hexlify(os.urandom(10)) 56 | body = POST % locals() 57 | if ':' in host: 58 | host, port = host.split(':') 59 | else: 60 | port = 80 61 | c = httplib.HTTPConnection(host, port) 62 | c.request('POST', session, 63 | headers = {'Content-type':'multipart/form-data; boundary='+boundary, 64 | 'Content-length':str(len(body)), 65 | 'Host':host}, 66 | body=body) 67 | resp = c.getresponse() 68 | data = resp.read() 69 | # result code should be redirect 70 | c.close() 71 | 72 | def transfer(host, secret, srcdir): 73 | secret = secret.encode('ascii') 74 | t = threading.Thread(target=doit, args=(host, secret, srcdir)) 75 | t.start() 76 | -------------------------------------------------------------------------------- /legacy_passwords.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import hashlib 3 | 4 | import passlib.exc as exc 5 | import passlib.utils.handlers as uh 6 | 7 | from passlib.registry import get_crypt_handler 8 | from passlib.utils import to_unicode 9 | from passlib.utils.compat import uascii_to_str 10 | 11 | 12 | passlib_bcrypt = get_crypt_handler("bcrypt") 13 | 14 | 15 | class bcrypt_sha1(uh.StaticHandler): 16 | 17 | name = "bcrypt_sha1" 18 | _hash_prefix = u"$bcrypt_sha1$" 19 | 20 | def _calc_checksum(self, secret): 21 | # Hash the secret with sha1 first 22 | secret = hashlib.sha1(secret).hexdigest() 23 | 24 | # Hash it with bcrypt 25 | return passlib_bcrypt.hash(secret) 26 | 27 | def to_string(self): 28 | assert self.checksum is not None 29 | return uascii_to_str(self._hash_prefix + base64.b64encode(self.checksum)) 30 | 31 | @classmethod 32 | def from_string(cls, hash, **context): 33 | # default from_string() which strips optional prefix, 34 | # and passes rest unchanged as checksum value. 35 | hash = to_unicode(hash, "ascii", "hash") 36 | hash = cls._norm_hash(hash) 37 | # could enable this for extra strictness 38 | ##pat = cls._hash_regex 39 | ##if pat and pat.match(hash) is None: 40 | ## raise ValueError("not a valid %s hash" % (cls.name,)) 41 | prefix = cls._hash_prefix 42 | if prefix: 43 | if hash.startswith(prefix): 44 | hash = hash[len(prefix):] 45 | else: 46 | raise exc.InvalidHashError(cls) 47 | 48 | # Decode the base64 stored actual hash 49 | hash = unicode(base64.b64decode(hash)) 50 | 51 | return cls(checksum=hash, **context) 52 | 53 | @classmethod 54 | def verify(cls, secret, hash, **context): 55 | # NOTE: classes with multiple checksum encodings should either 56 | # override this method, or ensure that from_string() / _norm_checksum() 57 | # ensures .checksum always uses a single canonical representation. 58 | uh.validate_secret(secret) 59 | self = cls.from_string(hash, **context) 60 | chk = self.checksum 61 | if chk is None: 62 | raise exc.MissingDigestError(cls) 63 | 64 | # Actually use the verify from passlib_bcrypt after hashing the secret 65 | # with sha1 66 | secret = hashlib.sha1(secret).hexdigest() 67 | return passlib_bcrypt.verify(secret, chk) 68 | -------------------------------------------------------------------------------- /mini_pkg_resources.py: -------------------------------------------------------------------------------- 1 | # Subset of setuptools' pkg_resources module 2 | # for file name normalization 3 | import re 4 | 5 | def safe_name(name): 6 | """Convert an arbitrary string to a standard distribution name 7 | 8 | Any runs of non-alphanumeric/. characters are replaced with a single '-'. 9 | """ 10 | return re.sub('[^A-Za-z0-9.]+', '-', name) 11 | 12 | def safe_version(version): 13 | """Convert an arbitrary string to a standard version string 14 | 15 | Spaces become dots, and all other non-alphanumeric characters become 16 | dashes, with runs of multiple dashes condensed to a single dash. 17 | """ 18 | version = version.replace(' ','.') 19 | return re.sub('[^A-Za-z0-9.]+', '-', version) 20 | 21 | def to_filename(name): 22 | """Convert a project or version name to its filename-escaped form 23 | 24 | Any '-' characters are currently replaced with '_'. 25 | """ 26 | return name.replace('-','_') 27 | 28 | -------------------------------------------------------------------------------- /mirrors.txt: -------------------------------------------------------------------------------- 1 | b: appengine, martin@v.loewis.de 2 | c: 178.77.74.179, pypi@zopyx.com 3 | d: pypi.websushi.org, jezdez+pypi@enn.io 4 | e: pypi.tuna.tsinghua.edu.cn, thu-opensource-mirror-admin@googlegroups.com 5 | f: services02.fe.rzob.gocept.net, support@gocept.com 6 | g: 129.21.171.98, rc-help@rit.edu 7 | -------------------------------------------------------------------------------- /mod_python.conf: -------------------------------------------------------------------------------- 1 | 2 | SetHandler mod_python 3 | PythonPath "['/data/pypi/src/pypi']+sys.path" 4 | PythonHandler pypi::handle 5 | PythonDebug On 6 | 7 | -------------------------------------------------------------------------------- /pypi.cgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # $Id$ 4 | 5 | import sys, os, cgi, StringIO, traceback 6 | from BaseHTTPServer import BaseHTTPRequestHandler, DEFAULT_ERROR_MESSAGE 7 | 8 | # turn this on if you need to do server maintenance 9 | if 0: 10 | sys.stdout.write('Status: 503 Server down for maintenance\r\n') 11 | sys.stdout.write('Content-Type: text/plain\r\n\r\n') 12 | print 'The Cheese Shop server is down for a short time for maintenance.' 13 | print 'Please try to connect later.' 14 | sys.exit(0) 15 | 16 | # 17 | # Provide interface to CGI HTTP response handling 18 | # 19 | class RequestWrapper: 20 | '''Used to make the CGI server look like a BaseHTTPRequestHandler 21 | ''' 22 | def __init__(self, config, rfile, wfile): 23 | self.wfile = wfile 24 | self.rfile = rfile 25 | self.config = config 26 | def send_response(self, code, message=''): 27 | self.wfile.write('Status: %s %s\r\n'%(code, message)) 28 | def send_header(self, keyword, value): 29 | self.wfile.write("%s: %s\r\n" % (keyword, value)) 30 | def set_content_type(self, content_type): 31 | self.send_header('Content-Type', content_type) 32 | def end_headers(self): 33 | self.wfile.write("\r\n") 34 | 35 | # 36 | # Now do the actual CGI handling 37 | # 38 | try: 39 | sys.path.insert(0, '/data/pypi/src/pypi') 40 | from webui import WebUI 41 | import config 42 | cfg = config.Config(os.environ.get("PYPI_COFNIG", "config.ini")) 43 | request = RequestWrapper(cfg, sys.stdin, sys.stdout) 44 | handler = WebUI(request, os.environ) 45 | handler.run() 46 | except SystemExit: 47 | pass 48 | except: 49 | sys.stdout.write('Status: 500 Internal Server Error\r\n') 50 | sys.stdout.write('Content-Type: text/html\r\n\r\n') 51 | sys.stdout.write("
")
52 |     s = StringIO.StringIO()
53 |     traceback.print_exc(None, s)
54 |     sys.stdout.write(cgi.escape(s.getvalue()))
55 |     sys.stdout.write("
\n") 56 | 57 | # vim: set filetype=python ts=4 sw=4 et si 58 | -------------------------------------------------------------------------------- /pypi.fcgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import thfcgi, os, sys, StringIO, traceback, cgi, syslog 3 | 4 | # 5 | # Provide interface to CGI HTTP response handling 6 | # 7 | class RequestWrapper: 8 | '''Used to make the FCGI server look like a BaseHTTPRequestHandler 9 | ''' 10 | def __init__(self, config, req): 11 | self.wfile = req.out 12 | self.rfile = req.stdin 13 | self.config = config 14 | def send_response(self, code, message=''): 15 | self.wfile.write('Status: %s %s\r\n'%(code, message)) 16 | def send_header(self, keyword, value): 17 | self.wfile.write("%s: %s\r\n" % (keyword, value)) 18 | def set_content_type(self, content_type): 19 | self.send_header('Content-Type', content_type) 20 | def end_headers(self): 21 | self.wfile.write("\r\n") 22 | 23 | def handle_request(req, env): 24 | try: 25 | import store 26 | store.keep_conn = True 27 | from webui import WebUI 28 | request = RequestWrapper(cfg, req) 29 | handler = WebUI(request, env) 30 | handler.run() 31 | except SystemExit: 32 | pass 33 | except: 34 | req.out.write('Status: 500 Internal Server Error\r\n') 35 | req.out.write('Content-Type: text/html\r\n\r\n') 36 | req.out.write("
")
37 |         s = StringIO.StringIO()
38 |         traceback.print_exc(None, s)
39 |         req.out.write(cgi.escape(s.getvalue()))
40 |         req.out.write("
\n") 41 | req.finish() 42 | 43 | # 44 | # Now do the actual CGI handling 45 | # 46 | os.umask(002) # make directories group-writable 47 | prefix = os.path.dirname(__file__) 48 | sys.path.insert(0, prefix) 49 | import config 50 | cfg = config.Config('/data/pypi/config.ini') 51 | fcg = thfcgi.FCGI(handle_request, 52 | max_requests=-1, 53 | backlog=50, 54 | max_threads=1) 55 | try: 56 | try: 57 | fcg.run() 58 | syslog.syslog("pypi.fcgi: run completed") 59 | except: 60 | import traceback 61 | syslog.syslog(''.join(traceback.format_exception(*sys.exc_info()))) 62 | raise 63 | finally: 64 | syslog.syslog("pypi.fcgi: exiting") 65 | 66 | 67 | -------------------------------------------------------------------------------- /pypi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # $Id$ 4 | 5 | import os.path 6 | import sys 7 | 8 | prefix = os.path.dirname(__file__) 9 | sys.path.insert(0, prefix) 10 | 11 | from mod_python import apache 12 | import sys, os, cgi, StringIO, traceback 13 | from BaseHTTPServer import BaseHTTPRequestHandler, DEFAULT_ERROR_MESSAGE 14 | from webui import WebUI 15 | import config 16 | 17 | CONFIG_FILE = os.environ.get("PYPI_CONFIG", os.path.join(prefix, 'config.ini')) 18 | 19 | class RequestWrapper: 20 | '''Used to make the CGI server look like a BaseHTTPRequestHandler 21 | ''' 22 | def __init__(self, config, req): 23 | self.wfile = self.req = req 24 | self.rfile = StringIO.StringIO(req.read()) 25 | self.config = config 26 | def send_response(self, code, message=''): 27 | self.req.status = code 28 | def send_header(self, keyword, value): 29 | self.req.headers_out[keyword] = value 30 | def set_content_type(self, content_type): 31 | self.req.content_type = content_type 32 | def end_headers(self): 33 | pass 34 | 35 | def handle(req): 36 | req.content_type = req.headers_out['Content-Type'] = 'text/html' 37 | cfg = config.Config(os.environ.get("PYPI_COFNIG", CONFIG_FILE)) 38 | request = RequestWrapper(cfg, req) 39 | pseudoenv = {} 40 | pseudoenv['CONTENT_TYPE'] = req.headers_in.get('content-type', '') 41 | pseudoenv['REMOTE_ADDR'] = req.get_remote_host(apache.REMOTE_NOLOOKUP) 42 | pseudoenv['HTTP_USER_AGENT'] = req.headers_in.get('user-agent', '') 43 | pseudoenv['QUERY_STRING'] = req.args 44 | pseudoenv['HTTP_CGI_AUTHORIZATION'] = req.headers_in.get('authorization', 45 | '') 46 | pseudoenv['REQUEST_METHOD'] = req.method 47 | path_info = req.path_info 48 | pseudoenv['PATH_INFO'] = path_info 49 | try: 50 | handler = WebUI(request, pseudoenv) 51 | handler.run() 52 | except: 53 | s = StringIO.StringIO() 54 | traceback.print_exc(None, s) 55 | req.write("
")
56 |         req.write(cgi.escape(s.getvalue()))
57 |         req.write("
") 58 | req.headers_out['Content-Type'] = 'text/html' 59 | req.content_type = req.headers_out['Content-Type'] 60 | return apache.OK 61 | 62 | # vim: set filetype=python ts=4 sw=4 et si 63 | -------------------------------------------------------------------------------- /pypi.wsgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import os 4 | prefix = os.path.dirname(__file__) 5 | sys.path.insert(0, prefix) 6 | import cStringIO 7 | import webui 8 | import store 9 | import config 10 | import re 11 | from functools import partial 12 | 13 | store.keep_conn = True 14 | 15 | CONFIG_FILE = os.environ.get("PYPI_CONFIG", os.path.join(prefix, 'config.ini')) 16 | 17 | 18 | class Request: 19 | 20 | def __init__(self, environ, start_response): 21 | self.start_response = start_response 22 | try: 23 | length = int(environ.get('CONTENT_LENGTH', 0)) 24 | except ValueError: 25 | length = 0 26 | self.rfile = cStringIO.StringIO(environ['wsgi.input'].read(length)) 27 | self.wfile = cStringIO.StringIO() 28 | self.config = config.Config(CONFIG_FILE) 29 | self.status = None 30 | self.headers = [] 31 | 32 | def set_status(self, status): 33 | self.status = status 34 | 35 | def send_response(self, code, message='no details available'): 36 | self.status = '%s %s' % (code, message) 37 | self.headers = [] 38 | 39 | def send_header(self, keyword, value): 40 | self.headers.append((keyword, value)) 41 | 42 | def set_content_type(self, content_type): 43 | self.send_header('Content-Type', content_type) 44 | 45 | def end_headers(self): 46 | self.start_response(self.status, self.headers) 47 | 48 | 49 | class CacheControlMiddleware(object): 50 | 51 | def __init__(self, app): 52 | self.app = app 53 | 54 | def __call__(self, environ, start_response): 55 | 56 | def _start_response(status, headers, exc_info=None): 57 | script = environ.get("SCRIPT_NAME", None) 58 | if script in set(["/simple"]): 59 | # Cache for a day in Fastly, but 10 minutes in browsers 60 | headers += [ 61 | ("Surrogate-Control", "max-age=86400"), 62 | ("Cache-Control", "max-age=600, public"), 63 | ] 64 | elif script in set(["/packages"]): 65 | if status[:3] in ["200", "304"]: 66 | # Cache for a year 67 | headers += [("Cache-Control", "max-age=31557600, public")] 68 | else: 69 | # Cache for an hour 70 | headers += [("Cache-Control", "max-age=3600, public")] 71 | elif script in set(["/mirrors", "/security"]): 72 | # Cache these for a week 73 | headers += [("Cache-Control", "max-age=604800, public")] 74 | 75 | # http://www.gnuterrypratchett.com/ 76 | headers += [("X-Clacks-Overhead", "GNU Terry Pratchett")] 77 | 78 | return start_response(status, headers, exc_info) 79 | 80 | return self.app(environ, _start_response) 81 | 82 | 83 | class SecurityHeaderMiddleware(object): 84 | 85 | def __init__(self, app): 86 | self.app = app 87 | 88 | def __call__(self, environ, start_response): 89 | 90 | def _start_response(status, headers, exc_info=None): 91 | headers += [ 92 | ("X-Frame-Options", "deny"), 93 | ("X-XSS-Protection", "1; mode=block"), 94 | ("X-Content-Type-Options", "nosniff"), 95 | ] 96 | 97 | return start_response(status, headers, exc_info) 98 | 99 | return self.app(environ, _start_response) 100 | 101 | 102 | def debug(environ, start_response): 103 | if environ['PATH_INFO'].startswith("/auth") and \ 104 | "HTTP_AUTHORIZATION" not in environ: 105 | start_response("401 login", 106 | [('WWW-Authenticate', 'Basic realm="foo"')]) 107 | return 108 | start_response("200 ok", [('Content-type', 'text/plain')]) 109 | environ = environ.items() 110 | environ.sort() 111 | for k, v in environ: 112 | yield "%s=%s\n" % (k, v) 113 | return 114 | 115 | 116 | def application(environ, start_response): 117 | if "HTTP_AUTHORIZATION" in environ: 118 | environ["HTTP_CGI_AUTHORIZATION"] = environ["HTTP_AUTHORIZATION"] 119 | try: 120 | r = Request(environ, start_response) 121 | webui.WebUI(r, environ).run() 122 | return [r.wfile.getvalue()] 123 | except Exception, e: 124 | import traceback;traceback.print_exc() 125 | return ['Ooops, there was a problem (%s)' % e] 126 | #application=debug 127 | 128 | 129 | # Handle Caching at the WSGI layer 130 | application = CacheControlMiddleware(application) 131 | 132 | 133 | # Add some Security Headers to every response 134 | application = SecurityHeaderMiddleware(application) 135 | 136 | 137 | # pretend to be like the UWSGI configuration - set SCRIPT_NAME to the first 138 | # part of the PATH_INFO if it's valid and remove that part from the PATH_INFO 139 | def site_fake(app, environ, start_response): 140 | PATH_INFO = environ['PATH_INFO'] 141 | m = re.match('^/(pypi|simple|daytime|serversig|mirrors|id|oauth|google_login|' 142 | 'security|packages|openid_login|openid_claim)(.*)', PATH_INFO) 143 | if not m: 144 | start_response("404 not found", [('Content-type', 'text/plain')]) 145 | return ['Not Found: %s' % PATH_INFO] 146 | 147 | environ['SCRIPT_NAME'] = '/' + m.group(1) 148 | environ['PATH_INFO'] = m.group(2) 149 | 150 | return app(environ, start_response) 151 | 152 | 153 | if __name__ == '__main__': 154 | # very simple wsgi server so we can play locally 155 | from wsgiref.simple_server import make_server 156 | httpd = make_server('', 8000, partial(site_fake, application)) 157 | print "Serving on port 8000..." 158 | httpd.serve_forever() 159 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | authomatic 2 | argon2-cffi 3 | bcrypt 4 | bleach 5 | boto 6 | datadog 7 | defusedxml 8 | docutils 9 | fs 10 | html5lib 11 | itsdangerous 12 | packaging 13 | passlib 14 | perfmetrics 15 | psycopg2 16 | pybrowserid 17 | Pygments 18 | python-dateutil 19 | python-openid 20 | pytz 21 | raven 22 | readme_renderer 23 | redis 24 | requests[security] 25 | rfc3986 26 | rq 27 | six 28 | times 29 | uWSGI 30 | wsgiref 31 | zope.browser 32 | zope.component 33 | zope.configuration 34 | zope.contenttype 35 | zope.event 36 | zope.exceptions 37 | zope.i18n 38 | zope.i18nmessageid 39 | zope.interface 40 | zope.location 41 | zope.pagetemplate 42 | zope.proxy 43 | zope.publisher 44 | zope.schema 45 | zope.security 46 | zope.tal 47 | zope.tales 48 | zope.traversing 49 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file requirements.txt requirements.in 6 | # 7 | argon2-cffi==16.3.0 8 | arrow==0.7.0 # via times 9 | authomatic==0.1.0.post1 10 | bcrypt==3.1.1 11 | bleach==1.4.2 12 | boto==2.40.0 13 | cffi==1.6.0 # via argon2-cffi, bcrypt, cryptography 14 | click==6.6 # via rq 15 | contextlib2==0.5.3 # via raven 16 | cryptography==1.3.2 # via pyopenssl 17 | datadog==0.16.0 18 | decorator==4.1.1 # via datadog 19 | defusedxml==0.4.1 20 | docutils==0.12 21 | enum34==1.1.6 # via argon2-cffi, cryptography 22 | fs==0.5.4 23 | html5lib==0.9999999 24 | idna==2.1 # via cryptography 25 | ipaddress==1.0.16 # via cryptography 26 | itsdangerous==0.24 27 | ndg-httpsclient==0.4.0 # via requests 28 | packaging==16.8 29 | passlib==1.7 30 | perfmetrics==2.0 31 | psycopg2==2.6.1 32 | pyasn1==0.1.9 # via cryptography, requests 33 | pybrowserid==0.11.0 34 | pycparser==2.14 # via cffi 35 | pygments==2.1.3 36 | pyopenssl==16.0.0 # via ndg-httpsclient, requests 37 | pyparsing==2.1.4 # via packaging 38 | python-dateutil==2.5.3 39 | python-openid==2.2.5 40 | pytz==2016.4 41 | raven==5.17.0 42 | readme-renderer==0.7.0 43 | redis==2.10.5 44 | requests[security]==2.10.0 45 | rfc3986==0.3.1 46 | rq==0.6.0 47 | simplejson==3.11.1 # via datadog 48 | six==1.10.0 49 | times==0.7 50 | transaction==1.5.0 # via zope.traversing 51 | uwsgi==2.0.13.1 52 | wsgiref==0.1.2 53 | zope.browser==2.1.0 54 | zope.component==4.2.2 55 | zope.configuration==4.0.3 56 | zope.contenttype==4.1.0 57 | zope.event==4.2.0 58 | zope.exceptions==4.0.8 59 | zope.i18n==4.1.0 60 | zope.i18nmessageid==4.0.3 61 | zope.interface==4.1.3 62 | zope.location==4.0.3 63 | zope.pagetemplate==4.2.1 64 | zope.proxy==4.2.0 65 | zope.publisher==4.2.2 66 | zope.schema==4.4.2 67 | zope.security==4.0.3 68 | zope.tal==4.2.0 69 | zope.tales==4.1.1 70 | zope.traversing==4.0.0 71 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # $Id$ 4 | 5 | from distutils.core import setup 6 | 7 | # perform the setup action 8 | setup( 9 | name = "pypi", 10 | version = '2005-08-01', 11 | description = 12 | "PyPI is the Python Package Index at http://pypi.python.org/", 13 | long_description = '''PyPI has a new home at 14 | . Users should need not need to change 15 | anything, as the old "www" address should still work. Use of the new 16 | address in preference is encouraged. 17 | 18 | Developers interested in how PyPI works, or in contributing to the project, 19 | should visit http://wiki.python.org/moin/CheeseShopDev 20 | ''', 21 | author = "Richard Jones", 22 | author_email = "richard@python.org", 23 | url = 'http://wiki.python.org/moin/CheeseShopDev', 24 | classifiers = [ 25 | 'Development Status :: 5 - Production/Stable', 26 | 'Environment :: Web Environment', 27 | 'Intended Audience :: Developers', 28 | 'Intended Audience :: System Administrators', 29 | 'Natural Language :: English', 30 | 'License :: OSI Approved :: BSD License', 31 | 'Operating System :: OS Independent', 32 | 'Programming Language :: Python', 33 | 'Topic :: Software Development :: Libraries', 34 | ], 35 | ) 36 | 37 | # vim: set filetype=python ts=4 sw=4 et si 38 | -------------------------------------------------------------------------------- /standalone.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import config, webui, BaseHTTPServer, urllib, sys, getopt, os 3 | 4 | prefix = os.path.dirname(__file__) 5 | sys.path.insert(0, prefix) 6 | 7 | CONFIG_FILE = os.environ.get("PYPI_CONFIG", os.path.join(prefix, 'config.ini')) 8 | 9 | 10 | class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): 11 | config = config.Config(CONFIG_FILE) 12 | ssh_user = None 13 | 14 | def set_content_type(self, content_type): 15 | self.send_header('Content-Type', content_type) 16 | 17 | def run(self): 18 | if self.path == '/': 19 | self.send_response(301) 20 | self.send_header('Location', '/pypi') 21 | return 22 | for scriptname in ('/mirrors', '/simple', '/pypi', 23 | '/serversig', '/daytime', '/id'): 24 | if self.path.startswith(scriptname): 25 | rest = self.path[len(scriptname):] 26 | break 27 | else: 28 | # invalid URL 29 | return 30 | 31 | # The text below is mostly copied from CGIHTTPServer 32 | 33 | i = rest.rfind('?') 34 | if i >= 0: 35 | rest, query = rest[:i], rest[i+1:] 36 | else: 37 | query = '' 38 | 39 | env = {} 40 | #env['SERVER_SOFTWARE'] = self.version_string() 41 | #env['SERVER_NAME'] = self.server.server_name 42 | #env['SERVER_PORT'] = str(self.server.server_port) 43 | env['GATEWAY_INTERFACE'] = 'CGI/1.1' 44 | env['SERVER_PROTOCOL'] = self.protocol_version 45 | env['REQUEST_METHOD'] = self.command 46 | uqrest = urllib.unquote(rest) 47 | env['PATH_INFO'] = uqrest 48 | # env['PATH_TRANSLATED'] = self.translate_path(uqrest) 49 | env['SCRIPT_NAME'] = scriptname 50 | if query: 51 | env['QUERY_STRING'] = query 52 | host = self.address_string() 53 | if host != self.client_address[0]: 54 | env['REMOTE_HOST'] = host 55 | env['REMOTE_ADDR'] = self.client_address[0] 56 | if self.ssh_user: 57 | # ignore authorization headers if this is a SSH client 58 | authorization = None 59 | env['SSH_USER'] = self.ssh_user 60 | else: 61 | authorization = self.headers.getheader("authorization") 62 | if authorization: 63 | env['HTTP_CGI_AUTHORIZATION'] = authorization 64 | authorization = authorization.split() 65 | if len(authorization) == 2: 66 | import base64, binascii 67 | env['AUTH_TYPE'] = authorization[0] 68 | if self.headers.typeheader is None: 69 | env['CONTENT_TYPE'] = self.headers.type 70 | else: 71 | env['CONTENT_TYPE'] = self.headers.typeheader 72 | length = self.headers.getheader('content-length') 73 | if length: 74 | env['CONTENT_LENGTH'] = length 75 | referer = self.headers.getheader('referer') 76 | if referer: 77 | env['HTTP_REFERER'] = referer 78 | accept = [] 79 | for line in self.headers.getallmatchingheaders('accept'): 80 | if line[:1] in "\t\n\r ": 81 | accept.append(line.strip()) 82 | else: 83 | accept = accept + line[7:].split(',') 84 | env['HTTP_ACCEPT'] = ','.join(accept) 85 | ua = self.headers.getheader('user-agent') 86 | if ua: 87 | env['HTTP_USER_AGENT'] = ua 88 | co = filter(None, self.headers.getheaders('cookie')) 89 | if co: 90 | env['HTTP_COOKIE'] = ', '.join(co) 91 | ac = self.headers.getheader('accept-encoding') 92 | if ac: 93 | env['HTTP_ACCEPT_ENCODING'] = ac 94 | 95 | webui.WebUI(self, env).run() 96 | do_GET = do_POST = run 97 | 98 | class StdinoutHandler(RequestHandler): 99 | def __init__(self, remote_user): 100 | self.ssh_user = remote_user 101 | try: 102 | host,port,_ = os.environ['SSH_CLIENT'].split() 103 | except KeyError: 104 | host = port = '' 105 | # request, client_address, server 106 | RequestHandler.__init__(self, None, (host, port), None) 107 | def setup(self): 108 | self.rfile = sys.stdin 109 | #import StringIO 110 | #self.rfile = StringIO.StringIO('GET /pypi HTTP/1.0\r\n\r\n') 111 | self.wfile = sys.stdout 112 | 113 | def main(): 114 | os.umask(002) # make directories group-writable 115 | port = 8000 116 | remote_user = None 117 | opts, args = getopt.getopt(sys.argv[1:], 'ir:p:', 118 | ['interactive', 'remote-user=', 'port=']) 119 | assert not args 120 | for opt, val in opts: 121 | if opt in ('-i', '--interactive'): 122 | port = None 123 | elif opt in ('-r','--remote-user'): 124 | port = None # implies -i 125 | remote_user = val 126 | elif opt in ('-p', '--port'): 127 | port = int(val) 128 | if port: 129 | httpd = BaseHTTPServer.HTTPServer(('',port), RequestHandler) 130 | httpd.serve_forever() 131 | else: 132 | StdinoutHandler(remote_user) 133 | 134 | if __name__=='__main__': 135 | main() 136 | -------------------------------------------------------------------------------- /static/css/pygments.css: -------------------------------------------------------------------------------- 1 | pre .hll { background-color: #ffffcc } 2 | pre { background: #f0f0f0; } 3 | pre .c { color: #60a0b0; font-style: italic } /* Comment */ 4 | pre .err { border: 1px solid #FF0000 } /* Error */ 5 | pre .k { color: #007020; font-weight: bold } /* Keyword */ 6 | pre .o { color: #666666 } /* Operator */ 7 | pre .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ 8 | pre .cp { color: #007020 } /* Comment.Preproc */ 9 | pre .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ 10 | pre .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ 11 | pre .gd { color: #A00000 } /* Generic.Deleted */ 12 | pre .ge { font-style: italic } /* Generic.Emph */ 13 | pre .gr { color: #FF0000 } /* Generic.Error */ 14 | pre .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 15 | pre .gi { color: #00A000 } /* Generic.Inserted */ 16 | pre .go { color: #808080 } /* Generic.Output */ 17 | pre .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 18 | pre .gs { font-weight: bold } /* Generic.Strong */ 19 | pre .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 20 | pre .gt { color: #0040D0 } /* Generic.Traceback */ 21 | pre .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 22 | pre .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 23 | pre .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 24 | pre .kp { color: #007020 } /* Keyword.Pseudo */ 25 | pre .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 26 | pre .kt { color: #902000 } /* Keyword.Type */ 27 | pre .m { color: #40a070 } /* Literal.Number */ 28 | pre .s { color: #4070a0 } /* Literal.String */ 29 | pre .na { color: #4070a0 } /* Name.Attribute */ 30 | pre .nb { color: #007020 } /* Name.Builtin */ 31 | pre .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 32 | pre .no { color: #60add5 } /* Name.Constant */ 33 | pre .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 34 | pre .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 35 | pre .ne { color: #007020 } /* Name.Exception */ 36 | pre .nf { color: #06287e } /* Name.Function */ 37 | pre .nl { color: #002070; font-weight: bold } /* Name.Label */ 38 | pre .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 39 | pre .nt { color: #062873; font-weight: bold } /* Name.Tag */ 40 | pre .nv { color: #bb60d5 } /* Name.Variable */ 41 | pre .ow { color: #007020; font-weight: bold } /* Operator.Word */ 42 | pre .w { color: #bbbbbb } /* Text.Whitespace */ 43 | pre .mf { color: #40a070 } /* Literal.Number.Float */ 44 | pre .mh { color: #40a070 } /* Literal.Number.Hex */ 45 | pre .mi { color: #40a070 } /* Literal.Number.Integer */ 46 | pre .mo { color: #40a070 } /* Literal.Number.Oct */ 47 | pre .sb { color: #4070a0 } /* Literal.String.Backtick */ 48 | pre .sc { color: #4070a0 } /* Literal.String.Char */ 49 | pre .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 50 | pre .s2 { color: #4070a0 } /* Literal.String.Double */ 51 | pre .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 52 | pre .sh { color: #4070a0 } /* Literal.String.Heredoc */ 53 | pre .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 54 | pre .sx { color: #c65d09 } /* Literal.String.Other */ 55 | pre .sr { color: #235388 } /* Literal.String.Regex */ 56 | pre .s1 { color: #4070a0 } /* Literal.String.Single */ 57 | pre .ss { color: #517918 } /* Literal.String.Symbol */ 58 | pre .bp { color: #007020 } /* Name.Builtin.Pseudo */ 59 | pre .vc { color: #bb60d5 } /* Name.Variable.Class */ 60 | pre .vg { color: #bb60d5 } /* Name.Variable.Global */ 61 | pre .vi { color: #bb60d5 } /* Name.Variable.Instance */ 62 | pre .il { color: #40a070 } /* Literal.Number.Integer.Long */ 63 | -------------------------------------------------------------------------------- /static/css/pypi-screen.css: -------------------------------------------------------------------------------- 1 | /* PyPI styles that are specifically designed to override base / inherited 2 | * screen styles */ 3 | 4 | #download-button a.button { 5 | display: inline-block; 6 | zoom: 1; /* zoom and *display = ie7 hack for display:inline-block */ 7 | *display: inline; 8 | vertical-align: baseline; 9 | margin: 0 2px; 10 | outline: none; 11 | cursor: pointer; 12 | text-align: center; 13 | text-decoration: none; 14 | font: 14px/100% Arial, Helvetica, sans-serif; 15 | padding: .5em 2em .55em; 16 | text-shadow: 0 1px 1px rgba(0,0,0,.3); 17 | -webkit-border-radius: .5em; 18 | -moz-border-radius: .5em; 19 | border-radius: .5em; 20 | -webkit-box-shadow: 0 1px 2px rgba(0,0,0,.2); 21 | -moz-box-shadow: 0 1px 2px rgba(0,0,0,.2); 22 | box-shadow: 0 1px 2px rgba(0,0,0,.2); 23 | } 24 | #download-button a.button:hover { 25 | text-decoration: none; 26 | } 27 | #download-button a.button:active { 28 | position: relative; 29 | top: 1px; 30 | } 31 | 32 | #download-button a.green { 33 | color: #e8f0de; 34 | border: solid 1px #538312; 35 | background: #64991e; 36 | background: -webkit-gradient(linear, left top, left bottom, from(#7db72f), to(#4e7d0e)); 37 | background: -moz-linear-gradient(top, #7db72f, #4e7d0e); 38 | filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#7db72f', endColorstr='#4e7d0e'); 39 | } 40 | #download-button a.green:hover { 41 | background: #538018; 42 | background: -webkit-gradient(linear, left top, left bottom, from(#6b9d28), to(#436b0c)); 43 | background: -moz-linear-gradient(top, #6b9d28, #436b0c); 44 | filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#6b9d28', endColorstr='#436b0c'); 45 | } 46 | #download-button a.green:active { 47 | color: #a9c08c; 48 | background: -webkit-gradient(linear, left top, left bottom, from(#4e7d0e), to(#7db72f)); 49 | background: -moz-linear-gradient(top, #4e7d0e, #7db72f); 50 | filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#4e7d0e', endColorstr='#7db72f'); 51 | } 52 | 53 | 54 | pre { 55 | margin: 0 2em 0 2em; 56 | padding: 10px; 57 | /* border: thin solid #AAF; 58 | border-radius: .5em;*/ 59 | } 60 | -------------------------------------------------------------------------------- /static/images/PythonPoweredAnimSmall.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/PythonPoweredAnimSmall.gif -------------------------------------------------------------------------------- /static/images/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/blank.gif -------------------------------------------------------------------------------- /static/images/bullet.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/bullet.gif -------------------------------------------------------------------------------- /static/images/button-on-bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/button-on-bg.png -------------------------------------------------------------------------------- /static/images/header-bg2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/header-bg2.png -------------------------------------------------------------------------------- /static/images/nav-off-bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/nav-off-bg.png -------------------------------------------------------------------------------- /static/images/nav-on-bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/nav-on-bg.png -------------------------------------------------------------------------------- /static/images/python-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/python-logo.png -------------------------------------------------------------------------------- /static/images/testing-site-right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/testing-site-right.png -------------------------------------------------------------------------------- /static/images/testing-site.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/testing-site.png -------------------------------------------------------------------------------- /static/images/trans.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/static/images/trans.gif -------------------------------------------------------------------------------- /static/styles/netscape4.css: -------------------------------------------------------------------------------- 1 | #left-hand-navigation 2 | { 3 | /*/*//*/ 4 | position:absolute; 5 | left:5px; 6 | top:10em; 7 | z-index:1; 8 | font-family: Arial, Helvetica, sans-serif; 9 | font-size:0.75em; 10 | /* */ 11 | } 12 | 13 | #left-hand-navigation a 14 | { 15 | /*/*//*/ 16 | color: #3C4B6B; 17 | /* */ 18 | } 19 | 20 | #google 21 | { 22 | /*/*//*/ 23 | display:none; 24 | /* */ 25 | } 26 | 27 | #content-body 28 | { 29 | /*/*//*/ 30 | position:absolute; 31 | top:7em; 32 | left:9em; 33 | width:100%; 34 | z-index:1; 35 | font-family: Arial, Helvetica, sans-serif; 36 | font-size:0.82em; 37 | /* */ 38 | } 39 | 40 | #content-body a 41 | { 42 | /*/*//*/ 43 | color: #3C4B6B; 44 | /* */ 45 | } 46 | 47 | #document-navigation 48 | { 49 | /*/*//*/ 50 | position: relative; 51 | top:20px; 52 | left:0%; 53 | z-index:1; 54 | padding:10px; 55 | /* */ 56 | } 57 | 58 | #border-corner 59 | { 60 | /*/*//*/ 61 | display:none; 62 | /* */ 63 | } 64 | 65 | #smallcorner 66 | { 67 | /*/*//*/ 68 | display:none; 69 | /* */ 70 | } 71 | 72 | #searchcorner 73 | { 74 | /*/*//*/ 75 | display:none; 76 | /* */ 77 | } 78 | 79 | 80 | 81 | #logoheader 82 | { 83 | /*/*//*/ 84 | position:absolute; 85 | top:0; 86 | border:0 none; 87 | left:0; 88 | padding:10px; 89 | font-size: medium; 90 | z-index: 1 91 | /* */ 92 | } 93 | 94 | #logo 95 | { 96 | /*/*//*/ 97 | width:20px; 98 | /* */ 99 | } 100 | 101 | #logolink 102 | { 103 | /*/*//*/ 104 | border:none; 105 | /* */ 106 | } 107 | 108 | #search 109 | { 110 | /*/*//*/ 111 | border: none; 112 | padding-top:20px; 113 | font-family: Verdana, Helvetica, sans-serif; 114 | font-size:0.9em; 115 | padding-top:5em; 116 | /* */ 117 | } 118 | 119 | #skiptonav 120 | { 121 | /*/*//*/ 122 | border: 0px none; 123 | display:none; 124 | /* */ 125 | } 126 | 127 | #breadcrumb 128 | { 129 | /*/*//*/ 130 | position:absolute; 131 | top:0px; 132 | /* */ 133 | } 134 | 135 | #utility-menu 136 | { 137 | /*/*//*/ 138 | position:absolute; 139 | top:8em; 140 | font-family: Arial, Helvetica, sans-serif; 141 | font-size:0.75em; 142 | /* */ 143 | } 144 | 145 | #utility-menu a 146 | { 147 | /*/*//*/ 148 | color: #3C4B6B; 149 | /* */ 150 | } 151 | 152 | ul 153 | { 154 | /*/*//*/ 155 | list-style: none; 156 | padding:0; 157 | margin-left:-20px; 158 | /* */ 159 | } -------------------------------------------------------------------------------- /static/styles/print.css: -------------------------------------------------------------------------------- 1 | #left-hand-navigation, #google, #document-navigation, #border-corner, #smallcorner, #searchcorner, #logo, #search, #utility-menu, #skiptonav 2 | { 3 | display:none; 4 | } 5 | 6 | #content-body 7 | { 8 | font-family: Georgia, "Bitstream Vera Serif", "New York", Palatino, serif; 9 | font-size:11pt; 10 | } 11 | 12 | #content-body a 13 | { 14 | color: #000000; 15 | text-decoration:none; 16 | display:inline; 17 | } 18 | 19 | pre { 20 | font-size:10pt; 21 | } 22 | 23 | #body-main a 24 | { 25 | font-weight:bold; 26 | } 27 | 28 | 29 | h1 30 | { 31 | font-size:14pt; 32 | } 33 | 34 | iframe { 35 | display:none; 36 | } 37 | 38 | #footer { 39 | display:none; 40 | } 41 | -------------------------------------------------------------------------------- /static/styles/screen-switcher-default.css: -------------------------------------------------------------------------------- 1 | @import url(../styles/styles.css); 2 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import urlparse 3 | 4 | import redis 5 | import requests 6 | 7 | from fncache import RedisLru 8 | 9 | def purge_redis_cache(cache_redis_url, tags): 10 | cache_redis = redis.StrictRedis.from_url(cache_redis_url) 11 | lru_cache = RedisLru(cache_redis) 12 | 13 | all_tags = set(tags) 14 | for tag in set(all_tags): 15 | print tag 16 | lru_cache.purge(tag) 17 | 18 | def purge_fastly_tags(domain, api_key, service_id, tags): 19 | session = requests.session() 20 | headers = {"Fastly-Key": api_key, "Accept": "application/json"} 21 | 22 | all_tags = set(tags) 23 | purges = {} 24 | 25 | for tag in set(all_tags): 26 | # Build the URL 27 | url_path = "/service/%s/purge/%s" % (service_id, tag) 28 | url = urlparse.urljoin(domain, url_path) 29 | 30 | # Issue the Purge 31 | resp = session.post(url, headers=headers) 32 | resp.raise_for_status() 33 | 34 | # Store the Purge ID so we can track it later 35 | purges[tag] = resp.json()["id"] 36 | 37 | # for tag, purge_id in purges.iteritems(): 38 | # # Ensure that the purge completed successfully 39 | # url = urlparse.urljoin(domain, "/purge") 40 | # status = session.get(url, params={"id": purge_id}) 41 | # status.raise_for_status() 42 | 43 | # # If the purge completely successfully remove the tag from 44 | # # our list. 45 | # if status.json().get("results", {}).get("complete", None): 46 | # all_tags.remove(tag) 47 | 48 | import config 49 | import store 50 | 51 | from zope.pagetemplate.pagetemplatefile import PageTemplateFile 52 | 53 | 54 | class PyPiPageTemplate(PageTemplateFile): 55 | def pt_getContext(self, args=(), options={}, **kw): 56 | """Add our data into ZPT's defaults""" 57 | rval = PageTemplateFile.pt_getContext(self, args=args) 58 | options.update(rval) 59 | return options 60 | 61 | def rss_regen(): 62 | root = os.path.dirname(os.path.abspath(__file__)) 63 | conf = config.Config(os.path.join(root, "config.ini")) 64 | template_dir = os.path.join(root, 'templates') 65 | 66 | if conf.cache_redis_url: 67 | cache_redis = redis.StrictRedis.from_url(conf.cache_redis_url) 68 | else: 69 | return True 70 | 71 | (protocol, machine, path, x, x, x) = urlparse.urlparse(conf.url) 72 | 73 | context = {} 74 | context['store'] = store.Store(conf) 75 | context['url_machine'] = '%s://%s'%(protocol, machine) 76 | context['url_path'] = path 77 | context['test'] = '' 78 | if 'testpypi' in conf.url: 79 | context['test'] = 'Test ' 80 | 81 | # generate the releases RSS 82 | template = PyPiPageTemplate('rss.xml', template_dir) 83 | content = template(**context) 84 | cache_redis.set('rss~main', content) 85 | 86 | # generate the packages RSS 87 | template = PyPiPageTemplate('packages-rss.xml', template_dir) 88 | content = template(**context) 89 | cache_redis.set('rss~pkgs', content) 90 | -------------------------------------------------------------------------------- /templates/about.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 10 |

About

11 |

12 | Welcome to pypi.python.org, the Python community's software hosting service. 13 |

14 | 15 |

16 | Here you'll find a Python package to suit almost any need. 17 |

18 | 19 |

20 | The project was started in 2002, 21 | and has since grown to include the contributions of over a dozen Python 22 | programmers, packages 23 | and millions of downloads. 24 |

25 | 26 |

27 | The Python Package Index (PyPI) is hosted by the Python Software Foundation (who run the python.org systems) but the project is community-run. Currently there's two maintainers, Richard Jones and Martin von Löwis. 28 | To contact the admins, please 29 | use the 30 | Get help 31 | or 32 | Bug reports 33 | links. 34 |

35 | 36 |

37 | The index itself is written in Python but pre-dates modern Python web frameworks (more detail). 38 |

39 | 40 |

41 | To submit a package use 42 | "python setup.py upload" 43 | and to use a package from this index either 44 | "pip install package" 45 | or download, unpack and "python setup.py install" it. 46 |

47 | 48 |

49 | Security: GPG and SSH ... XXX details? 50 |

51 | 52 |

You can access the data in PyPI from a program using either the per-package 53 | JSON data or the XML-RPC interface.

54 | 55 |

PyPI now provides a Mirror infrastructure to allow 56 | users to continue downloading file even if pypi.python.org is unavailable.

57 | 58 |

59 | PyPI has also been known as The Python Cheese Shop (and indeed cheeseshop.python.org still works). 60 |

61 | 62 |

And now for something completely different...

63 |

64 | Customer: Now then, some cheese please, my good man. 65 |
66 | Owner: (lustily) Certainly, sir. What would you like? 67 |
68 | Customer: Well, eh, how about a little red Leicester. 69 |
70 | Owner: I'm, a-fraid we're fresh out of red Leicester, sir. 71 |

72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /templates/browse.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | 19 | 20 |
14 | 15 | 17 | [unselect] 18 |
21 | 22 | 23 |
24 |
25 | 26 | 27 | 28 | 29 | 30 | 31 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 49 | 50 | 51 | 52 | 53 | 54 | 55 |
Packages
32 | 33 | There are packages in the categories you have selected. 34 | Please select more categories, or show all 35 | matching packages now. 36 | 37 | 38 | Please select a category from below. 39 | 40 |
PackageDescription
package namepackage summary
56 | 57 | 58 |
59 |

Category

60 | 61 | 64 | 65 |
66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /templates/confirm.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 |
9 | 10 | 11 | 12 |

13 | User name: 14 |

15 |
16 |

17 | By registering to upload materials to PyPI, I agree and 18 | affirmatively acknowledge the following: 19 |

20 |
    21 |
  • 22 |

    23 | "Content" means any data, content, code, video, images or 24 | other materials of any type that I upload, submit, or 25 | otherwise transmit to or through PyPI. Acceptable Content is 26 | restricted to Python packages and related information. 27 |

    28 |
    • 29 |
  • 30 |

    31 | Any Content uploaded to PyPI is provided on a non-confidential 32 | basis, and is not a trade secret. 33 |

    34 |
    • 35 |
  • 36 |

    37 | I retain all right, title, and interest in the Content (to the 38 | same extent I possessed such right, title and interest prior to 39 | uploading the Content), but by uploading I grant or warrant (as 40 | further set forth below) that the PSF is free to disseminate 41 | the Content, in the form provided to the PSF. Specifically, 42 | that means: 43 |

    44 |
      45 |
    • 46 |

      47 | If I upload Content covered by a royalty-free license 48 | included with such Content, giving the PSF the right to 49 | copy and redistribute such Content unmodified on PyPI as I 50 | have uploaded it, with no further action required by the 51 | PSF (an "Included License"), I represent and warrant that 52 | the uploaded Content meets all the requirements necessary 53 | for free redistribution by the PSF and any mirroring 54 | facility, public or private, under the Included License. 55 |

      56 |
      • 57 |
    • 58 |

      59 | If I upload Content other than under an Included License, 60 | then I grant the PSF and all other users of the web site an 61 | irrevocable, worldwide, royalty-free, nonexclusive license 62 | to reproduce, distribute, transmit, display, perform, and 63 | publish the Content, including in digital form. 64 |

      65 |
      • 66 |
    67 |
    • 68 |
  • 69 |

    70 | For the avoidance of doubt, my warranty or license as set forth 71 | above applies to the Content exactly in the form provided to 72 | the PSF, and does not, other than as may be provided in the 73 | Included License, grant the PSF or any users of the website a 74 | license to make or distribute derivative works based on my 75 | Content. 76 |

    77 |
    • 78 |
  • 79 |

    80 | I further warrant that I do not know of any applicable 81 | unlicensed patent or trademark rights that would be infringed 82 | by my uploaded Content. 83 |

    84 |
    • 85 |
86 |

87 | I represent and warrant that I have complied with all government 88 | regulations concerning the transfer or export of any Content I 89 | upload to PyPI. In particular, if I am subject to United States 90 | law, I represent and warrant that I have obtained any required 91 | governmental authorization for the export of the Content I upload. 92 | I further affirm that any Content I provide is not intended for use 93 | by a government end-user engaged in the manufacture or distribution 94 | of items or services controlled on the Wassenaar Munitions List as 95 | defined in part 772 of the United States Export Administration 96 | Regulations, or by any other embargoed user. 97 |

98 |
99 |

I agree

100 |

101 |
102 | 103 | 104 | -------------------------------------------------------------------------------- /templates/dialog.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 11 | 12 | 13 | 17 | 18 |

19 | 20 |

21 | 22 | 24 | 25 | 26 | 27 |
28 | 29 | 30 | -------------------------------------------------------------------------------- /templates/edit_form.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 |

9 | 10 |

12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 |
highlightedinformation is required
20 |
21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /templates/files.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 |

9 | 10 |

For all activities relating to Projects/Releases you must use 11 | 12 | the Warehouse test instance 13 | 14 | 15 | Warehouse 16 | 17 |

18 | 19 |

To upload files to the index, see The Python Packaging User Guide

20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /templates/home.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 |

The Python Package Index is a repository of software for the Python 9 | programming language. There are currently 10 | 12343 11 | packages here. 12 |
13 | To contact the PyPI admins, please use the 14 | Support 15 | or 16 | Bug reports 17 | links. 18 |

19 | 20 |
21 | 22 |
23 | Get Packages 24 |

25 | To use a package from this index either 26 | "pip install package" 27 | (get pip) 28 | or download, unpack and "python setup.py install" it. 29 |

30 |
31 | 32 |
33 | Package Authors 34 |

35 | Submit packages with 36 | "python setup.py upload". 37 | You can also use twine! 38 | The index hosts package docs. 39 | You must register. 40 | Testing? Use testpypi. 41 |

42 |
43 | 44 |
45 | Infrastructure 46 |

47 | To interoperate with the index 48 | use the 49 | JSON, 50 | XML-RPC or 51 | HTTP 52 | interfaces. 53 | Use local mirroring or caching to make installation more robust. 54 |

55 |
56 | 57 |
58 | 59 | 60 | 61 | 62 | 63 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 |
UpdatedPackageDescription
datelinksummary
79 | 80 |

And now for something completely different...

81 |

82 | Customer: Now then, some cheese please, my good man. 83 |
84 | Owner: (lustily) Certainly, sir. What would you like? 85 |
86 | Customer: Well, eh, how about a little red Leicester. 87 |
88 | Owner: I'm, a-fraid we're fresh out of red Leicester, sir. 89 |

90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /templates/index.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 |

10 | There were no matches.

11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 24 | 27 | 28 | 29 | 30 | 31 | 32 |
PackageWeight*Description
23 | link 26 | summary
33 | 34 |

*: occurrence of search term weighted by 35 | field (name, summary, keywords, description, author, maintainer)

36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /templates/login.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 |
8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
Username:
Password:
24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /templates/message.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 |

8 |

9 | 10 |

11 | 12 | 13 | -------------------------------------------------------------------------------- /templates/mirrors.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 |

The public PyPI mirrors using *.pypi.python.org DNS aliases have been 9 | deprecated as specified in 10 | PEP 449 (Removal of 11 | the PyPI Mirror Auto Discovery and Naming Scheme). Package 12 | files uploaded to PyPI are now served through a CDN which is more reliable. 13 | You are welcome to continue to run your own private mirrors, but the DNS 14 | aliases for the mirror network ([a..z].pypi.python.org) were shut down on 15 | February 15th, 2014.

16 | 17 |

If you'd like to use a mirror, resources like 18 | http://www.pypi-mirrors.org/ 19 | are available to find public mirrors of PyPI.

20 | 21 |

Additionally, the "mirror authenticity" API specified in the PEP below 22 | will be deprecated in the near future, and use of it is discouraged.

23 | 24 |

Use the bandersnatch 25 | mirroring client to set up your own mirror.

26 | 27 |

The mirroring protocol is defined in http://www.python.org/dev/peps/pep-0381

28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /templates/openid.pt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 |

Use the field below to enter your OpenID, to either login (if the ID is already known to us), or register with PyPI. Alternatively, select one of the icons to have your provider guide you in the identifier selection.

9 |
10 | Enter your OpenID: 11 | 12 |
13 |

If you would like to see additional provider icons added on the main page, please report that to the bug tracker. Notice that we have the following requirements for adding a new provider:

14 |
    15 |
  • must be in wide use, using procedures that the community trusts
    • 16 |
  • must support OpenID 2.0
    • 17 |
  • must support provider-driven identifier selection
    • 18 |
  • must support direct communication over https
    • 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /templates/packages-rss.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | <tal:x tal:replace="test" />PyPI Newest Packages 8 | 9 | Newest packages registered at the Python Package Index 10 | en 11 | 12 | 13 | 14 | <link tal:content="python:'http://pypi.python.org%s'%store.package_url(url_path, release['name'], None)" /> 15 | <guid tal:content="python:'http://pypi.python.org%s'%store.package_url(url_path, release['name'], None)" /> 16 | <description tal:content="release/summary" /> 17 | <pubDate tal:content="python:release['submitted_date'].strftime('%d %b %Y %H:%M:%S GMT')" /> 18 | </item> 19 | </channel> 20 | </rss> 21 | -------------------------------------------------------------------------------- /templates/password_reset.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | 8 | <p>Initiating password resets via site is no longer supported.</p> 9 | 10 | <p>Please reset your password using 11 | <tal:x tal:condition="python:'testpypi' in app.config.url"> 12 | <a href="https://test.pypi.org/account/request-password-reset/">the Warehouse test instance</a> 13 | </tal:x> 14 | <tal:x tal:condition="python:'testpypi' not in app.config.url"> 15 | <a href="https://pypi.org/account/request-password-reset/">Warehouse</a> 16 | </tal:x> 17 | in order to continue. 18 | </p> 19 | 20 | </metal:fill> 21 | </html> 22 | -------------------------------------------------------------------------------- /templates/password_reset_change.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | <p>Please enter your new account password below.</p> 8 | 9 | <p tal:condition="data/retry | nothing"> 10 | <strong tal:content="data/retry" /> 11 | </p> 12 | 13 | <form tal:attributes="action app/url_path" method="POST"> 14 | <input type="hidden" name=":action" value="pw_reset_change" /> 15 | <input type="hidden" name="otk" tal:attributes="value data/otk" /> 16 | <table class="form"> 17 | <tr> 18 | <th>Password:</th> 19 | <td><input type="password" name="password" /> 20 | </td> 21 | </tr> 22 | <tr> 23 | <th>Password again:</th> 24 | <td><input type="password" name="confirm" /></td> 25 | </tr> 26 | <tr> 27 | <td></td> 28 | <td><input type="submit" value="Set Password" /></td> 29 | </tr> 30 | </table> 31 | </form> 32 | <p>Please ensure your password is of a reasonable length (>8 characters) and 33 | mixes letters, cases and numbers. Please don't use the same password 34 | as with other services.</p> 35 | </metal:fill> 36 | </html> 37 | -------------------------------------------------------------------------------- /templates/pkg_edit.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | 8 | <p tal:replace="structure app/release_nav" /> 9 | 10 | <p> 11 | Each project may have a release for each version of the 12 | project that is released. You may use this form to hide releases 13 | from users. 14 | </p> 15 | 16 | <p>Administer the <a tal:attributes="href string:${app/url_path}?:action=role_form&package_name=${app/form/name}">Role</a> 17 | assigned to users for this project.</p> 18 | 19 | <p><b>For all activities relating to Project/Release changes except for hiding releases, you must use 20 | <tal:x tal:condition="python:'testpypi' in app.config.url"> 21 | <a href="https://test.pypi.org/manage/projects/">the Warehouse test instance</a> 22 | </tal:x> 23 | <tal:x tal:condition="python:'testpypi' not in app.config.url"> 24 | <a href="https://pypi.org/manage/projects/">Warehouse</a> 25 | </tal:x></b> 26 | </p> 27 | 28 | <form tal:attributes="action app/url_path" method="POST"> 29 | <input type="hidden" name=":action" value="pkg_edit" /> 30 | <input type="hidden" name="name" tal:attributes="value data/name" /> 31 | 32 | <table class="list" style="width: auto"> 33 | <tr><th>Version</th><th>Hide?</th><th>Summary</th> 34 | <th colspan="2">Links</th></tr> 35 | 36 | <tr tal:repeat="release data/releases" 37 | tal:attributes="class repeat/release/parity"> 38 | <td tal:content="release/version" /> 39 | <td> 40 | <select tal:attributes="name string:hid_${release/version}"> 41 | <div tal:omit-tag="" 42 | tal:condition="release/_pypi_hidden"> 43 | <option value="0">No</option> 44 | <option value="1" selected="selected">Yes</option> 45 | </div> 46 | <div tal:omit-tag="" 47 | tal:condition="not:release/_pypi_hidden"> 48 | <option value="0" selected="selected">No</option> 49 | <option value="1">Yes</option> 50 | </div> 51 | </select> 52 | </td> 53 | <td><input size="40" tal:attributes="name string:sum_${release/version}; 54 | value release/summary" /></td> 55 | <td><a tal:attributes="href string:${app/url_path}?:action=display&name=${data/name}&version=${release/version}">show</a></td> 56 | <td><a tal:attributes="href string:${app/url_path}?:action=files&name=${data/name}&version=${release/version}">files</a></td> 57 | </tr> 58 | 59 | <tr> 60 | <td id="last"> </td> 61 | <td id="last" colspan="5"> 62 | <input type="submit" name="submit_submit" value="Update Releases" /> 63 | </td> 64 | </tr> 65 | </table> 66 | </form> 67 | 68 | <p>By default, each new release will hide all other release from the 69 | regular display. If you want to select yourself which projects to hide, 70 | uncheck the button below</p> 71 | <form tal:attributes="action app/url_path" method="POST" 72 | enctype="multipart/form-data"> 73 | <input type="hidden" name=":action" value="pkg_edit" /> 74 | <input type="hidden" name="name" tal:attributes="value data/name" /> 75 | 76 | <!-- Apparently, we run TAL in XML mode, so the auto-remove feature 77 | for the checked attribute will not work --> 78 | <input tal:condition="data/autohide" type="checkbox" name="autohide" 79 | checked="checked">Auto-hide old releases</input> 80 | <input tal:condition="not:data/autohide" type="checkbox" name="autohide">Auto-hide old releases</input> 81 | <input type="submit" name="submit_autohide" value="Change" /> 82 | <br/> 83 | </form> 84 | 85 | 86 | <p>If you would like to <b>DESTROY</b> any existing documentation hosted at 87 | <a tal:attributes="href string:http://pythonhosted.org/${data/name}" 88 | tal:content="string:http://pythonhosted.org/${data/name}"></a>: Please use 89 | <tal:x tal:condition="python:'testpypi' in app.config.url"> 90 | <a href="https://test.pypi.org/manage/projects/">the Warehouse test instance</a> 91 | </tal:x> 92 | <tal:x tal:condition="python:'testpypi' not in app.config.url"> 93 | <a href="https://pypi.org/manage/projects/">Warehouse</a> 94 | </tal:x> 95 | </p> 96 | <p></p> 97 | 98 | </metal:fill> 99 | </html> 100 | -------------------------------------------------------------------------------- /templates/print_the_world.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns="http://www.w3.org/1999/xhtml" 4 | xmlns:tal="http://xml.zope.org/namespaces/tal" 5 | xmlns:metal="http://xml.zope.org/namespaces/metal" 6 | metal:use-macro="standard_template/macros/page"> 7 | <metal:slot fill-slot="body"> 8 | 9 | <p> 10 | The list of all packages previously hosted at this URL has been deprecated, 11 | the resources necessary to render this page have become untenable as PyPI has 12 | grown. 13 | </p> 14 | 15 | <p> 16 | If you previously used this list for some purpose, you may consider using the 17 | simple index itself as a replacement. 18 | </p> 19 | 20 | </metal:slot> 21 | </html> 22 | -------------------------------------------------------------------------------- /templates/register.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | 8 | <p tal:condition="not:data/new_user"> 9 | This form allows to modify your PyPI account information, and to delete your account. 10 | <form tal:attributes="action app/url_path" method="POST" 11 | tal:condition="not:data/owns_packages"> 12 | <input type="hidden" name=":action" value="delete_user" /> 13 | <input type="submit" value="Delete account"/> 14 | </form> 15 | <tal:block tal:condition="data/owns_packages">You cannot delete your account since you 16 | are still listed as owner of some packages.</tal:block> 17 | </p> 18 | <form tal:attributes="action app/url_path" method="POST"> 19 | <input type="hidden" name=":action" value="user" /> 20 | <input type="hidden" name="name" tal:condition="not:data/new_user" 21 | tal:attributes="value data/name"/> 22 | <tal:block tal:repeat="field data/openid_fields"> 23 | <input type="hidden" tal:attributes="name python:field[0];value python:field[1].decode('utf-8')"/> 24 | </tal:block> 25 | <table class="form"> 26 | <tr> 27 | <th>Username:</th> 28 | <td tal:condition="data/new_user"><input name="name" tal:attributes="value data/name"/></td> 29 | <td tal:condition="not:data/new_user" tal:content="data/name"></td> 30 | </tr> 31 | <tal:block tal:condition="not:data/openid_fields"> 32 | <tr> 33 | <th>Password:</th> 34 | <td><input type="password" name="password"/></td> 35 | </tr> 36 | <tr> 37 | <th>Confirm:</th> 38 | <td><input type="password" name="confirm" /></td> 39 | </tr> 40 | </tal:block> 41 | <tr tal:condition="data/openid"> 42 | <th>OpenID:</th> 43 | <td tal:content="data/openid"/> 44 | </tr> 45 | <tr> 46 | <th>Email Address:</th> 47 | <td><input name="email" tal:attributes="value data/email" /></td> 48 | </tr> 49 | <tr> 50 | <th></th> 51 | <td><input type="submit" tal:attributes="value data/action" /></td> 52 | </tr> 53 | </table> 54 | </form> 55 | 56 | <p>Please ensure your password is of a reasonable length (>8 characters) and 57 | mixes letters, cases and numbers. Please don't use the same password 58 | as with other services.</p> 59 | 60 | <tal:block tal:condition="data/new_user"> 61 | <p><b>A confirmation email will be sent to the address you nominate above.</b></p> 62 | <p>Please ensure you will be able to receive email from admin@mail.pypi.python.org 63 | (check any "sender confirmation" systems you might be using.)</p> 64 | <p>To complete the registration process, you must visit the link indicated in the 65 | email.</p> 66 | </tal:block> 67 | 68 | <tal:block tal:condition="not:data/new_user"> 69 | <form tal:attributes="action app/url_path" method="POST" tal:repeat="key data/sshkeys"> 70 | <input type="hidden" name=":action" value="delkey" /> 71 | <input type="hidden" name="id" tal:attributes="value key/id" /> 72 | <textarea name="key" cols="50" rows="4" readonly="1" tal:content="key/key"/> 73 | <input type="submit" value="Delete this key"/> 74 | </form> 75 | </tal:block> 76 | 77 | <tal:block tal:condition="data/openids"> 78 | <p>Your OpenIDs:</p> 79 | <ul> 80 | <li tal:repeat="curid data/openids"><span tal:replace="curid/id"/> 81 | <form tal:attributes="action app/url_path" method="POST"> 82 | <input type="hidden" name=":action" value="dropid"/> 83 | <input type="hidden" name="openid" tal:attributes="value curid/id"/> 84 | <input type="submit" value="Drop this ID"/> 85 | </form> 86 | </li> 87 | </ul> 88 | </tal:block> 89 | 90 | <p tal:condition="not:data/new_user">Claim OpenID, either from one of these providers: 91 | <tal:block tal:repeat="prov data/providers"> 92 | <a style="border: none;" tal:attributes="href prov/claim"><img width="16" height="16" tal:attributes="src prov/favicon; title prov/title; alt prov/title"/></a> 93 | </tal:block> 94 | or enter your ID explicitly: 95 | <form action="/openid_claim" method='POST'> 96 | <input name='openid_identifier' size='60'/> 97 | <input type='submit' value='Claim'/> 98 | </form> 99 | </p> 100 | </metal:fill> 101 | </html> 102 | -------------------------------------------------------------------------------- /templates/register_gone.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | 8 | <p>Registration via this site is no longer supported.</p> 9 | 10 | <p>Please register via 11 | <tal:x tal:condition="python:'testpypi' in app.config.url"> 12 | <a href="https://test.pypi.org/account/register/">the Warehouse test instance</a> 13 | </tal:x> 14 | <tal:x tal:condition="python:'testpypi' not in app.config.url"> 15 | <a href="https://pypi.org/account/register/">Warehouse</a> 16 | </tal:x> 17 | in order to continue. 18 | </p> 19 | 20 | <p>This change is occuring a bit earlier than we hoped due to ongoing concerns regarding spammers attacking PyPI.</p> 21 | 22 | </metal:fill> 23 | </html> 24 | -------------------------------------------------------------------------------- /templates/role_form.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | 8 | <p> 9 | You must use 10 | <tal:x tal:condition="python:'testpypi' in app.config.url"> 11 | <a href="https://test.pypi.org/manage/projects/">the Warehouse test instance</a> 12 | </tal:x> 13 | <tal:x tal:condition="python:'testpypi' not in app.config.url"> 14 | <a href="https://pypi.org/manage/projects/">Warehouse</a> 15 | </tal:x> 16 | to manage roles now. 17 | </p> 18 | 19 | </metal:fill> 20 | </html> 21 | -------------------------------------------------------------------------------- /templates/rss.xml: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="UTF-8"?> 2 | <!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" "http://my.netscape.com/publish/formats/rss-0.91.dtd"> 3 | <rss version="0.91" 4 | xmlns:tal="http://xml.zope.org/namespaces/tal" 5 | xmlns:metal="http://xml.zope.org/namespaces/metal"> 6 | <channel> 7 | <title><tal:x tal:replace="test" />PyPI Recent Updates 8 | 9 | Recent updates to the Python Package Index 10 | en 11 | 12 | 13 | 14 | <link tal:content="python:'http://pypi.python.org%s'%store.package_url( 15 | url_path, release['name'], release['version'])" /> 16 | <description tal:content="release/summary" /> 17 | <pubDate tal:content="python:release['submitted_date'].strftime('%d %b %Y %H:%M:%S GMT')" /> 18 | </item> 19 | </channel> 20 | </rss> 21 | -------------------------------------------------------------------------------- /templates/rss1hour.xml: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="UTF-8"?> 2 | <!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" "http://my.netscape.com/publish/formats/rss-0.91.dtd"> 3 | <rss version="0.91" 4 | xmlns:tal="http://xml.zope.org/namespaces/tal" 5 | xmlns:metal="http://xml.zope.org/namespaces/metal"> 6 | <channel> 7 | <title>PyPI changes 8 | 9 | Updates to the Python Package Index in the last hour 10 | en 11 | 12 | 13 | 14 | <link tal:content="python:'http://pypi.python.org%s'%app.packageURL( 15 | change['name'], change['version'])" /> 16 | <description tal:content="change/action" /> 17 | <pubDate tal:content="python:change['submitted_date'].strftime('%d %b %Y %H:%M:%S GMT')" /> 18 | </item> 19 | </channel> 20 | </rss> 21 | -------------------------------------------------------------------------------- /templates/security.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | 8 | <h2>Reporting</h2> 9 | 10 | <p>If you have a query or report to make regarding security please contact 11 | Donald Stufft and/or Ernest W. Durbin III. Both have GPG keys on key 12 | servers like pgp.mit.edu.</p> 13 | 14 | 15 | <p>Donald's GPG key has key id 0x6E3CBCE93372DCFA (full fingerprint 7C6B 7C5D 5E2B 6356 A926 F04F 6E3C BCE9 3372 DCFA) and his email address is donald@python.org</p> 16 | 17 | <p>Ernest's GPG key has key id 0x88159C24830F6F7E (full fingerprint 11CD 3DD9 8D7E 61C7 6D1A 3224 8815 9C24 830F 6F7E) and his email address is ernest@python.org</p> 18 | 19 | <p>You may also report issues in the <a href="https://sourceforge.net/tracker/?func=add&group_id=66150&atid=513503">PyPI bug tracker</a> where reports may be made private.</p> 20 | 21 | <h2>Your Security</h2> 22 | 23 | <p>You may sign your uploads with GPG using the "--sign" argument to "python setup.py upload".</p> 24 | 25 | <p>The file checksums provided with files on PyPI exists <b>only</b> to provide some download corruption protection. It is <b>not</b> intended to provide any sort of security regarding tampering. Please use GPG signing and verification via your <a href="https://en.wikipedia.org/wiki/Web_of_trust">Web of Trust</a> for that.</p> 26 | 27 | </metal:fill> 28 | </html> 29 | 30 | -------------------------------------------------------------------------------- /templates/tos.pt: -------------------------------------------------------------------------------- 1 | <?xml version="1.0" encoding="utf-8"?> 2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 3 | <html xmlns:tal="http://xml.zope.org/namespaces/tal" 4 | xmlns:metal="http://xml.zope.org/namespaces/metal" 5 | metal:use-macro="standard_template/macros/page"> 6 | <metal:fill fill-slot="body"> 7 | <div> 8 | <p> 9 | By registering to upload materials to PyPI, I agree and 10 | affirmatively acknowledge the following: 11 | </p> 12 | <ul> 13 | <li> 14 | <p> 15 | "Content" means any data, content, code, video, images or 16 | other materials of any type that I upload, submit, or 17 | otherwise transmit to or through PyPI. Acceptable Content is 18 | restricted to Python packages and related information. 19 | </p> 20 | </li> 21 | <li> 22 | <p> 23 | Any Content uploaded to PyPI is provided on a non-confidential 24 | basis, and is not a trade secret. 25 | </p> 26 | </li> 27 | <li> 28 | <p> 29 | I retain all right, title, and interest in the Content (to the 30 | same extent I possessed such right, title and interest prior to 31 | uploading the Content), but by uploading I grant or warrant (as 32 | further set forth below) that the PSF is free to disseminate 33 | the Content, in the form provided to the PSF. Specifically, 34 | that means: 35 | </p> 36 | <ul> 37 | <li> 38 | <p> 39 | If I upload Content covered by a royalty-free license 40 | included with such Content, giving the PSF the right to 41 | copy and redistribute such Content unmodified on PyPI as I 42 | have uploaded it, with no further action required by the 43 | PSF (an "Included License"), I represent and warrant that 44 | the uploaded Content meets all the requirements necessary 45 | for free redistribution by the PSF and any mirroring 46 | facility, public or private, under the Included License. 47 | </p> 48 | </li> 49 | <li> 50 | <p> 51 | If I upload Content other than under an Included License, 52 | then I grant the PSF and all other users of the web site an 53 | irrevocable, worldwide, royalty-free, nonexclusive license 54 | to reproduce, distribute, transmit, display, perform, and 55 | publish the Content, including in digital form. 56 | </p> 57 | </li> 58 | </ul> 59 | </li> 60 | <li> 61 | <p> 62 | For the avoidance of doubt, my warranty or license as set forth 63 | above applies to the Content exactly in the form provided to 64 | the PSF, and does not, other than as may be provided in the 65 | Included License, grant the PSF or any users of the website a 66 | license to make or distribute derivative works based on my 67 | Content. 68 | </p> 69 | </li> 70 | <li> 71 | <p> 72 | I further warrant that I do not know of any applicable 73 | unlicensed patent or trademark rights that would be infringed 74 | by my uploaded Content. 75 | </p> 76 | </li> 77 | </ul> 78 | <p> 79 | I represent and warrant that I have complied with all government 80 | regulations concerning the transfer or export of any Content I 81 | upload to PyPI. In particular, if I am subject to United States 82 | law, I represent and warrant that I have obtained any required 83 | governmental authorization for the export of the Content I upload. 84 | I further affirm that any Content I provide is not intended for use 85 | by a government end-user engaged in the manufacture or distribution 86 | of items or services controlled on the Wassenaar Munitions List as 87 | defined in part 772 of the United States Export Administration 88 | Regulations, or by any other embargoed user. 89 | </p> 90 | </div> 91 | </metal:fill> 92 | </html> 93 | 94 | -------------------------------------------------------------------------------- /tests/rpc-test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import config 3 | import store 4 | import unittest 5 | import rpc 6 | import cStringIO 7 | 8 | class XMLRPC ( unittest.TestCase ): 9 | 10 | def setUp( self ): 11 | # get a storage object to use in calls to xmlrpc functions 12 | self.store = store.Store( config.Config( "/tmp/pypi/config.ini" ) ) 13 | 14 | def testEcho( self ): 15 | result = rpc.echo(self.store, 'a', 'b', 1, 3.4) 16 | self.failUnlessEqual(result, ('a', 'b', 1, 3.4)) 17 | 18 | def testIndex( self ): 19 | result = rpc.index( self.store ) 20 | self.failUnless( len( result ) > 0 ) 21 | 22 | def testSearch( self ): 23 | result = rpc.search( self.store, "sql" ) 24 | self.failUnless( len( result ) > 0 ) 25 | 26 | if __name__ == '__main__': 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /thfcgi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/thfcgi.py -------------------------------------------------------------------------------- /tools/apache_count.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys, os, re, psycopg, ConfigParser, urlparse, gzip, bz2 4 | from mx.DateTime import DateTime 5 | from mx.DateTime.Timezone import utc_offset 6 | 7 | logre=re.compile(r"\[(?P<day>..)/(?P<month>...)/(?P<year>....):" 8 | r"(?P<hour>..):(?P<min>..):(?P<sec>..) " 9 | r'(?P<zone>.*)\] "GET (?P<path>[^ "]+) HTTP/1.." 200') 10 | 11 | month_names=['jan','feb','mar','apr','may','jun', 12 | 'jul','aug','sep','oct','nov','dec'] 13 | month_index = {} 14 | for i in range(12): 15 | month_index[month_names[i]] = i+1 16 | 17 | def main(argv): 18 | if len(argv) != 3: 19 | print "Usage: apache_count.py configfile logfile" 20 | raise SystemExit 21 | # Read config file 22 | p = ConfigParser.ConfigParser() 23 | p.read(argv[1]) 24 | # Read server-relative URI prefix 25 | files_url = urlparse.urlsplit(p.get('webui', 'files_url'))[2] 26 | # Setup database connection 27 | dbname = p.get('database', 'name') 28 | dbuser = p.get('database', 'user') 29 | dbpass = p.get('database', 'password') 30 | dbconn = psycopg.connect(database=dbname, user=dbuser, password=dbpass) 31 | cursor = dbconn.cursor() 32 | 33 | filename = argv[2] 34 | if filename.endswith(".gz"): 35 | f = gzip.open(filename) 36 | elif filename.endswith(".bz2"): 37 | f = bz2.BZ2File(filename) 38 | else: 39 | f = open(filename) 40 | 41 | cursor.execute("select value from timestamps where name='http'") 42 | last_http = cursor.fetchone()[0] 43 | 44 | downloads = {} 45 | for line in f: 46 | m = logre.search(line) 47 | if not m: 48 | continue 49 | path = m.group('path') 50 | if not path.startswith(files_url): 51 | continue 52 | day = int(m.group('day')) 53 | month = m.group('month').lower() 54 | month = month_index[month] 55 | year = int(m.group('year')) 56 | hour = int(m.group('hour')) 57 | minute = int(m.group('min')) 58 | sec = int(m.group('sec')) 59 | date = DateTime(year, month, day, hour, minute, sec) 60 | zone = utc_offset(m.group('zone')) 61 | date = date - zone 62 | 63 | if date < last_http: 64 | continue 65 | 66 | filename = os.path.basename(path) 67 | # see if we have already read the old download count 68 | if not downloads.has_key(filename): 69 | cursor.execute("select downloads from release_files " 70 | "where filename=%s", (filename,)) 71 | record = cursor.fetchone() 72 | if not record: 73 | # No file entry. Could be a .sig file 74 | continue 75 | # make sure we're working with a number 76 | downloads[filename] = record[0] or 0 77 | # add a download 78 | downloads[filename] += 1 79 | 80 | if not downloads: 81 | return 82 | 83 | # Update the download counts 84 | for filename, count in downloads.items(): 85 | cursor.execute("update release_files set downloads=%s " 86 | "where filename=%s", (count, filename)) 87 | # Update the download timestamp 88 | date = psycopg.TimestampFromMx(date) 89 | cursor.execute("update timestamps set value=%s " 90 | "where name='http'", (date,)) 91 | dbconn.commit() 92 | 93 | if __name__=='__main__': 94 | main(sys.argv) 95 | -------------------------------------------------------------------------------- /tools/apache_reader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reads apache log files 3 | """ 4 | import bz2 5 | import gzip 6 | import re 7 | import os 8 | 9 | # list of recognized user agents 10 | SETUPTOOLS_UA = (re.compile((r'^.* setuptools/(?P<version>[0-9]\..*)$')), 'setuptools/%s') 11 | URLLIB_UA = (re.compile(r'^Python-urllib/(?P<version>[23]\.[0-9])$'), 'Python-urllib/%s') 12 | SAFARI_UA = (re.compile(r'^Mozilla.* .* Version/(?P<version>.*) Safari/.*$'), 'Safari/%s') 13 | GOOGLEBOT = (re.compile(r'Googlebot-Mobile/(?P<version>.*);'), 'Googlebot-Mobile/%s') 14 | MSNBOT = (re.compile(r'^msnbot/(?P<version>.*) '), 'msnbot/%s') 15 | FIREFOX_UA = (re.compile(r'^Mozilla.*? Firefox/(?P<version>[23])\..*$'), 'Firefox/%s') 16 | PLAIN_MOZILLA = (re.compile(r'^Mozilla/(?P<version>.*?) '), 'Mozilla/%s') 17 | 18 | logre = re.compile(r"\[(?P<day>..)/(?P<month>...)/(?P<year>....):" 19 | r"(?P<hour>..):(?P<min>..):(?P<sec>..) " 20 | r'(?P<zone>.*)\] "GET (?P<path>[^ "]+) HTTP/1.." 200 .*? (?:".*?")? ' 21 | r'"(User-Agent: )?(?P<useragent>.*)"$', re.DOTALL) 22 | 23 | month_names=['jan','feb','mar','apr','may','jun', 24 | 'jul','aug','sep','oct','nov','dec'] 25 | month_index = {} 26 | 27 | for i in range(12): 28 | month_index[month_names[i]] = i+1 29 | 30 | def month_to_index(month): 31 | return month_index[month.lower()] 32 | 33 | class ApacheLogReader(object): 34 | """provides an iterator over apache logs""" 35 | 36 | def __init__(self, filename, files_url='', mode=None): 37 | if mode is None: 38 | ext = os.path.splitext(filename)[-1] 39 | if ext in ('.bz2', '.gz'): 40 | mode = 'r:%s' % ext[1:] 41 | else: 42 | mode = 'r' 43 | if ':' in mode: 44 | mode, compr = mode.split(':') 45 | else: 46 | mode, compr = mode, None 47 | if compr not in ('bz2', 'gz', None): 48 | raise ValueError('%s mode not supported' % compr) 49 | if compr == 'bz2': 50 | self._data = bz2.BZ2File(filename, mode) 51 | elif compr == 'gz': 52 | self._data = gzip.open(filename) 53 | else: 54 | self._data = open(filename, mode) 55 | 56 | self.files_url = files_url 57 | 58 | def __iter__(self): 59 | return self 60 | 61 | def package_name(self, path): 62 | path = [p for p in path.split('/') if p != ''] 63 | return path[-2] 64 | 65 | def get_simplified_ua(self, user_agent): 66 | """returns a simplified version of the user agent""" 67 | for expr, repl in (URLLIB_UA, SETUPTOOLS_UA, SAFARI_UA, GOOGLEBOT, 68 | MSNBOT, FIREFOX_UA, PLAIN_MOZILLA): 69 | res = expr.search(user_agent) 70 | if res is not None: 71 | return repl % res.group('version') 72 | return user_agent 73 | 74 | def next(self): 75 | 76 | while True: 77 | line = self._data.next().strip() 78 | m = logre.search(line) 79 | if m is None: 80 | continue 81 | path = m.group('path') 82 | filename = os.path.basename(path) 83 | filename = filename.split('?')[0] 84 | if not path.startswith(self.files_url) or filename == '': 85 | continue 86 | res = m.groupdict() 87 | res['month'] = month_to_index(res['month']) 88 | res['useragent'] = self.get_simplified_ua(res['useragent']) 89 | res['filename'] = filename 90 | res['packagename'] = self.package_name(path) 91 | res['day'] = int(res['day']) 92 | res['year'] = int(res['year']) 93 | res['hour'] = int(res['hour']) 94 | res['minute'] = int(res['min']) 95 | return res 96 | 97 | raise StopIteration 98 | 99 | -------------------------------------------------------------------------------- /tools/daily.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import os 4 | 5 | prefix = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 6 | sys.path.insert(0, prefix) 7 | 8 | # Workaround current bug in docutils: 9 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 10 | import docutils.utils 11 | 12 | import config 13 | import store 14 | 15 | CONFIG_FILE = os.environ.get("PYPI_CONFIG", os.path.join(prefix, 'config.ini')) 16 | 17 | conf = config.Config(CONFIG_FILE) 18 | store = store.Store(conf) 19 | 20 | cursor = store.get_cursor() 21 | 22 | cursor.execute("delete from cookies where last_seen < now()-INTERVAL'1day';") 23 | cursor.execute("delete from openid_sessions where expires < now();") 24 | cursor.execute("delete from openid_nonces where created < now()-INTERVAL'1day'; ") 25 | cursor.execute("delete from openids where name in (select name from rego_otk where date < now()-INTERVAL'7days');") 26 | cursor.execute("delete from accounts_email where user_id in (select id from accounts_user where username in (select name from rego_otk where date < now()-INTERVAL'7days' and name not in (select user_name from roles)));") 27 | cursor.execute("delete from accounts_user where username in (select name from rego_otk where date < now()-INTERVAL'7days' and name not in (select user_name from roles));") 28 | 29 | store.commit() 30 | -------------------------------------------------------------------------------- /tools/daily.sql: -------------------------------------------------------------------------------- 1 | -- run as a cronjob: psql packages -f tools/daily.sql -o /dev/null 2 | delete from cookies where last_seen < now()-INTERVAL'1day'; 3 | delete from openid_sessions where expires < now(); 4 | delete from openid_nonces where created < now()-INTERVAL'1day'; 5 | delete from openids where name in (select name from rego_otk where date < now()-INTERVAL'7days'); 6 | delete from users where name in (select name from rego_otk where date < now()-INTERVAL'7days' and name not in (select user_name from roles)); 7 | -------------------------------------------------------------------------------- /tools/demodata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys, os, urllib 3 | 4 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 5 | sys.path.append(root) 6 | # Work around http://sourceforge.net/p/docutils/bugs/214/ 7 | import docutils.utils 8 | import admin, store, config 9 | 10 | cfg = config.Config(root+'/config.ini') 11 | st = store.Store(cfg) 12 | 13 | # Demo data starts here 14 | 15 | # an admin 16 | otk = st.store_user('fred', 'fredpw', 'fred@python.test') 17 | st.delete_otk(otk) 18 | st.add_role('fred', 'Admin', None) 19 | # an owner 20 | otk = st.store_user('barney', 'barneypw', 'barney@python.test') 21 | st.delete_otk(otk) 22 | 23 | # package spam 24 | st.set_user('barney', '127.0.0.1', True) 25 | for version in ('0.8', '0.9', '1.0'): 26 | st.store_package('spam', version, { 27 | 'author':'Barney Geroellheimer', 28 | 'author_email':'barney@python.test', 29 | 'homepage':'http://spam.python.test/', 30 | 'license':'GPL', 31 | 'summary':'The spam package', 32 | 'description': 'spam '*500, 33 | 'classifiers':["Development Status :: 6 - Mature", 34 | "Programming Language :: Python :: 2"], 35 | '_pypi_hidden':False 36 | }) 37 | 38 | # package eggs 39 | for version in ('0.1', '0.2', '0.3', '0.4'): 40 | st.store_package('eggs', version, { 41 | 'author':'Barney Geroellheimer', 42 | 'author_email':'barney@python.test', 43 | 'homepage':'http://eggs.python.test/', 44 | 'license':'GPL', 45 | 'summary':'The eggs package', 46 | 'description':'Does anybody want to provide real data here?', 47 | 'classifiers':["Development Status :: 3 - Alpha", 48 | "Programming Language :: Python :: 3"], 49 | 'requires_dist':['spam'], 50 | '_pypi_hidden':version!='0.4' 51 | }) 52 | 53 | st.commit() 54 | -------------------------------------------------------------------------------- /tools/downloadstats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Generates download stats for all days in the given log files, 3 | # except for the oldest and the newest day. 4 | import sys, os, csv 5 | import apache_reader, apache_stats 6 | 7 | statsdir = '/data/www/pypi/local-stats/' 8 | 9 | days = set() 10 | records = [] 11 | for fn in sys.argv[1:]: 12 | for record in apache_reader.ApacheLogReader(fn, '/packages'): 13 | days.add((record['year'], record['month'], record['day'])) 14 | records.append(record) 15 | 16 | days = sorted(days)[1:-1] 17 | 18 | class Stats(apache_stats.LocalStats): 19 | def _get_logs(self, logfile, files_url): 20 | return records 21 | stats = Stats() 22 | for year,month,day in days: 23 | stats.build_local_stats(year, month, day, None, statsdir+'days') 24 | 25 | -------------------------------------------------------------------------------- /tools/dumpstats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import csv 3 | import psycopg2 4 | import bz2 5 | import ConfigParser 6 | 7 | 8 | def main(config, out): 9 | # Setup database connection 10 | c = ConfigParser.ConfigParser({'user': '', 'password': ''}) 11 | c.read(config) 12 | dbname = c.get('database', 'name') 13 | dbuser = c.get('database', 'user') 14 | dbpass = c.get('database', 'password') 15 | dbhost = c.get('database', 'host') 16 | dbconn = psycopg2.connect(database=dbname, user=dbuser, password=dbpass, 17 | host=dbhost) 18 | cursor = dbconn.cursor() 19 | cursor.execute('select name,version,filename,downloads from release_files') 20 | with bz2.BZ2File(out, 'w') as f: 21 | w = csv.writer(f) 22 | w.writerows(cursor.fetchall()) 23 | dbconn.close() 24 | 25 | if __name__ == '__main__': 26 | import sys 27 | main(sys.argv[1], sys.argv[2]) 28 | -------------------------------------------------------------------------------- /tools/duplicate_users.py: -------------------------------------------------------------------------------- 1 | #!usr/bin/env python 2 | import os 3 | import sys 4 | import itertools 5 | 6 | # Workaround current bug in docutils: 7 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 8 | import docutils.utils 9 | 10 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 11 | sys.path = [root] + sys.path 12 | 13 | import store 14 | import config 15 | 16 | 17 | c = config.Config("config.ini") 18 | store = store.Store(c) 19 | store.open() 20 | cursor = store.get_cursor() 21 | 22 | cursor.execute("SELECT LOWER(name) FROM users GROUP BY LOWER(name) HAVING COUNT(*) > 1") 23 | duplicated = set([x[0] for x in cursor.fetchall()]) 24 | 25 | duplicates = {} 26 | users = {} 27 | 28 | for username in duplicated: 29 | cursor.execute("SELECT name, email, last_login FROM users WHERE LOWER(name)=LOWER(%s)", (username,)) 30 | dups = cursor.fetchall() 31 | 32 | duplicates[username] = [x[0] for x in dups] 33 | 34 | for x in dups: 35 | users[x[0]] = x 36 | 37 | print len(users), "duplicated users found with", len(duplicated), "total ci unique" 38 | 39 | total_users = users.values() 40 | total_names = set(x[0] for x in total_users) 41 | 42 | delete = set(total_names) 43 | 44 | # Exclude any user who has ever submitted a journal from deletion 45 | cursor.execute("SELECT DISTINCT ON (submitted_by) submitted_by FROM journals") 46 | journaled = set(x[0] for x in cursor.fetchall()) 47 | delete -= journaled 48 | 49 | # Exclude any user who is assigned a role on a package 50 | cursor.execute("SELECT DISTINCT ON (user_name) user_name FROM roles") 51 | roles = set(x[0] for x in cursor.fetchall()) 52 | delete -= roles 53 | 54 | # Exclude any user who has logged in 55 | cursor.execute("SELECT DISTINCT ON (name) name FROM users WHERE last_login != NULL") 56 | logged_in = set(x[0] for x in cursor.fetchall()) 57 | delete -= logged_in 58 | 59 | if delete: 60 | cursor.execute("DELETE FROM users WHERE name in %s", (tuple(delete),)) 61 | 62 | store.commit() 63 | store.close() 64 | -------------------------------------------------------------------------------- /tools/email_renamed_users.py: -------------------------------------------------------------------------------- 1 | import smtplib 2 | import pickle 3 | import sys 4 | import os 5 | 6 | from email.mime.text import MIMEText 7 | 8 | # Workaround current bug in docutils: 9 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 10 | import docutils.utils 11 | 12 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path = [root] + sys.path 14 | 15 | import config 16 | import store 17 | 18 | config = config.Config("config.ini") 19 | store = store.Store(config) 20 | 21 | EMAIL_PLURAL = """ 22 | Hello there! 23 | 24 | PyPI has begun to enforce restrictions on what a valid Python package name 25 | contains. 26 | 27 | These rules are: 28 | 29 | * Must contain ONLY ASCII letters, digits, underscores, hyphens, and periods 30 | * Must begin and end with an ASCII letter or digit 31 | 32 | You are listed as an owner or maintainer on %(old)s. 33 | 34 | Due to the new rules these packages will be renamed to %(new)s. 35 | 36 | These new names represent what someone using pip or easy_install would already 37 | have had to use in order to install your packages. 38 | 39 | I am sorry for any inconvenience this may have caused you. 40 | """ 41 | 42 | 43 | EMAIL_SINGLE = """ 44 | Hello there! 45 | 46 | PyPI has begun to enforce restrictions on what a valid Python package name 47 | contains. 48 | 49 | These rules are: 50 | 51 | * Must contain ONLY ASCII letters, digits, underscores, hyphens, and periods 52 | * Must begin and end with an ASCII letter or digit 53 | 54 | You are listed as an owner or maintainer on "%(old)s". 55 | 56 | Due to the new rules this package will be renamed to "%(new)s". 57 | 58 | This new name represents what someone using pip or easy_install would 59 | already have had to use in order to install your package. 60 | 61 | I am sorry for any inconvenience this may have caused you. 62 | """ 63 | 64 | with open("renamed.pkl") as pkl: 65 | renamed = pickle.load(pkl) 66 | 67 | 68 | # Build up a list of all users to email 69 | users = {} 70 | for old, new in renamed: 71 | for role in store.get_package_roles(new): 72 | user_packages = users.setdefault(role["user_name"], []) 73 | user_packages.append((old, new)) 74 | 75 | sent = [] 76 | 77 | # Email each user 78 | server = smtplib.SMTP(config.mailgun_hostname) 79 | if config.smtp_starttls: 80 | server.starttls() 81 | if config.smtp_auth: 82 | server.login(config.smtp_login, config.smtp_password) 83 | for username, packages in users.iteritems(): 84 | packages = sorted(set(packages)) 85 | 86 | user = store.get_user(username) 87 | 88 | if not user["email"]: 89 | continue 90 | 91 | if len(packages) > 1: 92 | msg = MIMEText(EMAIL_PLURAL % { 93 | "old": ", ".join(['"%s"' % x[0] for x in packages]), 94 | "new": ", ".join(['"%s"' % x[1] for x in packages]), 95 | }) 96 | elif packages: 97 | msg = MIMEText(EMAIL_SINGLE % { 98 | "old": packages[0][0], 99 | "new": packages[0][1], 100 | }) 101 | 102 | msg["Subject"] = "Important notice about your PyPI packages" 103 | msg["From"] = "donald@python.org" 104 | msg["To"] = user["email"] 105 | 106 | server.sendmail("donald@python.org", [user["email"]], msg.as_string()) 107 | sent.append(("donald@python.org", [user["email"]], msg.as_string())) 108 | server.quit() 109 | 110 | with open("sent.pkl", "w") as pkl: 111 | pickle.dump(sent, pkl) 112 | -------------------------------------------------------------------------------- /tools/index-trove.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import os 4 | import time 5 | 6 | prefix = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 7 | sys.path.insert(0, prefix) 8 | 9 | # Workaround current bug in docutils: 10 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 11 | import docutils.utils 12 | 13 | from psycopg2.extras import RealDictCursor 14 | import json 15 | import requests 16 | 17 | import config 18 | import store 19 | 20 | CONFIG_FILE = os.environ.get("PYPI_CONFIG", os.path.join(prefix, 'config.ini')) 21 | 22 | conf = config.Config(CONFIG_FILE) 23 | 24 | if conf.database_releases_index_name is None or conf.database_releases_index_url is None: 25 | sys.exit() 26 | 27 | new_index = "trove-%s-%s" % (conf.database_releases_index_name, int(time.time())) 28 | print("creating new index %s" % (new_index,)) 29 | store = store.Store(conf) 30 | store.open() 31 | 32 | cursor = store._conn.cursor(cursor_factory=RealDictCursor) 33 | 34 | cursor.execute("BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE") 35 | cursor.execute("SET statement_timeout = '600s'") 36 | cursor.execute("select r.name as name, rl.summary as summary, array_agg(distinct trove_id) as trove_classifiers, array_agg(distinct t.l2) || array_agg(distinct t.l3) || array_agg(distinct t.l4) || array_agg(distinct t.l5) as categories from release_classifiers r join releases rl on (rl.name=r.name and rl.version=r.version) join trove_classifiers t on r.trove_id=t.id where not rl._pypi_hidden group by r.name, rl.summary") 37 | while True: 38 | packages = cursor.fetchmany(1000) 39 | if len(packages) == 0: 40 | break 41 | operations = [] 42 | for package in packages: 43 | operations.append(json.dumps({"index": {"_index": new_index, "_type": "release_classifiers", "_id": package['name']}})) 44 | operations.append(json.dumps(package)) 45 | r = requests.post(conf.database_releases_index_url + "/_bulk", data="\n".join(operations)) 46 | 47 | actions = [{"add": {"index": new_index, "alias": "trove-%s" % (conf.database_releases_index_name,)}}] 48 | r = requests.get(conf.database_releases_index_url + "/*/_alias/" + "trove-%s" % (conf.database_releases_index_name,)) 49 | for alias, data in r.json().iteritems(): 50 | if conf.database_releases_index_name in data['aliases'].keys(): 51 | actions.append({"remove": {"index": alias, "alias": conf.database_releases_index_name}}) 52 | 53 | print("updating alias for %s to %s" % (conf.database_releases_index_name, new_index)) 54 | r = requests.post(conf.database_releases_index_url + "/_aliases", json={'actions': actions}) 55 | if r.status_code != 200: 56 | print("failed to update alias for %s to %s" % (conf.database_releases_index_name, new_index)) 57 | sys.exit(1) 58 | 59 | r = requests.get(conf.database_releases_index_url + "/_stats") 60 | indicies = [i for i in r.json()['indices'].keys() if i.startswith("trove-%s" % (conf.database_releases_index_name,) + '-')] 61 | for index in indicies: 62 | if index != new_index: 63 | print('deleting %s, because it is not %s' % (index, new_index)) 64 | r = requests.delete(conf.database_releases_index_url + "/" + index) 65 | if r.status_code != 200: 66 | print('failed to delete %s' % (index)) 67 | -------------------------------------------------------------------------------- /tools/index.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import os 4 | import time 5 | 6 | prefix = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 7 | sys.path.insert(0, prefix) 8 | 9 | # Workaround current bug in docutils: 10 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 11 | import docutils.utils 12 | 13 | from psycopg2.extras import RealDictCursor 14 | import json 15 | import requests 16 | 17 | import config 18 | import store 19 | from pkg_resources import safe_name 20 | 21 | CONFIG_FILE = os.environ.get("PYPI_CONFIG", os.path.join(prefix, 'config.ini')) 22 | 23 | conf = config.Config(CONFIG_FILE) 24 | 25 | if conf.database_releases_index_name is None or conf.database_releases_index_url is None: 26 | sys.exit() 27 | 28 | new_index = "%s-%s" % (conf.database_releases_index_name, int(time.time())) 29 | print("creating new index %s" % (new_index,)) 30 | store = store.Store(conf) 31 | store.open() 32 | 33 | cursor = store._conn.cursor(cursor_factory=RealDictCursor) 34 | 35 | cursor.execute("BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE") 36 | cursor.execute("SET statement_timeout = '600s'") 37 | cursor.execute("SELECT DISTINCT ON (name, _pypi_hidden) name, version, _pypi_ordering, _pypi_hidden, author, author_email, maintainer, maintainer_email, home_page, license, summary, description, keywords, platform, download_url FROM releases ORDER BY name, _pypi_hidden, _pypi_ordering DESC") 38 | while True: 39 | releases = cursor.fetchmany(10000) 40 | if len(releases) == 0: 41 | break 42 | operations = [] 43 | for release in releases: 44 | operations.append(json.dumps({"index": {"_index": new_index, "_type": "release", "_id": "%s-%s" % (release['name'], release['version'])}})) 45 | release['name_exact'] = safe_name(release['name']).lower() 46 | operations.append(json.dumps(release)) 47 | r = requests.post(conf.database_releases_index_url + "/_bulk", data="\n".join(operations)) 48 | 49 | actions = [{"add": {"index": new_index, "alias": conf.database_releases_index_name}}] 50 | r = requests.get(conf.database_releases_index_url + "/*/_alias/" + conf.database_releases_index_name) 51 | for alias, data in r.json().iteritems(): 52 | if conf.database_releases_index_name in data['aliases'].keys(): 53 | actions.append({"remove": {"index": alias, "alias": conf.database_releases_index_name}}) 54 | 55 | print("updating alias for %s to %s" % (conf.database_releases_index_name, new_index)) 56 | r = requests.post(conf.database_releases_index_url + "/_aliases", json={'actions': actions}) 57 | if r.status_code != 200: 58 | print("failed to update alias for %s to %s" % (conf.database_releases_index_name, new_index)) 59 | sys.exit(1) 60 | 61 | r = requests.get(conf.database_releases_index_url + "/_stats") 62 | indicies = [i for i in r.json()['indices'].keys() if i.startswith(conf.database_releases_index_name + '-')] 63 | for index in indicies: 64 | if index != new_index: 65 | print('deleting %s, because it is not %s' % (index, new_index)) 66 | r = requests.delete(conf.database_releases_index_url + "/" + index) 67 | if r.status_code != 200: 68 | print('failed to delete %s' % (index)) 69 | -------------------------------------------------------------------------------- /tools/integrate-redis-stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import datetime 5 | import redis 6 | import string 7 | 8 | from itertools import izip, izip_longest 9 | 10 | # Workaround current bug in docutils: 11 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 12 | import docutils.utils 13 | 14 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 15 | sys.path = [root] + sys.path 16 | 17 | import store 18 | import config 19 | 20 | c = config.Config(os.path.join(root, "config.ini")) 21 | redis = redis.Redis.from_url(c.count_redis_url) 22 | 23 | # Get our search for the previous hour keys 24 | current = datetime.datetime.utcnow() 25 | lasthour = current - datetime.timedelta(hours=1) 26 | search = "downloads:hour:%s:*:*" % lasthour.strftime("%y-%m-%d-%H") 27 | 28 | # Make sure we haven't integrated this already 29 | if redis.sismember("downloads:integrated", search): 30 | print("Already Integrated '%s'" % search) 31 | sys.exit(0) 32 | 33 | # Fetch all of the keys 34 | keys = redis.keys(search) 35 | 36 | if not keys: 37 | print("No keys match '%s'" % search) 38 | sys.exit(0) 39 | 40 | # Fetch all of the download counts (in batches of 200) 41 | counts = [] 42 | for batch in izip_longest(*[iter(keys)] * 200): 43 | batch = [x for x in batch if x is not None] 44 | counts.extend(redis.mget(*batch)) 45 | 46 | # Combine the keys with the counts 47 | downloads = izip( 48 | (int(y) for y in counts), 49 | (x.split(":")[-1] for x in keys), 50 | ) 51 | 52 | # Update the database 53 | store = store.Store(c) 54 | cursor = store.get_cursor() 55 | cursor.executemany( 56 | "UPDATE release_files SET downloads = downloads + %s WHERE filename = %s", 57 | (d for d in downloads if not set(d[1]) - set(string.printable)), 58 | ) 59 | store.commit() 60 | store.close() 61 | 62 | # Add this to our integrated set 63 | redis.sadd("downloads:integrated", search) 64 | -------------------------------------------------------------------------------- /tools/integratestats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys, os, socket, psycopg2, urllib, re, bz2, cStringIO, ConfigParser 3 | sys.path.append(os.path.dirname(__file__)+"/..") 4 | import apache_stats 5 | 6 | statsdir = '/data/www/pypi/stats' 7 | 8 | def integrate(config, data, dbconn): 9 | for (filename, browser, package), count in data.items(): 10 | cursor.execute('update release_files set downloads=downloads+%s where filename=%s', 11 | (count, filename)) 12 | dbconn.commit() 13 | 14 | def integrate_remote(config, host, dbconn, dbupdate=True): 15 | url = 'http://%s.pypi.python.org/local-stats/days/' % host 16 | try: 17 | index = urllib.urlopen(url).read() 18 | except Exception, e: 19 | print 'ERROR %s for %s' % (e, url) 20 | return 21 | files = set(re.findall('href=.(20..-..-..).bz2', index)) 22 | try: 23 | integrated = open(statsdir+'/integrated/'+host).readlines() 24 | integrated = set([x.strip() for x in integrated]) 25 | except IOError: 26 | integrated = set() 27 | missing = files-integrated 28 | stats = apache_stats.LocalStats() 29 | for m in missing: 30 | url = 'http://%s.pypi.python.org/local-stats/days/%s.bz2' % (host, m) 31 | try: 32 | data = urllib.urlopen(url).read() 33 | except Exception, e: 34 | print 'ERROR %s for %s' % (e, url) 35 | continue 36 | data = bz2.decompress(data) 37 | data = cStringIO.StringIO(data) 38 | year, month, day = m.split('-') 39 | # index integration 40 | delta = stats.integrate_stats(statsdir, year, month, day, data) 41 | if dbupdate: 42 | # database integration 43 | integrate(config, delta, dbconn=dbconn) 44 | integrated.add(m) 45 | open(statsdir+'/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) 46 | 47 | def main(): 48 | # Setup database connection 49 | c = ConfigParser.ConfigParser({'user':'', 'password':''}) 50 | c.read(sys.argv[1]) 51 | dbname = c.get('database', 'name') 52 | dbuser = c.get('database', 'user') 53 | dbpass = c.get('database', 'password') 54 | dbhost = c.get('database', 'host') 55 | dbconn = psycopg2.connect(database=dbname, user=dbuser, password=dbpass, host=dbhost) 56 | 57 | lasts = socket.gethostbyname_ex('last.pypi.python.org') 58 | # look for name X.pypi.python.org 59 | lasts = [lasts[0]] + lasts[1] 60 | for last in lasts: 61 | if last[1:] == '.pypi.python.org': 62 | break 63 | else: 64 | raise ValueError, "Could not properly resolve last mirror name" 65 | last = last.split('.')[0] 66 | host = 'a' 67 | while True: 68 | if host == "d": 69 | # This is just a CNAME back to a.pypi.python.org 70 | continue 71 | 72 | integrate_remote(sys.argv[1], host, dbconn=dbconn) 73 | if host == last: 74 | break 75 | host = chr(ord(host)+1) 76 | dbconn.close() 77 | 78 | main() 79 | 80 | -------------------------------------------------------------------------------- /tools/keygen: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | openssl dsaparam -out param 2048 3 | openssl gendsa -out privkey param 4 | openssl dsa -in privkey -pubout -out pubkey 5 | rm param 6 | -------------------------------------------------------------------------------- /tools/migrate.py: -------------------------------------------------------------------------------- 1 | # migrate a historical SQLITE database to a postgres one 2 | 3 | import sqlite 4 | import psycopg 5 | 6 | old = sqlite.connect(db='pkgbase.db') 7 | cursor = old.cursor() 8 | e = cursor.execute 9 | 10 | new = psycopg.connect(database='pypi') 11 | new_cursor = new.cursor() 12 | f = new_cursor.execute 13 | 14 | t = [ 15 | 'users name password email public_key', 16 | 'packages name stable_version', 17 | 'releases name version author author_email maintainer maintainer_email home_page license summary description keywords platform download_url _pypi_ordering _pypi_hidden', 18 | 'trove_classifiers id classifier', 19 | 'release_classifiers name version trove_id', 20 | 'journals name version action submitted_date submitted_by submitted_from', 21 | 'rego_otk name otk', 22 | 'roles role_name user_name package_name', 23 | ] 24 | for table in t: 25 | l = table.split() 26 | tn = l[0] 27 | print tn 28 | cols = ', '.join(l[1:]) 29 | args = ', '.join(['%s']*(len(l)-1)) 30 | e('select %(cols)s from %(tn)s'%locals()) 31 | for row in cursor.fetchall(): 32 | d = list(row) 33 | if '_pypi_ordering' in l[1:]: d[-2] = d[-2] and int(float(d[-2])) 34 | f('insert into %(tn)s (%(cols)s) values (%(args)s)'%locals(), 35 | tuple(d)) 36 | 37 | new.commit() 38 | -------------------------------------------------------------------------------- /tools/mksqlite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | dbpath = "packages.db" 4 | 5 | if os.path.exists(dbpath): 6 | print "Remove",dbpath,"first" 7 | raise SystemExit 8 | 9 | print "Creating database", dbpath 10 | sqlite = os.popen('sqlite3 '+dbpath, "w") 11 | passthrough = True 12 | for line in open('pkgbase_schema.sql'): 13 | if 'nosqlite-end' in line: 14 | # end of disabled block 15 | passthrough = True 16 | print >>sqlite 17 | continue 18 | if 'nosqlite' in line: 19 | passthrough = False 20 | print >>sqlite 21 | continue 22 | if not passthrough: 23 | print >> sqlite 24 | continue 25 | # make sqlite happy: SERIAL is not a valid type 26 | sqlite.write(line.replace('SERIAL PRIMARY KEY', 'INTEGER PRIMARY KEY')) 27 | sqlite.close() 28 | -------------------------------------------------------------------------------- /tools/pw_reset_mailout.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import csv 3 | import time 4 | import smtplib 5 | 6 | import store 7 | import config 8 | 9 | 10 | def main(): 11 | c = config.Config('/data/pypi/config.ini') 12 | if len(sys.argv) != 2: 13 | st = store.Store(c) 14 | st.open() 15 | w = csv.writer(sys.stdout) 16 | for user in st.get_users(): 17 | w.writerow((user['name'], user['email'])) 18 | else: 19 | with open(sys.argv[1]) as f: 20 | for name, email in csv.reader(f): 21 | message = TEXT.format(name=name, email=email) 22 | # send email 23 | s = smtplib.SMTP() 24 | s.connect('localhost') 25 | s.sendmail('richard@python.org', [email], message) 26 | s.close() 27 | time.sleep(.1) 28 | 29 | TEXT = '''From: richard@python.org 30 | To: {email} 31 | Subject: PyPI security notice 32 | 33 | 34 | TL;DR: please log into PyPI and change your password. 35 | 36 | Dear PyPI user {name}, 37 | 38 | Recently we have been auditing and improving security of the Python Package 39 | Index (PyPI) and other python.org hosts. 40 | 41 | You may be aware that the wiki.python.org host was compromised. Since we must 42 | assume that all passwords stored in that system are also compromised, and we 43 | also assume that some users share passwords between python.org systems, we are 44 | performing a password reset of all PyPI accounts in one week's time, at 45 | 2013-02-22 00:00 UTC. 46 | 47 | If you log in before that deadline and change your password then you'll be 48 | fine, otherwise you'll need to use the password recovery form after the reset 49 | has occurred. 50 | 51 | Additionally, we would ask you to begin to access PyPI using HTTPS through the 52 | web. We're in the process of installing a new SSL certificate so the current 53 | Big Red Certificate Warning should go away very soon. 54 | 55 | We are in the process of updating the Python packaging toolset to use HTTPS. 56 | 57 | These steps are but a couple of those we're intending to take to better secure 58 | PyPI. If you are interested in these matters I encourage you to participate in 59 | the discussion on the catalog SIG: 60 | 61 | http://mail.python.org/mailman/listinfo/catalog-sig 62 | 63 | Finally, we apologise for any inconvenience these changes have caused. 64 | 65 | 66 | Richard Jones 67 | richard@python.org 68 | PyPI Maintainer 69 | ''' 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /tools/rsyslog-cdn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -u 2 | import sys 3 | import redis 4 | import csv 5 | import os 6 | import posixpath 7 | import datetime 8 | import logging 9 | import logging.handlers 10 | 11 | from email.utils import parsedate 12 | 13 | # Make sure our PyPI directory is on the sys.path 14 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 15 | sys.path = [root] + sys.path 16 | 17 | import config 18 | 19 | conf = config.Config(os.environ.get("PYPI_CONFIG", os.path.join(root, "config.ini"))) 20 | 21 | PRECISIONS = [ 22 | ("hour", "%y-%m-%d-%H", datetime.timedelta(days=2)), 23 | ("daily", "%y-%m-%d", datetime.timedelta(days=32)), 24 | ] 25 | 26 | 27 | logger = logging.getLogger("rsyslog-cdn") 28 | logger.setLevel(logging.DEBUG) 29 | logger.addHandler(logging.handlers.SysLogHandler(address="/dev/log")) 30 | 31 | store = redis.Redis.from_url(conf.count_redis_url) 32 | 33 | 34 | def make_key(precision, when, key): 35 | return "downloads:%s:%s:%s" % ( 36 | precision[0], when.strftime(precision[1]), key) 37 | 38 | 39 | def incr(when, project, filename): 40 | # Increment our rolling counts in Redis 41 | for prec in PRECISIONS: 42 | key = make_key(prec, when, project) 43 | store.incr(key) 44 | store.expireat(key, when + prec[2]) 45 | 46 | # Increment our filename based bucket in Redis 47 | for prec in PRECISIONS: 48 | key = make_key(prec, when, ":".join([project, filename])) 49 | store.incr(key) 50 | store.expireat(key, when + prec[2]) 51 | 52 | 53 | def process(line): 54 | if "last message repeated" in line: 55 | logger.error("Duplicate Line in rsyslog-cdn") 56 | 57 | try: 58 | row = list(csv.reader([line], delimiter=" "))[0] 59 | path = row[6].split(" ", 1)[1] 60 | except Exception: 61 | logger.error("Invalid Fastly Log Line: '%s'" % line) 62 | return 63 | 64 | # We only care about /packages/ urls 65 | if not path.startswith("/packages/"): 66 | return 67 | 68 | # We need to get the Project and Filename 69 | directory, filename = posixpath.split(path) 70 | project = posixpath.basename(directory) 71 | 72 | # We need to get the time this request occurred 73 | rtime = datetime.datetime(*parsedate(row[4])[:6]) 74 | 75 | incr(rtime, project, filename) 76 | 77 | 78 | if __name__ == "__main__": 79 | line = sys.stdin.readline() 80 | while line: 81 | try: 82 | process(line) 83 | except Exception: 84 | logger.exception("Error occured while processing '%s'", line) 85 | raise 86 | line = sys.stdin.readline() 87 | -------------------------------------------------------------------------------- /tools/sanitize_package_names.py: -------------------------------------------------------------------------------- 1 | #!usr/bin/env python 2 | import re 3 | import os 4 | import sys 5 | import pickle 6 | 7 | # Workaround current bug in docutils: 8 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 9 | import docutils.utils 10 | 11 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 12 | sys.path = [root] + sys.path 13 | 14 | import store 15 | import config 16 | 17 | 18 | c = config.Config("config.ini") 19 | store = store.Store(c) 20 | store.open() 21 | cursor = store.get_cursor() 22 | 23 | valid_name = re.compile( 24 | r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", 25 | re.IGNORECASE, 26 | ) 27 | 28 | 29 | def bad_names(): 30 | cursor.execute(""" 31 | SELECT name from packages WHERE 32 | name !~* '^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$' 33 | """) 34 | return set(x[0] for x in cursor.fetchall()) 35 | 36 | renamed = [] 37 | 38 | junk_names = bad_names() 39 | print("Found %s bad names" % len(junk_names)) 40 | 41 | print("Rename packages where the only invalid name is a space") 42 | for name in junk_names: 43 | if name.strip() != name: 44 | continue 45 | 46 | if valid_name.search(name.replace(" ", "-")) is not None: 47 | new_name = re.sub("\s+", "-", name) 48 | renamed.append((name, new_name)) 49 | store.rename_package(name, new_name) 50 | else: 51 | new_name = re.sub("[^A-Za-z0-9.]+", "-", name).strip("_-.") 52 | if valid_name.search(new_name) is None: 53 | continue 54 | num = 2 55 | onew_name = new_name 56 | while [x for x in store.find_package(new_name) if x != name]: 57 | new_name = "%s%s" % (onew_name, str(num)) 58 | num += 1 59 | renamed.append((name, new_name)) 60 | store.rename_package(name, new_name) 61 | 62 | with open("renamed.pkl", "w") as pkl: 63 | pickle.dump(renamed, pkl) 64 | 65 | # Commit the changes 66 | store.commit() 67 | 68 | junk_names = bad_names() 69 | print("Found %s bad names" % len(junk_names)) 70 | -------------------------------------------------------------------------------- /tools/sql-migrate-20050320-2.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Table structure for table: release_provides 3 | CREATE TABLE release_provides ( 4 | name TEXT, 5 | version TEXT, 6 | specifier TEXT, 7 | FOREIGN KEY (name, version) REFERENCES releases (name, version) 8 | ); 9 | CREATE INDEX rel_prov_name_idx ON release_provides(name); 10 | CREATE INDEX rel_prov_version_id_idx ON release_provides(version); 11 | 12 | 13 | -- Table structure for table: release_requires 14 | CREATE TABLE release_requires ( 15 | name TEXT, 16 | version TEXT, 17 | specifier TEXT, 18 | FOREIGN KEY (name, version) REFERENCES releases (name, version) 19 | ); 20 | CREATE INDEX rel_req_name_idx ON release_requires(name); 21 | CREATE INDEX rel_req_version_id_idx ON release_requires(version); 22 | 23 | 24 | -- Table structure for table: release_obsoletes 25 | CREATE TABLE release_obsoletes ( 26 | name TEXT, 27 | version TEXT, 28 | specifier TEXT, 29 | FOREIGN KEY (name, version) REFERENCES releases (name, version) 30 | ); 31 | CREATE INDEX rel_obs_name_idx ON release_obsoletes(name); 32 | CREATE INDEX rel_obs_version_id_idx ON release_obsoletes(version); 33 | -------------------------------------------------------------------------------- /tools/sql-migrate-20050320.sql: -------------------------------------------------------------------------------- 1 | 2 | DROP TABLE package_files; 3 | 4 | DROP TABLE package_urls; 5 | 6 | CREATE TABLE release_files ( 7 | name TEXT, 8 | version TEXT, 9 | python_version TEXT, 10 | packagetype TEXT, 11 | comment_text TEXT, 12 | filename TEXT UNIQUE, 13 | md5_digest TEXT UNIQUE, 14 | FOREIGN KEY (name, version) REFERENCES releases (name, version) 15 | ); 16 | CREATE INDEX release_files_name_idx ON release_files(name); 17 | CREATE INDEX release_files_version_idx ON release_files(version); 18 | CREATE INDEX release_files_packagetype_idx ON release_files(packagetype); 19 | 20 | 21 | CREATE TABLE release_urls ( 22 | name TEXT, 23 | version TEXT, 24 | url TEXT, 25 | packagetype TEXT, 26 | FOREIGN KEY (name, version) REFERENCES releases (name, version) 27 | ); 28 | CREATE INDEX release_urls_name_idx ON release_urls(name); 29 | CREATE INDEX release_urls_version_idx ON release_urls(version); 30 | CREATE INDEX release_urls_packagetype_idx ON release_urls(packagetype); 31 | 32 | 33 | ALTER TABLE users DROP COLUMN public_key; 34 | 35 | ALTER TABLE USERS ADD COLUMN gpg_keyid TEXT; 36 | 37 | -------------------------------------------------------------------------------- /tools/sql-migrate-20050322.sql: -------------------------------------------------------------------------------- 1 | 2 | ALTER TABLE releases ADD COLUMN description_html TEXT; 3 | 4 | -------------------------------------------------------------------------------- /tools/sql-migrate-20050402.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE timestamps ( 2 | name TEXT PRIMARY KEY, 3 | value TIMESTAMP 4 | ); 5 | INSERT INTO timestamps(name, value) VALUES('http','1970-01-01'); 6 | INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01'); 7 | 8 | ALTER TABLE release_files ADD COLUMN downloads INTEGER; 9 | ALTER TABLE release_files ALTER downloads SET DEFAULT 0; 10 | 11 | -------------------------------------------------------------------------------- /tools/sql-migrate-20070409.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01'); 2 | CREATE TABLE browse_tally ( 3 | trove_id INTEGER PRIMARY KEY, 4 | tally INTEGER 5 | ); 6 | 7 | -------------------------------------------------------------------------------- /tools/sql-migrate-20070706.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE trove_classifiers ADD COLUMN l2 INTEGER; 2 | ALTER TABLE trove_classifiers ADD COLUMN l3 INTEGER; 3 | ALTER TABLE trove_classifiers ADD COLUMN l4 INTEGER; 4 | ALTER TABLE trove_classifiers ADD COLUMN l5 INTEGER; 5 | 6 | CREATE INDEX rel_class_name_version_idx ON release_classifiers(name, version); 7 | 8 | -- run 'store.py config.ini checktrove' after installing this change 9 | -------------------------------------------------------------------------------- /tools/sql-migrate-20070709.sql: -------------------------------------------------------------------------------- 1 | create index journals_latest_releases on journals(submitted_date, name, version) where version is not null and action='new release'; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20070713.sql: -------------------------------------------------------------------------------- 1 | create index journals_changelog on journals(submitted_date, name, version, action); 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20070721.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE description_urls ( 2 | name TEXT, 3 | version TEXT, 4 | url TEXT, 5 | FOREIGN KEY (name, version) REFERENCES releases (name, version) 6 | ); 7 | CREATE INDEX description_urls_name_idx ON description_urls(name); 8 | CREATE INDEX description_urls_name_version_idx ON description_urls(name, version); 9 | grant all on description_urls to pypi; 10 | 11 | -- run 'store.py config.ini updateurls' after installing this change 12 | -------------------------------------------------------------------------------- /tools/sql-migrate-20080512.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE packages ADD COLUMN normalized_name TEXT; 2 | \echo Run store.py update_normalized_text now 3 | ALTER TABLE users ADD COLUMN last_login TIMESTAMP; 4 | -------------------------------------------------------------------------------- /tools/sql-migrate-20081007.sql: -------------------------------------------------------------------------------- 1 | -- before running this, verify that the constraints 2 | -- that get deleted are the same ones that get recreated 3 | -- (with cascading update) 4 | begin; 5 | alter table releases drop constraint "$1"; 6 | alter table releases add foreign key (name) references packages(name) on update cascade; 7 | 8 | alter table release_provides drop constraint "$1"; 9 | alter table release_provides add foreign key (name, version) references releases(name, version) on update cascade; 10 | 11 | alter table release_requires drop constraint "$1"; 12 | alter table release_requires add foreign key (name, version) references releases(name, version) on update cascade; 13 | 14 | alter table release_obsoletes drop constraint "$1"; 15 | alter table release_obsoletes add foreign key (name, version) references releases(name, version) on update cascade; 16 | 17 | alter table release_files drop constraint "$1"; 18 | alter table release_files add foreign key (name, version) references releases(name, version) on update cascade; 19 | 20 | alter table release_urls drop constraint "$1"; 21 | alter table release_urls add foreign key (name, version) references releases(name, version) on update cascade; 22 | 23 | alter table description_urls drop constraint "$1"; 24 | alter table description_urls add foreign key (name, version) references releases(name, version) on update cascade; 25 | 26 | alter table release_classifiers drop constraint "$2"; 27 | alter table release_classifiers add foreign key (name, version) references releases(name, version) on update cascade; 28 | 29 | alter table roles drop constraint "$2"; 30 | alter table roles add foreign key (package_name) references packages(name) on update cascade; 31 | 32 | commit; 33 | 34 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090124.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE packages ADD COLUMN autohide BOOLEAN DEFAULT TRUE; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090208.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE release_files ADD COLUMN upload_time TIMESTAMP; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090327.sql: -------------------------------------------------------------------------------- 1 | -- Table structure for table: mirrors 2 | CREATE TABLE mirrors ( 3 | root_url TEXT PRIMARY KEY, 4 | user_name TEXT REFERENCES users, 5 | index_url TEXT, 6 | last_modified_url TEXT, 7 | local_stats_url TEXT, 8 | stats_url TEXT, 9 | mirrors_url TEXT 10 | ); 11 | 12 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090329.sql: -------------------------------------------------------------------------------- 1 | alter table mirrors drop index_url; 2 | alter table mirrors drop last_modified_url; 3 | alter table mirrors drop local_stats_url; 4 | alter table mirrors drop stats_url; 5 | alter table mirrors drop mirrors_url; 6 | 7 | 8 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090331.sql: -------------------------------------------------------------------------------- 1 | alter table mirrors rename root_url to ip; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090829.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE openids ( 2 | id TEXT PRIMARY KEY, 3 | name TEXT REFERENCES users 4 | ); 5 | 6 | CREATE TABLE openid_sessions ( 7 | id SERIAL PRIMARY KEY, 8 | provider TEXT, 9 | url TEXT, 10 | assoc_handle TEXT, 11 | expires TIMESTAMP, 12 | mac_key TEXT 13 | ); 14 | 15 | CREATE TABLE openid_stypes ( 16 | id INTEGER REFERENCES openid_sessions ON DELETE CASCADE, 17 | stype TEXT 18 | ); 19 | CREATE INDEX openid_stypes_id ON openid_stypes(id); 20 | 21 | CREATE TABLE openid_nonces ( 22 | created TIMESTAMP, 23 | nonce TEXT 24 | ); 25 | CREATE INDEX openid_nonces_created ON openid_nonces(created); 26 | CREATE INDEX openid_nonces_nonce ON openid_nonces(nonce); 27 | 28 | CREATE TABLE cookies ( 29 | cookie text PRIMARY KEY, 30 | name text references users, 31 | last_seen timestamp 32 | ); 33 | CREATE INDEX cookies_last_seen ON cookies(last_seen); 34 | -------------------------------------------------------------------------------- /tools/sql-migrate-20090914.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE ratings( 2 | name TEXT, 3 | version TEXT, 4 | user_name TEXT REFERENCES users ON DELETE CASCADE, 5 | date TIMESTAMP, 6 | rating INTEGER, 7 | message TEXT, 8 | PRIMARY KEY (name, version, user_name), 9 | FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE 10 | ); 11 | CREATE INDEX rating_name_version ON ratings(name, version); 12 | GRANT ALL ON ratings TO pypi; -------------------------------------------------------------------------------- /tools/sql-migrate-20091015.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE ratings ADD id SERIAL UNIQUE; 4 | GRANT ALL ON ratings_id_seq TO pypi; 5 | 6 | CREATE TABLE comments( 7 | id SERIAL PRIMARY KEY, 8 | rating INTEGER REFERENCES ratings(id) ON DELETE CASCADE, 9 | user_name TEXT REFERENCES users ON DELETE CASCADE, 10 | date TIMESTAMP, 11 | message TEXT, 12 | in_reply_to INTEGER REFERENCES comments ON DELETE CASCADE 13 | ); 14 | GRANT ALL ON comments TO pypi; 15 | GRANT ALL ON comments_id_seq TO pypi; 16 | 17 | INSERT INTO comments(rating, user_name, date, message) 18 | SELECT id, user_name, date, message FROM ratings WHERE message!=''; 19 | 20 | ALTER TABLE ratings DROP COLUMN message; 21 | 22 | CREATE TABLE comments_journal( 23 | name text, 24 | version text, 25 | id INTEGER, 26 | submitted_by TEXT REFERENCES users ON DELETE CASCADE, 27 | date TIMESTAMP, 28 | action TEXT, 29 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON DELETE CASCADE 30 | ); 31 | GRANT ALL ON comments_journal TO pypi; 32 | 33 | END; 34 | -------------------------------------------------------------------------------- /tools/sql-migrate-20091125.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rego_otk ADD COLUMN date TIMESTAMP; 2 | UPDATE rego_otk SET date=now(); 3 | BEGIN; 4 | ALTER TABLE rego_otk DROP CONSTRAINT "$1"; 5 | ALTER TABLE rego_otk ADD CONSTRAINT "$1" FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; 6 | END; -------------------------------------------------------------------------------- /tools/sql-migrate-20091204.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE packages ADD COLUMN comments BOOLEAN DEFAULT TRUE; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20091228.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX rel_req_name_version_idx ON release_requires (name,version); 2 | CREATE INDEX rel_prov_name_version_idx ON release_provides (name,version); 3 | CREATE INDEX rel_obs_name_version_idx ON release_obsoletes (name,version); 4 | CREATE INDEX release_files_name_version_idx ON release_files(name,version); 5 | -------------------------------------------------------------------------------- /tools/sql-migrate-20100102.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE sshkeys( 2 | id SERIAL PRIMARY KEY, 3 | name TEXT REFERENCES users ON DELETE CASCADE, 4 | key TEXT 5 | ); 6 | CREATE INDEX sshkeys_name ON sshkeys(name); 7 | GRANT ALL ON sshkeys TO pypi; 8 | GRANT ALL ON sshkeys_id_seq TO pypi; 9 | -------------------------------------------------------------------------------- /tools/sql-migrate-20100103.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX rego_otk_otk_idx ON rego_otk(otk); 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20100313.sql: -------------------------------------------------------------------------------- 1 | -- New fields 2 | ALTER TABLE releases ADD COLUMN requires_python TEXT; 3 | 4 | -- 5 | -- New tables 6 | -- 7 | 8 | -- Table structure for table: release_requires_python 9 | CREATE TABLE release_requires_python ( 10 | name TEXT, 11 | version TEXT, 12 | specifier TEXT, 13 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 14 | ); 15 | CREATE INDEX rel_req_python_name_idx ON release_requires_python(name); 16 | CREATE INDEX rel_req_python_version_id_idx ON release_requires_python(version); 17 | CREATE INDEX rel_req_python_name_version_idx ON release_requires_python(name,version); 18 | 19 | -- Table structure for table: release_requires_external 20 | CREATE TABLE release_requires_external ( 21 | name TEXT, 22 | version TEXT, 23 | specifier TEXT, 24 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 25 | ); 26 | CREATE INDEX rel_req_ext_name_idx ON release_requires_external(name); 27 | CREATE INDEX rel_req_ext_version_id_idx ON release_requires_external(version); 28 | CREATE INDEX rel_req_ext_name_version_idx ON release_requires_external(name,version); 29 | 30 | -- Table structure for table: release_requires_dist 31 | CREATE TABLE release_requires_dist ( 32 | name TEXT, 33 | version TEXT, 34 | specifier TEXT, 35 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 36 | ); 37 | CREATE INDEX rel_req_dist_name_idx ON release_requires_dist(name); 38 | CREATE INDEX rel_req_dist_version_id_idx ON release_requires_dist(version); 39 | CREATE INDEX rel_req_dist_name_version_idx ON release_requires_dist(name,version); 40 | 41 | -- Table structure for table: release_provides_dist 42 | CREATE TABLE release_provides_dist ( 43 | name TEXT, 44 | version TEXT, 45 | specifier TEXT, 46 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 47 | ); 48 | CREATE INDEX rel_prov_dist_name_idx ON release_provides_dist(name); 49 | CREATE INDEX rel_prov_dist_version_id_idx ON release_provides_dist(version); 50 | CREATE INDEX rel_prov_dist_name_version_idx ON release_provides_dist(name,version); 51 | 52 | -- Table structure for table: release_obsoletes_dist 53 | CREATE TABLE release_obsoletes_dist ( 54 | name TEXT, 55 | version TEXT, 56 | specifier TEXT, 57 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 58 | ); 59 | CREATE INDEX rel_obs_dist_name_idx ON release_obsoletes_dist(name); 60 | CREATE INDEX rel_obs_dist_version_id_idx ON release_obsoletes_dist(version); 61 | CREATE INDEX rel_obs_dist_name_version_idx ON release_obsoletes_dist(name,version); 62 | 63 | -- Table structure for table: release_project_url 64 | CREATE TABLE release_project_url ( 65 | name TEXT, 66 | version TEXT, 67 | specifier TEXT, 68 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 69 | ); 70 | CREATE INDEX rel_proj_url_name_idx ON release_project_url(name); 71 | CREATE INDEX rel_proj_url_version_id_idx ON release_project_url(version); 72 | CREATE INDEX rel_proj_url_name_version_idx ON release_project_url(name,version); 73 | 74 | 75 | -------------------------------------------------------------------------------- /tools/sql-migrate-20100724.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | alter table ratings drop constraint ratings_pkey; 3 | alter table ratings add primary key (id); 4 | alter table ratings add unique(name,version,user_name); 5 | end; -------------------------------------------------------------------------------- /tools/sql-migrate-20100814.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | CREATE TABLE release_dependencies ( 3 | name TEXT, 4 | version TEXT, 5 | kind INTEGER, 6 | specifier TEXT, 7 | FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE 8 | ); 9 | grant all on release_dependencies to pypi; 10 | 11 | insert into release_dependencies(name, version, kind, specifier) 12 | select name, version, 1, specifier from release_requires; 13 | insert into release_dependencies(name, version, kind, specifier) 14 | select name, version, 2, specifier from release_provides; 15 | insert into release_dependencies(name, version, kind, specifier) 16 | select name, version, 3, specifier from release_obsoletes; 17 | insert into release_dependencies(name, version, kind, specifier) 18 | select name, version, 4, specifier from release_requires_dist; 19 | insert into release_dependencies(name, version, kind, specifier) 20 | select name, version, 5, specifier from release_provides_dist; 21 | insert into release_dependencies(name, version, kind, specifier) 22 | select name, version, 6, specifier from release_obsoletes_dist; 23 | insert into release_dependencies(name, version, kind, specifier) 24 | select name, version, 7, specifier from release_requires_external; 25 | insert into release_dependencies(name, version, kind, specifier) 26 | select name, version, 8, specifier from release_project_url; 27 | 28 | CREATE INDEX rel_dep_name_idx ON release_dependencies(name); 29 | CREATE INDEX rel_dep_name_version_idx ON release_dependencies(name, version); 30 | CREATE INDEX rel_dep_name_version_kind_idx ON release_dependencies(name, version, kind); 31 | 32 | drop table release_requires; 33 | drop table release_provides; 34 | drop table release_obsoletes; 35 | drop table release_requires_dist; 36 | drop table release_provides_dist; 37 | drop table release_obsoletes_dist; 38 | drop table release_requires_external; 39 | drop table release_project_url; 40 | 41 | COMMIT; 42 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110129.sql: -------------------------------------------------------------------------------- 1 | alter TABLE rego_otk add constraint rego_otk_unique unique(otk); 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110220.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | alter table comments_journal drop constraint comments_journal_name_fkey; 3 | alter table comments_journal 4 | add foreign key (name, version) references releases 5 | on update cascade on delete cascade; 6 | end; 7 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110822.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE openids DROP CONSTRAINT openids_name_fkey; 2 | ALTER TABLE openids ADD CONSTRAINT openids_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; 3 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110823-2.sql: -------------------------------------------------------------------------------- 1 | create table csrf_tokens ( 2 | name text REFERENCES users(name) ON DELETE CASCADE, 3 | token text, 4 | end_date timestamp without time zone, 5 | PRIMARY KEY(name) 6 | ); 7 | 8 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110823.sql: -------------------------------------------------------------------------------- 1 | alter table packages add bugtrack_url text; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110831.sql: -------------------------------------------------------------------------------- 1 | insert into users(name, password, email, gpg_keyid, last_login) 2 | values('deleted user', 'invalid', '', '', '2000-01-01'); 3 | -------------------------------------------------------------------------------- /tools/sql-migrate-20110905.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE openid_whitelist 2 | ( 3 | "name" text NOT NULL, 4 | trust_root text NOT null, 5 | created timestamp without time zone, 6 | CONSTRAINT openid_whitelist__pkey PRIMARY KEY (name, trust_root) 7 | ); 8 | -------------------------------------------------------------------------------- /tools/sql-migrate-20111119.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE openid_discovered ( 2 | created TIMESTAMP, 3 | url TEXT PRIMARY KEY, 4 | services BYTEA, 5 | op_endpoint TEXT, 6 | op_local TEXT 7 | ); 8 | alter table openid_sessions drop provider; 9 | drop table openid_stypes; 10 | 11 | 12 | -------------------------------------------------------------------------------- /tools/sql-migrate-20111120.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | CREATE TABLE oid_nonces 3 | ( 4 | server_url VARCHAR(2047) NOT NULL, 5 | timestamp INTEGER NOT NULL, 6 | salt CHAR(40) NOT NULL, 7 | PRIMARY KEY (server_url, timestamp, salt) 8 | ); 9 | 10 | CREATE TABLE oid_associations 11 | ( 12 | server_url VARCHAR(2047) NOT NULL, 13 | handle VARCHAR(255) NOT NULL, 14 | secret BYTEA NOT NULL, 15 | issued INTEGER NOT NULL, 16 | lifetime INTEGER NOT NULL, 17 | assoc_type VARCHAR(64) NOT NULL, 18 | PRIMARY KEY (server_url, handle), 19 | CONSTRAINT secret_length_constraint CHECK (LENGTH(secret) <= 128) 20 | ); 21 | 22 | GRANT ALL ON oid_nonces, oid_associations TO pypi; 23 | commit; 24 | -------------------------------------------------------------------------------- /tools/sql-migrate-20111130.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | alter table journals drop constraint "$1"; 3 | alter table journals add foreign key (submitted_by) references users (name) on update cascade; 4 | end; 5 | begin; 6 | alter table roles drop constraint "$1"; 7 | alter table roles add foreign key (user_name) references users (name) on update cascade; 8 | end; 9 | begin; 10 | alter table cookies drop constraint cookies_name_fkey; 11 | alter table cookies add foreign key (name) references users (name) on update cascade on delete cascade; 12 | end; 13 | begin; 14 | alter table openids drop constraint openids_name_fkey; 15 | alter table openids add foreign key (name) references users (name) on update cascade on delete cascade; 16 | end; 17 | begin; 18 | alter table sshkeys drop constraint sshkeys_name_fkey; 19 | alter table sshkeys add foreign key (name) references users (name) on update cascade on delete cascade; 20 | end; 21 | begin; 22 | alter table csrf_tokens drop constraint csrf_tokens_name_fkey; 23 | alter table csrf_tokens add foreign key (name) references users (name) on update cascade on delete cascade; 24 | end; 25 | -------------------------------------------------------------------------------- /tools/sql-migrate-20120312.sql: -------------------------------------------------------------------------------- 1 | alter table releases add description_from_readme BOOLEAN; 2 | 3 | -------------------------------------------------------------------------------- /tools/sql-migrate-20120314.sql: -------------------------------------------------------------------------------- 1 | -- tables for the oauth library 2 | 3 | CREATE TABLE oauth_consumers ( 4 | consumer varchar(32) primary key, 5 | secret varchar(64) not null, 6 | date_created date not null, 7 | created_by TEXT REFERENCES users ON UPDATE CASCADE, 8 | last_modified date not null, 9 | description varchar(255) not null 10 | ); 11 | 12 | CREATE TABLE oauth_request_tokens ( 13 | token varchar(32) primary key, 14 | secret varchar(64) not null, 15 | consumer varchar(32) not null, 16 | callback text, 17 | date_created date not null, 18 | "user" TEXT REFERENCES users ON UPDATE CASCADE ON DELETE CASCADE 19 | ); 20 | 21 | CREATE TABLE oauth_access_tokens ( 22 | token varchar(32) primary key, 23 | secret varchar(64) not null, 24 | consumer varchar(32) not null, 25 | date_created date not null, 26 | last_modified date not null, 27 | "user" TEXT REFERENCES users ON UPDATE CASCADE ON DELETE CASCADE 28 | ); 29 | 30 | CREATE TABLE oauth_nonce ( 31 | timestamp integer not null, 32 | consumer varchar(32) not null, 33 | nonce varchar(32) not null, 34 | token varchar(32) 35 | ); 36 | 37 | -------------------------------------------------------------------------------- /tools/sql-migrate-20120325.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | CREATE TABLE oauth_request_tokens2 ( 3 | token varchar(32) primary key, 4 | secret varchar(64) not null, 5 | consumer varchar(32) not null, 6 | callback text, 7 | date_created date not null, 8 | user_name TEXT REFERENCES users ON UPDATE CASCADE ON DELETE CASCADE 9 | ); 10 | 11 | CREATE TABLE oauth_access_tokens2 ( 12 | token varchar(32) primary key, 13 | secret varchar(64) not null, 14 | consumer varchar(32) not null, 15 | date_created date not null, 16 | last_modified date not null, 17 | user_name TEXT REFERENCES users ON UPDATE CASCADE ON DELETE CASCADE 18 | ); 19 | 20 | insert into oauth_request_tokens2(token, secret, consumer, 21 | callback, date_created, user_name) 22 | select token, secret, consumer, callback, date_created, 23 | user from oauth_request_tokens; 24 | drop table oauth_request_tokens; 25 | alter table oauth_request_tokens2 rename to oauth_request_tokens; 26 | 27 | insert into oauth_access_tokens2(token, secret, consumer, 28 | date_created, last_modified, user_name) 29 | select token, secret, consumer, date_created, last_modified, 30 | user from oauth_access_tokens; 31 | drop table oauth_access_tokens; 32 | alter table oauth_access_tokens2 rename to oauth_access_tokens; 33 | 34 | end; 35 | -------------------------------------------------------------------------------- /tools/sql-migrate-20130114.sql: -------------------------------------------------------------------------------- 1 | alter table journals add column id serial; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20130319-2.sql: -------------------------------------------------------------------------------- 1 | alter table description_urls add id serial primary key; 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20130319.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "packages" ADD COLUMN "hosting_mode" text NOT NULL DEFAULT 'pypi-scrape-crawl'; 2 | ALTER TABLE "packages" ALTER COLUMN "hosting_mode" SET DEFAULT 'pypi-explicit'; 3 | -------------------------------------------------------------------------------- /tools/sql-migrate-20130523.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX ON users (LOWER(name)); 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20130917.sql: -------------------------------------------------------------------------------- 1 | -- Placeholder for Warehouse 2 | -------------------------------------------------------------------------------- /tools/sql-migrate-20140702.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO accounts_user (password, last_login, is_superuser, username, name, 2 | is_staff, is_active, date_joined) 3 | VALUES ('!', now(), false, 'deleted-user', 'deleted user', false, false, now()); -------------------------------------------------------------------------------- /tools/sqlite_create.py: -------------------------------------------------------------------------------- 1 | # HISTORICAL USE ONLY 2 | 3 | # IS THE CODE THAT USED TO CONSTRUCT THE SQLITE DATABASE 4 | 5 | # IS NOT COMPLETE 6 | 7 | # HISTORICAL USE ONLY 8 | cursor.execute(''' 9 | create table ids ( 10 | name varchar, 11 | num varchar 12 | )''') 13 | cursor.execute(''' 14 | create table packages ( 15 | name varchar, 16 | stable_version varchar 17 | )''') 18 | cursor.execute(''' 19 | create table releases ( 20 | name varchar, 21 | version varchar, 22 | author varchar, 23 | author_email varchar, 24 | maintainer varchar, 25 | maintainer_email varchar, 26 | home_page varchar, 27 | license varchar, 28 | summary varchar, 29 | description varchar, 30 | keywords varchar, 31 | platform varchar, 32 | download_url varchar, 33 | _pypi_ordering varchar, 34 | _pypi_hidden varchar 35 | )''') 36 | cursor.execute(''' 37 | create table trove_classifiers ( 38 | id varchar, 39 | classifier varchar 40 | )''') 41 | cursor.execute(''' 42 | create table release_classifiers ( 43 | name varchar, 44 | version varchar, 45 | trove_id varchar 46 | )''') 47 | cursor.execute(''' 48 | create table journals ( 49 | id integer primary key autoincrement, 50 | name varchar, 51 | version varchar, 52 | action varchar, 53 | submitted_date timestamp, 54 | submitted_by varchar, 55 | submitted_from varchar 56 | )''') 57 | cursor.execute(''' 58 | create table users ( 59 | name varchar, 60 | password varchar, 61 | email varchar, 62 | public_key varchar 63 | )''') 64 | cursor.execute(''' 65 | create table rego_otk ( 66 | name varchar, 67 | otk varchar 68 | )''') 69 | cursor.execute(''' 70 | create table roles ( 71 | role_name varchar, 72 | user_name varchar, 73 | package_name varchar 74 | )''') 75 | 76 | # init the id counter 77 | cursor.execute('''insert into ids (name, num) values 78 | ('trove_classifier', 1)''') 79 | 80 | # indexes 81 | SQLs = [ 82 | "create index ids_name_idx on ids(name)", 83 | "create index journals_name_idx on journals(name)", 84 | "create index journals_version_idx on journals(version)", 85 | "create index packages_name_idx on packages(name)", 86 | "create index rego_otk_name_idx on rego_otk(name)", 87 | "create index reset_otk_name_idx on reset_otk(name)", 88 | "create index reset_otk_otk_idx on reset_otk(otk)", 89 | "create index rel_class_name_idx on release_classifiers(name)", 90 | "create index rel_class_trove_id_idx on " 91 | "release_classifiers(trove_id)", 92 | "create index rel_class_version_id_idx on " 93 | "release_classifiers(version)", 94 | "create index release_name_idx on releases(name)", 95 | "create index release_pypi_hidden_idx on releases(_pypi_hidden)", 96 | "create index release_version_idx on releases(version)", 97 | "create index roles_pack_name_idx on roles(package_name)", 98 | "create index roles_user_name_idx on roles(user_name)", 99 | "create index trove_class_class_idx on " 100 | "trove_classifiers(classifier)", 101 | "create index trove_class_id_idx on trove_classifiers(id)", 102 | "create index users_email_idx on users(email)", 103 | "create index users_name_idx on users(name)", 104 | ] 105 | for sql in SQLs: 106 | cursor.execute(sql) 107 | 108 | # admin user 109 | adminpw = ''.join([random.choice(chars) for x in range(10)]) 110 | adminpw = sha.sha(adminpw).hexdigest() 111 | cursor.execute(''' 112 | insert into users (name, password, email) values 113 | ('admin', '%s', NULL) 114 | '''%adminpw) 115 | cursor.execute(''' 116 | insert into roles (user_name, role_name, package_name) values 117 | ('admin', 'Admin', NULL) 118 | ''') 119 | -------------------------------------------------------------------------------- /tools/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/tools/tests/__init__.py -------------------------------------------------------------------------------- /tools/tests/pypi.access.log.1.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypi/legacy/a418131180a14768328713df5533c600c425760d/tools/tests/pypi.access.log.1.bz2 -------------------------------------------------------------------------------- /tools/tests/pypi.cfg: -------------------------------------------------------------------------------- 1 | [webui] 2 | 3 | files_url = /packages 4 | 5 | [database] 6 | 7 | name = localhost 8 | user = tarek 9 | password = secret 10 | 11 | [mirrors] 12 | folder = tests/mirrors 13 | local-stats = tests/local-stats 14 | global-stats = tests/global-stats 15 | 16 | -------------------------------------------------------------------------------- /tools/touch_all_files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | This script touches all files known to the database, creating a skeletal 4 | mirror for local development. 5 | """ 6 | 7 | import sys, os 8 | import store 9 | 10 | def get_paths(cursor, prefix=None): 11 | store.safe_execute(cursor, "SELECT python_version, name, filename FROM release_files") 12 | 13 | for type, name, filename in cursor.fetchall(): 14 | yield os.path.join(prefix, type, name[0], name, filename) 15 | 16 | if __name__ == '__main__': 17 | import config 18 | try: 19 | config = config.Config(sys.argv[1]) 20 | except IndexError: 21 | print "Usage: touch_all_files.py config.ini" 22 | raise SystemExit 23 | 24 | datastore = store.Store(config) 25 | datastore.open() 26 | cursor = datastore.get_cursor() 27 | prefix = config.database_files_dir 28 | 29 | for path in get_paths(cursor, prefix): 30 | dir = os.path.dirname(path) 31 | if not os.path.exists(dir): 32 | print "Creating directory %s" % dir 33 | os.makedirs(dir) 34 | if not os.path.exists(path): 35 | print "Creating file %s" % path 36 | open(path, "a").write('Contents of '+path+'\n') 37 | -------------------------------------------------------------------------------- /tools/upgradepw.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import base64 3 | import os 4 | import sys 5 | 6 | # Workaround current bug in docutils: 7 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 8 | import docutils.utils 9 | 10 | 11 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 12 | sys.path.append(root) 13 | 14 | import config 15 | import store 16 | import passlib.registry 17 | 18 | bcrypt = passlib.registry.get_crypt_handler("bcrypt") 19 | bcrypt_sha1 = passlib.registry.get_crypt_handler("bcrypt_sha1") 20 | 21 | cfg = config.Config(os.path.join(root, "config.ini")) 22 | st = store.Store(cfg) 23 | 24 | print "Migrating passwords to bcrypt_sha1 from unsalted sha1....", 25 | 26 | st.open() 27 | for i, u in enumerate(st.get_users()): 28 | user = st.get_user(u['name']) 29 | # basic sanity check to allow it to run concurrent with users accessing 30 | if len(user['password']) == 40 and "$" not in user["password"]: 31 | # Hash the existing sha1 password with bcrypt 32 | bcrypted = bcrypt.encrypt(user["password"]) 33 | 34 | # Base64 encode the bcrypted password so that it's just a blob of data 35 | encoded = base64.b64encode(bcrypted) 36 | 37 | st.setpasswd(user['name'], bcrypt_sha1._hash_prefix + encoded, 38 | hashed=True, 39 | ) 40 | 41 | # Commit every 20 users 42 | if not i % 20: 43 | st.commit() 44 | st.open() 45 | 46 | st.commit() 47 | st.close() 48 | 49 | print "[ok]" 50 | -------------------------------------------------------------------------------- /tools/utf8convert.py: -------------------------------------------------------------------------------- 1 | ''' Implements a store of disutils PKG-INFO entries, keyed off name, version. 2 | ''' 3 | import sys, os, re, psycopg, time, sha, random, types, math, stat, errno 4 | import logging 5 | 6 | conn = psycopg.connect(database="pypi", 7 | user="pypi") 8 | 9 | cursor = conn.cursor() 10 | cursor.execute("SELECT * FROM releases") 11 | for release in cursor.dictfetchall(): 12 | newitem = {} 13 | for k,v in release.items(): 14 | if type(v) is str: 15 | newitem[k]=v.decode("latin-1").encode('utf-8') 16 | cols = [] 17 | values = [] 18 | for k,v in newitem.items(): 19 | cols.append("%s=%%s" % k) 20 | values.append(v) 21 | cols = ",".join(cols) 22 | values.append(newitem['name']) 23 | values.append(newitem['version']) 24 | stmt="update releases set %s where name=%%s and version=%%s" % cols 25 | cursor.execute(stmt, values) 26 | conn.commit() 27 | -------------------------------------------------------------------------------- /tools/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import os.path 4 | import sys 5 | 6 | import raven 7 | import redis 8 | import rq 9 | import rq.contrib.sentry 10 | 11 | # Workaround current bug in docutils: 12 | # http://permalink.gmane.org/gmane.text.docutils.devel/6324 13 | import docutils.utils 14 | 15 | # Make sure our PyPI directory is on the sys.path 16 | root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | sys.path = [root] + sys.path 18 | 19 | import config 20 | 21 | conf = config.Config(os.environ.get("PYPI_CONFIG", os.path.join(root, "config.ini"))) 22 | redis_conn = redis.Redis.from_url(conf.queue_redis_url) 23 | 24 | # Create our queues 25 | if sys.argv[1:]: 26 | queues = [rq.Queue(name, connection=redis_conn) for name in sys.argv[1:]] 27 | else: 28 | queues = [rq.Queue(connection=redis_conn)] 29 | 30 | # Create our Worker 31 | worker = rq.Worker(queues, connection=redis_conn) 32 | 33 | # Create our Sentry Client 34 | if conf.sentry_dsn: 35 | raven_client = raven.Client(conf.sentry_dsn) 36 | rq.contrib.sentry.register_sentry(raven_client, worker) 37 | 38 | # Run our worker, fetching jobs from the queue 39 | worker.work() 40 | -------------------------------------------------------------------------------- /trove.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys 4 | 5 | class Node: 6 | def __init__(self, id=None, name=None, path=None, path_split=None): 7 | self.arcs = {} 8 | self.id = id 9 | self.name = name 10 | self.path = path 11 | self.path_split = path_split 12 | if path_split: 13 | self.level = len(path_split) 14 | else: 15 | self.level = 1 16 | 17 | def __repr__(self): 18 | return '<Node %d %s>'%(self.id, self.name) 19 | 20 | def subtree_ids(self): 21 | result = [self.id] 22 | for node in self.arcs.values(): 23 | result.extend(node.subtree_ids()) 24 | return result 25 | 26 | class Trove: 27 | def __init__(self, cursor): 28 | self.root = Node() 29 | self.trove = {} 30 | cursor.execute('select id,classifier from trove_classifiers order by classifier') 31 | 32 | # now generate the tree 33 | for id, line in cursor.fetchall(): 34 | id = int(id) 35 | d = self.root 36 | # make this a tuple so we can use it as a key 37 | path_split = tuple([s.strip() for s in line.split('::')]) 38 | for arc in path_split: 39 | if d.arcs.has_key(arc): 40 | d = d.arcs[arc] 41 | else: 42 | n = Node(id, arc, line.strip(), path_split) 43 | self.trove[id] = n 44 | d.arcs[arc] = n 45 | d = n 46 | self.FIELDS = self.root.arcs.keys() 47 | 48 | def getid(self, path): 49 | node = self.root 50 | for arc in path: 51 | node = node.arcs[arc] 52 | return node.id 53 | 54 | def __getitem__(self, key): 55 | return self.trove[key] 56 | 57 | EXCLUSIVE_FIELDS = { 58 | 'Development Status':1, 59 | 'Natural Language':1, 60 | 'License':1, 61 | } 62 | 63 | --------------------------------------------------------------------------------