├── .gitignore ├── LICENSE ├── README.md ├── bin ├── gs-pypi └── gs-pypi-generate-db ├── docs ├── Makefile ├── gs-pypi-generate-db.8 ├── gs-pypi-generate-db.8.rst ├── gs-pypi.8 └── gs-pypi.8.rst ├── gs-pypi-overlays.xml ├── gs-pypi.json ├── gs_pypi ├── __init__.py ├── backend.py ├── data │ └── gs-pypi.eclass ├── ebuild.py ├── gs_pypi_generate_db.py └── pypi_db.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | var 11 | sdist 12 | develop-eggs 13 | .installed.cfg 14 | 15 | # Installer logs 16 | pip-log.txt 17 | 18 | # Unit test / coverage reports 19 | .coverage 20 | .tox 21 | 22 | #Translations 23 | *.mo 24 | 25 | #Mr Developer 26 | .mr.developer.cfg 27 | 28 | #tmp files 29 | *\~ 30 | 31 | tst 32 | 33 | #layman 34 | layman 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is [g-sorcery](https://github.com/jauhien/g-sorcery) backend for Python PyPI packages. 2 | 3 | 4 | # Installation 5 | 6 | ``` 7 | emerge -va gs-pypi 8 | ``` 9 | 10 | This will install all the necessary dependencies, including g-sorcery with BSON support, 11 | layman with g-sorcery support and pymongo (used for BSON manipulation). 12 | 13 | # Usage 14 | 15 | There are two ways of using **gs-pypi** 16 | 17 | ## Using gs-pypi with [layman](https://wiki.gentoo.org/wiki/Layman) 18 | 19 | It the the recommend way and I strongly suggest it. 20 | 21 | Then you should just run `layman -L` as 22 | root and find an overlay you want. Type of overlay will be 23 | displayed as *g-sorcery*. Then you add this overlay as 24 | usually and emerge packages you want. It's all you need to do. Example: 25 | 26 | ``` 27 | layman -L 28 | layman -a pypi 29 | emerge -va pycallgraph 30 | ``` 31 | 32 | There is 1 gs-pypi overlay currently: [pypi](https://pypi.python.org/pypi). 33 | 34 | When using **gs-pypi** with layman you can populate overlay only with packages you want. 35 | To do so you should add a section named gs-pypi to */etc/g-sorcery/g-sorcery.cfg*. 36 | In this section you can add entries named REPO_packages (REPO here is the name 37 | of repository you want to add -- pypi) which are space separated lists of packages you need. 38 | Note, that at the moment **gs-pypi** doesn't support dependencies for pypi packages, so you need to add 39 | to this list package itself and all its dependencies. 40 | 41 | ``` 42 | [main] 43 | package_manager=portage 44 | 45 | [gs-pypi] 46 | pypi_packages=pycallgraph 47 | ``` 48 | I strongly recommend to do so, as pypi overlay is quite big and you may be 49 | do not want generate all the ebuilds from it. To list available packages use 50 | the list command from the next section giving it appropriate layman overlay directory as argument. 51 | 52 | 53 | ## Using gs-pypi as stand-alone tool (strongly not recommended) 54 | 55 | In this case you should create an overlay (see **portage** documentation), sync it and populate 56 | it with one or more ebuilds. Then ebuilds could be installed by emerge or by **gs-pypi** tool. 57 | This is not the recommended way and may be removed in the future. 58 | 59 | Create new user overlay: 60 | 61 | ``` 62 | gs-pypi -o $OVERLAY_DIRECTORY -r pypi sync 63 | ``` 64 | 65 | List packages: 66 | 67 | ``` 68 | gs-pypi -o $OVERLAY_DIRECTORY -r pypi list 69 | ``` 70 | 71 | Install any package you want: 72 | 73 | ``` 74 | gs-pypi -o $OVERLAY_DIRECTORY -r pypi install $PACKAGE 75 | ``` 76 | 77 | Note, that if you call **generate-tree** command your overlay 78 | will be wiped and overlay tree for a given repository will be generated. Be careful! 79 | 80 | See man page of **gs-pypi** for further information. 81 | -------------------------------------------------------------------------------- /bin/gs-pypi: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | g-sorcery gs-pypi $@ 4 | -------------------------------------------------------------------------------- /bin/gs-pypi-generate-db: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | gs-pypi-generate-db 6 | ~~~~~~~~~~~~~~~~~~~ 7 | 8 | PyPI database generation 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import sys 15 | from gs_pypi import gs_pypi_generate_db 16 | 17 | if __name__ == "__main__": 18 | sys.exit(gs_pypi_generate_db.main()) 19 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | MAN_SOURCES=gs-pypi gs-pypi-generate-db 2 | MANS=$(MAN_SOURCES:=.8) 3 | 4 | RST2MAN=rst2man.py 5 | 6 | all: ${MANS} 7 | 8 | %.8: %.8.rst 9 | $(RST2MAN) $< $@ 10 | -------------------------------------------------------------------------------- /docs/gs-pypi-generate-db.8: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH GS-PYPI-GENERATE-DB 8 "2015-04-22" "0.2.1" "g-sorcery" 4 | .SH NAME 5 | gs-pypi-generate-db \- generate DB for gs-pypi 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBgs\-pypi\-generate\-db\fP [\fB\-c\fP \fICOUNT\fP] [\fB\-\-layout\-version\fP \fILAYOUT_VERSION\fP] 36 | [\fB\-\-structure\-version\fP \fISTRUCTURE_VERSION\fP] [\fB\-f\fP \fIFORMAT\fP] \fIDB_DIRNAME\fP 37 | .SH DESCRIPTION 38 | .sp 39 | \fBgs\-pypi\-generate\-db\fP is a maintenance tool for \fBgs\-pypi\fP\&. It 40 | generates DB that later can be used during ebuild generation. 41 | .SH OPTIONS 42 | .INDENT 0.0 43 | .TP 44 | .B \fB\-c\fP \fICOUNT\fP 45 | Count of package entries that should be processed (if absent \-\- 46 | process all packages). 47 | .TP 48 | .B \fB\-\-layout\-version\fP \fILAYOUT_VERSION\fP 49 | \fBg\-sorcery\fP DB directory layout version (v.1 be default). 50 | .TP 51 | .B \fB\-\-structure\-version\fP \fISTRUCTURE_VERSION\fP 52 | \fBg\-sorcery\fP DB structure version (v.1 by default). 53 | .TP 54 | .B \fB\-f\fP \fIFORMAT\fP 55 | Packages data file format (BSON by default). 56 | .UNINDENT 57 | .SH SEE ALSO 58 | .sp 59 | \fBgs\-pypi\fP(8), \fBg\-sorcery.cfg\fP(8), \fBportage\fP(5), \fBemerge\fP(1), \fBlayman\fP(8) 60 | .SH AUTHOR 61 | Written by Jauhien Piatlicki . GSoC idea 62 | and mentorship by Rafael Martins. Lots of help and improvements 63 | by Brian Dolbec. 64 | .SH COPYRIGHT 65 | Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 66 | .\" Generated by docutils manpage writer. 67 | . 68 | -------------------------------------------------------------------------------- /docs/gs-pypi-generate-db.8.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | gs-pypi-generate-db 3 | =================== 4 | 5 | ----------------------- 6 | generate DB for gs-pypi 7 | ----------------------- 8 | 9 | :Author: Written by Jauhien Piatlicki . GSoC idea 10 | and mentorship by Rafael Martins. Lots of help and improvements 11 | by Brian Dolbec. 12 | :Date: 2015-04-22 13 | :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 14 | :Version: 0.2.1 15 | :Manual section: 8 16 | :Manual group: g-sorcery 17 | 18 | SYNOPSIS 19 | ======== 20 | 21 | **gs-pypi-generate-db** [**-c** *COUNT*] [**--layout-version** *LAYOUT_VERSION*] 22 | [**--structure-version** *STRUCTURE_VERSION*] [**-f** *FORMAT*] *DB_DIRNAME* 23 | 24 | DESCRIPTION 25 | =========== 26 | 27 | **gs-pypi-generate-db** is a maintenance tool for **gs-pypi**. It 28 | generates DB that later can be used during ebuild generation. 29 | 30 | OPTIONS 31 | ======= 32 | 33 | **-c** *COUNT* 34 | Count of package entries that should be processed (if absent -- 35 | process all packages). 36 | 37 | **--layout-version** *LAYOUT_VERSION* 38 | **g-sorcery** DB directory layout version (v.1 be default). 39 | 40 | **--structure-version** *STRUCTURE_VERSION* 41 | **g-sorcery** DB structure version (v.1 by default). 42 | 43 | **-f** *FORMAT* 44 | Packages data file format (BSON by default). 45 | 46 | SEE ALSO 47 | ======== 48 | 49 | **gs-pypi**\(8), **g-sorcery.cfg**\(8), **portage**\(5), **emerge**\(1), **layman**\(8) 50 | -------------------------------------------------------------------------------- /docs/gs-pypi.8: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH GS-PYPI 8 "2015-04-22" "0.2.1" "g-sorcery" 4 | .SH NAME 5 | gs-pypi \- manage overlays for PYPI repository 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBsync\fP 36 | .sp 37 | \fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBlist\fP 38 | .sp 39 | \fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBgenerate\fP \fIPACKAGE\fP 40 | .sp 41 | \fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBinstall\fP \fIPACKAGE\fP 42 | .sp 43 | \fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBgenerate\-tree\fP [\fB\-d\fP] 44 | .SH DESCRIPTION 45 | .sp 46 | \fBgs\-pypi\fP is an ebuild generator for PYPI PYTHON repository. 47 | .sp 48 | There are two ways of using \fBgs\-pypi\fP: 49 | .INDENT 0.0 50 | .INDENT 3.5 51 | .INDENT 0.0 52 | .IP \(bu 2 53 | use it with \fBlayman\fP 54 | .sp 55 | In this case all you need to do is install \fBgs\-pypi\fP\&. 56 | Then you should just run \fIlayman \-L\fP as 57 | root and find an overlay you want (\fBpypi\fP). Type of overlay will be 58 | displayed as \fIg\-sorcery\fP\&. Then you add this overlay as 59 | usual. That\(aqs the recommended way of 60 | using \fBgs\-pypi\fP\&. Be aware that by default \fBpypi\fP will 61 | contain lots of ebuilds, you\(aqll need to change config before 62 | adding the overlay (see below) to prevent this. 63 | .IP \(bu 2 64 | use it as stand\-alone tool (not recommended) 65 | .sp 66 | In this case you should create an overlay (see \fBportage\fP documentation), sync it and populate 67 | it with one or more ebuilds. Then ebuilds could be installed by emerge or by \fBgs\-pypi\fP tool. 68 | .UNINDENT 69 | .UNINDENT 70 | .UNINDENT 71 | .SH OPTIONS 72 | .INDENT 0.0 73 | .TP 74 | .B \fB\-\-overlay\fP \fIOVERLAY\fP, \fB\-o\fP \fIOVERLAY\fP 75 | Overlay directory. This option is mandatory if there is no 76 | \fBdefault_overlay\fP entry in a backend config. 77 | .TP 78 | .B \fB\-\-repository\fP \fIREPO\fP, \fB\-r\fP \fIREPO\fP 79 | Repository name. This option is not mandatory. If present should be \fBctan\fP\&. 80 | .UNINDENT 81 | .SH COMMANDS 82 | .INDENT 0.0 83 | .TP 84 | .B \fBsync\fP 85 | Synchronize a repository database. 86 | .TP 87 | .B \fBlist\fP 88 | List packages available in a repository. 89 | .TP 90 | .B \fBgenerate\fP 91 | Generate a given ebuild and all its dependencies. 92 | .TP 93 | .B \fBinstall\fP 94 | Generate and install an ebuild using your package mangler. 95 | .TP 96 | .B \fBgenerate\-tree\fP 97 | Generate entire overlay structure. Without option \fB\-d\fP after 98 | this command sources are not fetched during generation and there 99 | are no entries for them in Manifest files. 100 | .UNINDENT 101 | .SH FILES 102 | .INDENT 0.0 103 | .TP 104 | .B \fB/etc/g\-sorcery/gs\-pypi.json\fP 105 | Backend config. 106 | .TP 107 | .B \fB/etc/layman/overlays/gs\-pypi\-overlays.xml\fP 108 | List of available repositories. 109 | .UNINDENT 110 | .SH EXAMPLES 111 | .INDENT 0.0 112 | .TP 113 | .B Using gs\-pypi with layman 114 | Execute 115 | .sp 116 | \fBlayman \-L\fP 117 | .sp 118 | If you see there a \fBpypi\fP overlay then everything should work. 119 | .sp 120 | \fBIMPORTANT\fP 121 | .sp 122 | Change \fIg\-sorcery.cfg\fP so it includes a list of packages you need, 123 | otherwise \fBgs\-pypi\fP will generate a huge amount of ebuilds (see 124 | \fIg\-sorcery.cfg\fP man page): 125 | .INDENT 7.0 126 | .INDENT 3.5 127 | .sp 128 | .nf 129 | .ft C 130 | [main] 131 | package_manager=portage 132 | 133 | [gs\-pypi] 134 | pypi_packages=pycallgraph 135 | .ft P 136 | .fi 137 | .UNINDENT 138 | .UNINDENT 139 | .sp 140 | Packages list is whitespace separated. 141 | To list available packages use list 142 | command from the next section. 143 | .sp 144 | Add overlay as usual: 145 | .sp 146 | \fBlayman \-a pypi\fP 147 | .sp 148 | Emerge any package from it using \fBemerge\fP\&. 149 | .TP 150 | .B Generating user ebuilds in user overlay (not recommended) 151 | Create new user overlay. Run 152 | .sp 153 | \fBgs\-pypi \-o\fP \fIOVERLAY_DIRECTORY\fP \fB\-r ctan\fP \fBsync\fP 154 | .sp 155 | List packages: 156 | .sp 157 | \fBgs\-pypi \-o\fP \fIOVERLAY_DIRECTORY\fP \fB\-r ctan\fP \fBlist\fP 158 | .sp 159 | Install any package you want: 160 | .sp 161 | \fBgs\-pypi \-o\fP \fIOVERLAY_DIRECTORY\fP \fB\-r ctan\fP \fBinstall\fP \fIPACKAGE\fP 162 | .sp 163 | Note, that if you call \fBgenerate\-tree\fP command your overlay 164 | will be wiped and overlay tree for a given repository will be generated. Be careful! 165 | .UNINDENT 166 | .SH NOTES 167 | .INDENT 0.0 168 | .IP 1. 3 169 | At the moment the only package mangler \fBgs\-pypi\fP supports is \fBportage\fP\&. 170 | .UNINDENT 171 | .SH SEE ALSO 172 | .sp 173 | \fBgs\-elpa\fP(8), \fBg\-sorcery.cfg\fP(8), \fBportage\fP(5), \fBemerge\fP(1), \fBlayman\fP(8) 174 | .SH AUTHOR 175 | Written by Jauhien Piatlicki . GSoC idea 176 | and mentorship by Rafael Martins. Lots of help and improvements 177 | by Brian Dolbec. 178 | .SH COPYRIGHT 179 | Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 180 | .\" Generated by docutils manpage writer. 181 | . 182 | -------------------------------------------------------------------------------- /docs/gs-pypi.8.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | gs-pypi 3 | ======= 4 | 5 | ----------------------------------- 6 | manage overlays for PYPI repository 7 | ----------------------------------- 8 | 9 | :Author: Written by Jauhien Piatlicki . GSoC idea 10 | and mentorship by Rafael Martins. Lots of help and improvements 11 | by Brian Dolbec. 12 | :Date: 2015-04-22 13 | :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 14 | :Version: 0.2.1 15 | :Manual section: 8 16 | :Manual group: g-sorcery 17 | 18 | 19 | SYNOPSIS 20 | ======== 21 | 22 | **gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **sync** 23 | 24 | **gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **list** 25 | 26 | **gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **generate** *PACKAGE* 27 | 28 | **gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **install** *PACKAGE* 29 | 30 | **gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **generate-tree** [**-d**] 31 | 32 | DESCRIPTION 33 | =========== 34 | 35 | **gs-pypi** is an ebuild generator for PYPI PYTHON repository. 36 | 37 | There are two ways of using **gs-pypi**: 38 | 39 | * use it with **layman** 40 | 41 | In this case all you need to do is install **gs-pypi**. 42 | Then you should just run `layman -L` as 43 | root and find an overlay you want (**pypi**). Type of overlay will be 44 | displayed as *g-sorcery*. Then you add this overlay as 45 | usual. That's the recommended way of 46 | using **gs-pypi**. Be aware that by default **pypi** will 47 | contain lots of ebuilds, you'll need to change config before 48 | adding the overlay (see below) to prevent this. 49 | 50 | * use it as stand-alone tool (not recommended) 51 | 52 | In this case you should create an overlay (see **portage** documentation), sync it and populate 53 | it with one or more ebuilds. Then ebuilds could be installed by emerge or by **gs-pypi** tool. 54 | 55 | 56 | OPTIONS 57 | ======= 58 | 59 | **--overlay** *OVERLAY*, **-o** *OVERLAY* 60 | Overlay directory. This option is mandatory if there is no 61 | **default_overlay** entry in a backend config. 62 | 63 | **--repository** *REPO*, **-r** *REPO* 64 | Repository name. This option is not mandatory. If present should be **ctan**. 65 | 66 | COMMANDS 67 | ======== 68 | 69 | **sync** 70 | Synchronize a repository database. 71 | 72 | **list** 73 | List packages available in a repository. 74 | 75 | **generate** 76 | Generate a given ebuild and all its dependencies. 77 | 78 | **install** 79 | Generate and install an ebuild using your package mangler. 80 | 81 | **generate-tree** 82 | Generate entire overlay structure. Without option **-d** after 83 | this command sources are not fetched during generation and there 84 | are no entries for them in Manifest files. 85 | 86 | FILES 87 | ===== 88 | **/etc/g-sorcery/gs-pypi.json** 89 | Backend config. 90 | 91 | **/etc/layman/overlays/gs-pypi-overlays.xml** 92 | List of available repositories. 93 | 94 | EXAMPLES 95 | ======== 96 | 97 | Using gs-pypi with layman 98 | Execute 99 | 100 | **layman -L** 101 | 102 | If you see there a **pypi** overlay then everything should work. 103 | 104 | **IMPORTANT** 105 | 106 | Change *g-sorcery.cfg* so it includes a list of packages you need, 107 | otherwise **gs-pypi** will generate a huge amount of ebuilds (see 108 | *g-sorcery.cfg* man page): 109 | 110 | .. code-block:: 111 | 112 | [main] 113 | package_manager=portage 114 | 115 | [gs-pypi] 116 | pypi_packages=pycallgraph 117 | 118 | Packages list is whitespace separated. 119 | To list available packages use list 120 | command from the next section. 121 | 122 | Add overlay as usual: 123 | 124 | **layman -a pypi** 125 | 126 | Emerge any package from it using **emerge**. 127 | 128 | Generating user ebuilds in user overlay (not recommended) 129 | Create new user overlay. Run 130 | 131 | **gs-pypi -o** *OVERLAY_DIRECTORY* **-r ctan** **sync** 132 | 133 | List packages: 134 | 135 | **gs-pypi -o** *OVERLAY_DIRECTORY* **-r ctan** **list** 136 | 137 | Install any package you want: 138 | 139 | **gs-pypi -o** *OVERLAY_DIRECTORY* **-r ctan** **install** *PACKAGE* 140 | 141 | Note, that if you call **generate-tree** command your overlay 142 | will be wiped and overlay tree for a given repository will be generated. Be careful! 143 | 144 | NOTES 145 | ===== 146 | 147 | 1. At the moment the only package mangler **gs-pypi** supports is **portage**. 148 | 149 | SEE ALSO 150 | ======== 151 | 152 | **gs-elpa**\(8), **g-sorcery.cfg**\(8), **portage**\(5), **emerge**\(1), **layman**\(8) 153 | -------------------------------------------------------------------------------- /gs-pypi-overlays.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | pypi 6 | python packages 7 | https://pypi.python.org/pypi 8 | 9 | jauhien@gentoo.org 10 | Jauhien Piatlicki 11 | 12 | gs-pypi pypi 13 | 14 | 15 | -------------------------------------------------------------------------------- /gs-pypi.json: -------------------------------------------------------------------------------- 1 | { 2 | "backend": "gs-pypi", 3 | "package": "gs_pypi", 4 | "repositories": { 5 | "pypi": { 6 | "repo_uri": "https://pypi.python.org/", 7 | "sync_method": "git", 8 | "db_uri": "https://github.com/jauhien/gs-pypi-db/", 9 | "branch": "bson" 10 | } 11 | }, 12 | "common_config": { 13 | "licenses": { 14 | "Academic Free License (AFL)": "AFL-3.0", 15 | "Aladdin Free Public License (AFPL)": "Aladdin", 16 | "Apache Software License": "Apache-2.0", 17 | "Artistic License": "Artistic", 18 | "BSD License": "BSD", 19 | "CC0 1.0 Universal (CC0 1.0) Public Domain Dedication": "CC0-1.0", 20 | "Common Public License": "CPL-1.0", 21 | "GNU Affero General Public License v3": "AGPL-3", 22 | "GNU Affero General Public License v3 or later (AGPLv3+)": "AGPL-3", 23 | "GNU Free Documentation License (FDL)": "FDL-1.1+", 24 | "GNU General Public License (GPL)": "GPL-1+", 25 | "GNU General Public License v2 (GPLv2)": "GPL-2", 26 | "GNU General Public License v2 or later (GPLv2+)": "GPL-2+", 27 | "GNU General Public License v3 (GPLv3)": "GPL-3", 28 | "GNU General Public License v3 or later (GPLv3+)": "GPL-3+", 29 | "GNU Lesser General Public License v2 (LGPLv2)": "LGPL-2", 30 | "GNU Lesser General Public License v2 or later (LGPLv2+)": "LGPL-2+", 31 | "GNU Lesser General Public License v3 (LGPLv3)": "LGPL-3", 32 | "GNU Lesser General Public License v3 or later (LGPLv3+)": "LGPL-3+", 33 | "GNU Library or Lesser General Public License (LGPL)": "LGPL-2+", 34 | "ISC License (ISCL)": "ISC", 35 | "MIT License": "MIT", 36 | "Mozilla Public License 1.0 (MPL)": "MPL-1.0", 37 | "Mozilla Public License 1.1 (MPL 1.1)": "MPL-1.1", 38 | "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", 39 | "Public Domain": "public-domain", 40 | "Python License (CNRI Python License)": "CNRI", 41 | "Python Software Foundation License": "PYTHON", 42 | "Repoze Public License": "repoze", 43 | "W3C License": "W3C", 44 | "Zope Public License": "ZPL", 45 | "zlib/libpng License": "ZLIB" 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /gs_pypi/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | -------------------------------------------------------------------------------- /gs_pypi/backend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | backend.py 6 | ~~~~~~~~~~ 7 | 8 | PyPI backend 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | 16 | from g_sorcery.backend import Backend 17 | from g_sorcery.metadata import MetadataGenerator 18 | from g_sorcery.eclass import EclassGenerator 19 | from g_sorcery.fileutils import get_pkgpath 20 | 21 | from .pypi_db import PypiDBGenerator 22 | from .ebuild import PypiEbuildWithoutDigestGenerator, PypiEbuildWithDigestGenerator 23 | 24 | 25 | class PypiEclassGenerator(EclassGenerator): 26 | """ 27 | Implementation of eclass generator. Only specifies a data directory. 28 | """ 29 | def __init__(self): 30 | super(PypiEclassGenerator, self).__init__(os.path.join(get_pkgpath(__file__), 'data')) 31 | 32 | 33 | instance = Backend(PypiDBGenerator, 34 | PypiEbuildWithDigestGenerator, PypiEbuildWithoutDigestGenerator, 35 | PypiEclassGenerator, MetadataGenerator, sync_db=True) 36 | -------------------------------------------------------------------------------- /gs_pypi/data/gs-pypi.eclass: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2014 Gentoo Foundation 2 | # Distributed under the terms of the GNU General Public License v2 3 | # $Header: $ 4 | # automatically generated by gs-pypi 5 | # please do not edit this file 6 | # 7 | # Original Author: Jauhien Piatlicki 8 | # Purpose: support installation of python packages from PyPI repo 9 | # 10 | # Bugs to jauhien@gentoo.org 11 | # 12 | # @ECLASS: gs-pypi.eclass 13 | # 14 | 15 | inherit distutils-r1 g-sorcery 16 | -------------------------------------------------------------------------------- /gs_pypi/ebuild.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | ebuild.py 6 | ~~~~~~~~~ 7 | 8 | ebuild generation 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import collections 15 | import os 16 | 17 | from g_sorcery.ebuild import DefaultEbuildGenerator 18 | 19 | Layout = collections.namedtuple("Layout", 20 | ["vars_before_inherit", "inherit", "vars_after_description", "vars_after_keywords"]) 21 | 22 | 23 | class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator): 24 | """ 25 | Implementation of ebuild generator without sources digesting. 26 | """ 27 | def __init__(self, package_db): 28 | 29 | vars_before_inherit = \ 30 | ["realname", "realversion", 31 | {"name" : "repo_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/'}, 32 | {"name" : "sourcefile", "value" : '${REALNAME}-${REALVERSION}.tar.gz'}, {"name" : "python_compat", "raw" : True}] 33 | 34 | inherit = ["gs-pypi"] 35 | 36 | vars_after_description = \ 37 | ["homepage", "license"] 38 | 39 | vars_after_keywords = \ 40 | [] 41 | 42 | layout = Layout(vars_before_inherit, inherit, vars_after_description, vars_after_keywords) 43 | 44 | super(PypiEbuildWithoutDigestGenerator, self).__init__(package_db, layout) 45 | 46 | class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator): 47 | """ 48 | Implementation of ebuild generator with sources digesting. 49 | """ 50 | def __init__(self, package_db): 51 | 52 | vars_before_inherit = \ 53 | ["realname", "realversion", 54 | {"name" : "digest_sources", "value" : "yes"}, {"name" : "python_compat", "raw" : True}] 55 | 56 | inherit = ["gs-pypi"] 57 | 58 | vars_after_description = \ 59 | ["homepage", "license", 60 | {"name" : "src_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/${REALNAME}-${REALVERSION}.tar.gz'}] 61 | 62 | vars_after_keywords = \ 63 | [] 64 | 65 | layout = Layout(vars_before_inherit, inherit, vars_after_description, vars_after_keywords) 66 | 67 | super(PypiEbuildWithDigestGenerator, self).__init__(package_db, layout) 68 | -------------------------------------------------------------------------------- /gs_pypi/gs_pypi_generate_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | pypi_db.py 6 | ~~~~~~~~~~ 7 | 8 | PyPI database generation 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import argparse 15 | import os 16 | import sys 17 | 18 | from g_sorcery.compatibility import TemporaryDirectory 19 | from g_sorcery.exceptions import FileJSONError 20 | from g_sorcery.fileutils import copy_all, FileJSON 21 | from g_sorcery.logger import Logger 22 | 23 | from .pypi_db import PypiDBGenerator 24 | 25 | def main(): 26 | parser = argparse.ArgumentParser(description='Package DB generator for gs-pypi.') 27 | parser.add_argument('db_dirname', help='directory to store DB') 28 | parser.add_argument('-c', '--count', help='count of records that should be processed', 29 | default=None) 30 | parser.add_argument('--layout-version', help='DB layout version', default='1') 31 | parser.add_argument('--structure-version', help='DB structure version', default='1') 32 | parser.add_argument('-f', '--fmt', help='packages file format (json or bson)', default='bson') 33 | 34 | args = parser.parse_args(sys.argv[1:]) 35 | 36 | db_name = args.db_dirname 37 | count = args.count 38 | layout_version = int(args.layout_version) 39 | structure_version = int(args.structure_version) 40 | fmt = args.fmt 41 | if count: 42 | count = int(count) 43 | 44 | logger = Logger() 45 | cfg_path = None 46 | for path in '.', '~', '/etc/g-sorcery': 47 | current = os.path.join(path, "gs-pypi.json") 48 | if (os.path.isfile(current)): 49 | cfg_path = path 50 | break 51 | if not cfg_path: 52 | logger.error('no config file for gs-pypi backend\n') 53 | return -1 54 | cfg_f = FileJSON(cfg_path, "gs-pypi.json", ['package']) 55 | try: 56 | config = cfg_f.read() 57 | except FileJSONError as e: 58 | logger.error('error loading config file for gs-pypi: ' + str(e) + '\n') 59 | return -1 60 | 61 | generator = PypiDBGenerator(preferred_layout_version=layout_version, 62 | preferred_db_version=structure_version, 63 | preferred_category_format=fmt, 64 | count=count) 65 | temp_dir = TemporaryDirectory() 66 | pkg_db = generator(temp_dir.name, "pypi", 67 | config=config["repositories"]["pypi"], 68 | common_config=config["common_config"]) 69 | if os.path.exists(db_name): 70 | os.system('rm -rf ' + db_name + '/*') 71 | else: 72 | os.mkdir(db_name) 73 | copy_all(os.path.join(temp_dir.name, "pypi/db"), db_name) 74 | os.system('tar cvzf ' + db_name + '.tar.gz ' + db_name) 75 | 76 | if __name__ == "__main__": 77 | sys.exit(main()) 78 | -------------------------------------------------------------------------------- /gs_pypi/pypi_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | pypi_db.py 6 | ~~~~~~~~~~ 7 | 8 | PyPI package database 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import datetime 15 | import re 16 | import time 17 | 18 | import bs4 19 | 20 | from g_sorcery.db_layout import BSON_FILE_SUFFIX 21 | from g_sorcery.exceptions import DownloadingError 22 | from g_sorcery.g_collections import Package, serializable_elist 23 | from g_sorcery.package_db import DBGenerator, PackageDB 24 | 25 | class PypiDBGenerator(DBGenerator): 26 | """ 27 | Implementation of database generator for PYPI backend. 28 | """ 29 | 30 | def __init__(self, package_db_class=PackageDB, 31 | preferred_layout_version=1, 32 | preferred_db_version=1, 33 | preferred_category_format=BSON_FILE_SUFFIX, 34 | count=None): 35 | super(PypiDBGenerator, self).__init__(package_db_class=package_db_class, 36 | preferred_layout_version=preferred_layout_version, 37 | preferred_db_version=preferred_db_version, 38 | preferred_category_format=preferred_category_format) 39 | self.count = count 40 | 41 | def get_download_uries(self, common_config, config): 42 | """ 43 | Get URI of packages index. 44 | """ 45 | self.repo_uri = config["repo_uri"] 46 | return [{"uri": self.repo_uri + "pypi?%3Aaction=index", "output": "packages"}] 47 | 48 | def parse_data(self, data_f): 49 | """ 50 | Download and parse packages index. Then download and parse pages for all packages. 51 | """ 52 | soup = bs4.BeautifulSoup(data_f.read()) 53 | packages = soup.table 54 | data = {} 55 | data["index"] = {} 56 | 57 | pkg_uries = [] 58 | 59 | last = -1 60 | if self.count: 61 | last = self.count 62 | for entry in packages.find_all("tr")[1:last]: 63 | package, description = entry.find_all("td") 64 | 65 | if description.contents: 66 | description = description.contents[0] 67 | else: 68 | description = "" 69 | package, version = package.a["href"].split("/")[2:] 70 | data["index"][(package, version)] = description 71 | pkg_uries.append({"uri": self.repo_uri + "pypi/" + package + "/" + version, 72 | "parser": self.parse_package_page, 73 | "output": package + "-" + version, 74 | "timeout": 2}) 75 | entry.decompose() 76 | 77 | packages.decompose() 78 | soup.decompose() 79 | 80 | pkg_uries = self.decode_download_uries(pkg_uries) 81 | if self.count: 82 | pkg_uries = pkg_uries[:self.count] 83 | for uri in pkg_uries: 84 | attempts = 0 85 | while True: 86 | try: 87 | attempts += 1 88 | self.process_uri(uri, data) 89 | except DownloadingError as error: 90 | print(str(error)) 91 | time.sleep(5) 92 | if attempts < 100: 93 | continue 94 | break 95 | 96 | return data 97 | 98 | def parse_package_page(self, data_f): 99 | """ 100 | Parse package page. 101 | """ 102 | soup = bs4.BeautifulSoup(data_f.read()) 103 | data = {} 104 | data["files"] = [] 105 | data["info"] = {} 106 | try: 107 | for table in soup("table", class_ = "list")[-1:]: 108 | if not "File" in table("th")[0].string: 109 | continue 110 | 111 | for entry in table("tr")[1:-1]: 112 | fields = entry("td") 113 | 114 | FILE = 0 115 | URL = 0 116 | MD5 = 1 117 | 118 | TYPE = 1 119 | PYVERSION = 2 120 | UPLOADED = 3 121 | SIZE = 4 122 | 123 | file_inf = fields[FILE]("a")[0]["href"].split("#") 124 | file_url = file_inf[URL] 125 | file_md5 = file_inf[MD5][4:] 126 | 127 | file_type = fields[TYPE].string 128 | file_pyversion = fields[PYVERSION].string 129 | file_uploaded = fields[UPLOADED].string 130 | file_size = fields[SIZE].string 131 | 132 | data["files"].append({"url": file_url, 133 | "md5": file_md5, 134 | "type": file_type, 135 | "pyversion": file_pyversion, 136 | "uploaded": file_uploaded, 137 | "size": file_size}) 138 | entry.decompose() 139 | table.decompose() 140 | 141 | uls = soup("ul", class_ = "nodot") 142 | if uls: 143 | if "Downloads (All Versions):" in uls[0]("strong")[0].string: 144 | ul = uls[1] 145 | else: 146 | ul = uls[0] 147 | 148 | for entry in ul.contents: 149 | if not hasattr(entry, "name") or entry.name != "li": 150 | continue 151 | entry_name = entry("strong")[0].string 152 | if not entry_name: 153 | continue 154 | 155 | if entry_name == "Categories": 156 | data["info"][entry_name] = {} 157 | for cat_entry in entry("a"): 158 | cat_data = cat_entry.string.split(" :: ") 159 | if not cat_data[0] in data["info"][entry_name]: 160 | data["info"][entry_name][cat_data[0]] = cat_data[1:] 161 | else: 162 | data["info"][entry_name][cat_data[0]].extend(cat_data[1:]) 163 | continue 164 | 165 | if entry("span"): 166 | data["info"][entry_name] = entry("span")[0].string 167 | continue 168 | 169 | if entry("a"): 170 | data["info"][entry_name] = entry("a")[0]["href"] 171 | continue 172 | entry.decompose() 173 | ul.decompose() 174 | 175 | except Exception as error: 176 | print("There was an error during parsing: " + str(error)) 177 | print("Ignoring this package.") 178 | data = {} 179 | data["files"] = [] 180 | data["info"] = {} 181 | 182 | soup.decompose() 183 | return data 184 | 185 | def process_data(self, pkg_db, data, common_config, config): 186 | """ 187 | Process parsed package data. 188 | """ 189 | category = "dev-python" 190 | pkg_db.add_category(category) 191 | 192 | common_data = {} 193 | common_data["eclasses"] = ['g-sorcery', 'gs-pypi'] 194 | common_data["maintainer"] = [{'email' : 'jauhien@gentoo.org', 195 | 'name' : 'Jauhien Piatlicki'}] 196 | common_data["dependencies"] = serializable_elist(separator="\n\t") 197 | pkg_db.set_common_data(category, common_data) 198 | 199 | #todo: write filter functions 200 | allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ 201 | set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, 202 | ['+', '_', '-']))) 203 | 204 | allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ 205 | set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, 206 | ['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ',']))) 207 | 208 | now = datetime.datetime.now() 209 | pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day) 210 | 211 | for (package, version), description in data["packages"]["index"].items(): 212 | 213 | pkg = package + "-" + version 214 | if not pkg in data["packages"]: 215 | continue 216 | 217 | pkg_data = data["packages"][pkg] 218 | 219 | if not pkg_data["files"] and not pkg_data["info"]: 220 | continue 221 | 222 | files_src_uri = "" 223 | md5 = "" 224 | if pkg_data["files"]: 225 | for file_entry in pkg_data["files"]: 226 | if file_entry["type"] == "\n Source\n ": 227 | files_src_uri = file_entry["url"] 228 | md5 = file_entry["md5"] 229 | break 230 | 231 | download_url = "" 232 | info = pkg_data["info"] 233 | if info: 234 | if "Download URL:" in info: 235 | download_url = info["Download URL:"] 236 | 237 | if download_url: 238 | source_uri = download_url #todo: find how to define src_uri 239 | else: 240 | source_uri = files_src_uri 241 | 242 | if not source_uri: 243 | continue 244 | 245 | homepage = "" 246 | pkg_license = "" 247 | py_versions = [] 248 | if info: 249 | if "Home Page:" in info: 250 | homepage = info["Home Page:"] 251 | categories = {} 252 | if "Categories" in info: 253 | categories = info["Categories"] 254 | 255 | if 'Programming Language' in categories: 256 | for entry in categories['Programming Language']: 257 | if entry == '2': 258 | py_versions.extend(['2_7']) 259 | elif entry == '3': 260 | py_versions.extend(['3_3', '3_4', '3_5']) 261 | elif entry == '2.6': 262 | py_versions.extend(['2_7']) 263 | elif entry == '2.7': 264 | py_versions.extend(['2_7']) 265 | elif entry == '3.2': 266 | py_versions.extend(['3_3']) 267 | elif entry == '3.3': 268 | py_versions.extend(['3_3']) 269 | elif entry == '3.4': 270 | py_versions.extend(['3_4']) 271 | elif entry == '3.5': 272 | py_versions.extend(['3_5']) 273 | 274 | 275 | if "License" in categories: 276 | pkg_license = categories["License"][-1] 277 | pkg_license = self.convert([common_config, config], "licenses", pkg_license) 278 | 279 | if not py_versions: 280 | py_versions = ['2_7', '3_3', '3_4', '3_5'] 281 | 282 | if len(py_versions) == 1: 283 | python_compat = '( python' + py_versions[0] + ' )' 284 | else: 285 | python_compat = '( python{' + py_versions[0] 286 | for ver in py_versions[1:]: 287 | python_compat += ',' + ver 288 | python_compat += '} )' 289 | 290 | filtered_package = "".join([x for x in package if ord(x) in allowed_ords_pkg]) 291 | description = "".join([x for x in description if ord(x) in allowed_ords_desc]) 292 | filtered_version = version 293 | match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)", 294 | filtered_version) 295 | if not match_object: 296 | filtered_version = pseudoversion 297 | 298 | ebuild_data = {} 299 | ebuild_data["realname"] = package 300 | ebuild_data["realversion"] = version 301 | 302 | ebuild_data["description"] = description 303 | ebuild_data["longdescription"] = description 304 | 305 | ebuild_data["homepage"] = homepage 306 | ebuild_data["license"] = pkg_license 307 | ebuild_data["source_uri"] = source_uri 308 | ebuild_data["md5"] = md5 309 | ebuild_data["python_compat"] = python_compat 310 | 311 | pkg_db.add_package(Package(category, filtered_package, filtered_version), ebuild_data) 312 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | setup.py 6 | ~~~~~~~~ 7 | 8 | installation script 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | from distutils.core import setup 15 | 16 | setup(name = 'gs-pypi', 17 | version = '0.2.1', 18 | description = 'g-sorcery backend for pypi packages', 19 | author = 'Jauhien Piatlicki', 20 | author_email = 'jauhien@gentoo.org', 21 | packages = ['gs_pypi'], 22 | package_data = {'gs_pypi': ['data/*']}, 23 | scripts = ['bin/gs-pypi-generate-db', 'bin/gs-pypi'], 24 | data_files = [('/etc/g-sorcery/', ['gs-pypi.json']), 25 | ('/etc/layman/overlays/', ['gs-pypi-overlays.xml'])], 26 | license = 'GPL-2', 27 | ) 28 | --------------------------------------------------------------------------------